group_orchestrator.py 141 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602
  1. """Async scheduler for durable YaoYao group Agent runs.
  2. The store is the concurrency authority. This module only owns process-local
  3. Gateway generations and moves every blocking SQLite/Gateway call off the
  4. Dashboard event loop.
  5. """
  6. from __future__ import annotations
  7. import asyncio
  8. from contextlib import AsyncExitStack, asynccontextmanager
  9. from dataclasses import dataclass, field
  10. import json
  11. import logging
  12. import math
  13. import re
  14. import threading
  15. import uuid
  16. from typing import Any, Awaitable, Callable, Mapping, Protocol
  17. from .group_gateway import GroupGatewayAdapter, SessionIdentity
  18. from .group_protocol import MAX_MESSAGE_BYTES, MAX_TOOL_STATE_BYTES
  19. logger = logging.getLogger(__name__)
  20. DEFAULT_EVENT_QUEUE_SIZE = 256
  21. DEFAULT_COMPLETION_TIMEOUT = 15 * 60.0
  22. STREAM_FLUSH_INTERVAL = 0.05
  23. _ERROR_TEXT_LIMIT = 4096
  24. _TOOL_VALUE_LIMIT = 16 * 1024
  25. _TOOL_TEXT_LIMIT = 4096
  26. _TOOL_FINDING_LIMIT = 32
  27. _TOOL_COUNT_LIMIT = 128
  28. _TOOL_TRUNCATION_ID = "__hermes_group_tool_state_truncated__"
  29. _TOOL_STATE_INTERNAL_LIMIT = min(MAX_TOOL_STATE_BYTES, 192 * 1024)
  30. _PERSISTED_EVENTS = frozenset(
  31. {
  32. "message.start",
  33. "message.delta",
  34. "message.complete",
  35. "reasoning.delta",
  36. "tool.start",
  37. "tool.complete",
  38. "tool.output_risk",
  39. "approval.request",
  40. "clarify.request",
  41. "clarify.expire",
  42. "session.info",
  43. "error",
  44. }
  45. )
  46. _TERMINAL_RUN_STATUSES = frozenset({"completed", "failed", "interrupted"})
  47. _APPROVAL_CHOICES = frozenset({"once", "session", "always", "deny"})
  48. _STORE_RETRY_DELAYS = (0.01, 0.02, 0.04)
  49. _CLAIM_RETRY_MAX_SECONDS = 1.0
  50. _WORK_DISABLED_RETRY_SECONDS = 1.0
  51. _NO_REPLY_TOKEN = "[[YAOYAO_NO_REPLY_V1]]"
  52. _NO_REPLY_RESERVED_PATTERN = re.compile(r"\[\[YAOYAO_[A-Z0-9_]*(?:\]\])?")
  53. _HOST_FALLBACK_REPLY = (
  54. "我还不能确定你希望我处理什么,请补充具体目标、范围,或明确需要我协调的 Agent。"
  55. )
  56. def _is_no_reply_content(value: object) -> bool:
  57. return isinstance(value, str) and value.strip() == _NO_REPLY_TOKEN
  58. def _sanitize_no_reply_text(value: str) -> str:
  59. return _NO_REPLY_RESERVED_PATTERN.sub("", value)
  60. def _log_failure_type(message: str, error: BaseException) -> None:
  61. """Log stable context without persisting Gateway or Store exception text."""
  62. logger.error("%s [errorType=%s]", message, type(error).__name__)
  63. class GroupOrchestratorError(RuntimeError):
  64. """Raised for an invalid scheduler lifecycle or local runtime collision."""
  65. class _Store(Protocol):
  66. def initialize(self) -> None: ...
  67. def recover_after_restart(self) -> list[str]: ...
  68. def claim_next_runnable_run(self) -> dict[str, object] | None: ...
  69. def read_run_projection(self, run_id: str) -> dict[str, object]: ...
  70. def prepare_run_session_configuration(
  71. self, run_id: str, configuration: Mapping[str, object]
  72. ) -> bool: ...
  73. def bind_run_runtime(
  74. self, run_id: str, runtime_session_id: str
  75. ) -> dict[str, object]: ...
  76. def commit_prompt_submission(
  77. self, run_id: str, **kwargs: object
  78. ) -> dict[str, object]: ...
  79. def upsert_agent_message(
  80. self, run_id: str, **kwargs: object
  81. ) -> dict[str, object]: ...
  82. def settle_run(self, run_id: str, **kwargs: object) -> dict[str, object]: ...
  83. def transition_run(
  84. self, run_id: str, status: str, **kwargs: object
  85. ) -> dict[str, object]: ...
  86. def get_run(self, run_id: str) -> dict[str, object]: ...
  87. def get_message(self, message_id: str) -> dict[str, object]: ...
  88. def get_room(self, room_id: str) -> dict[str, object]: ...
  89. def get_interaction(self, interaction_id: str) -> dict[str, object]: ...
  90. def list_pending_cascades(self, **kwargs: object) -> dict[str, object]: ...
  91. def complete_cascade(self, source_run_id: str) -> dict[str, object]: ...
  92. def interrupt_agent_with_runtime_targets(
  93. self, room_id: str, agent_id: str, command: Mapping[str, object]
  94. ) -> dict[str, object]: ...
  95. def archive_room_with_runtime_targets(
  96. self, room_id: str, command: Mapping[str, object]
  97. ) -> dict[str, object]: ...
  98. def create_room(self, command: Mapping[str, object]) -> dict[str, object]: ...
  99. def update_room(
  100. self, room_id: str, command: Mapping[str, object]
  101. ) -> dict[str, object]: ...
  102. def update_topic(
  103. self, room_id: str, topic_id: str, command: Mapping[str, object]
  104. ) -> dict[str, object]: ...
  105. def mark_topic_read(
  106. self, room_id: str, topic_id: str, command: Mapping[str, object]
  107. ) -> dict[str, object]: ...
  108. def add_agent(
  109. self, room_id: str, command: Mapping[str, object]
  110. ) -> dict[str, object]: ...
  111. def update_agent_with_runtime_targets(
  112. self,
  113. room_id: str,
  114. agent_id: str,
  115. command: Mapping[str, object],
  116. ) -> dict[str, object]: ...
  117. def delete_agent_with_runtime_targets(
  118. self,
  119. room_id: str,
  120. agent_id: str,
  121. command: Mapping[str, object],
  122. ) -> dict[str, object]: ...
  123. def create_human_message(
  124. self,
  125. room_id: str,
  126. *,
  127. request_id: str,
  128. client_message_id: str,
  129. content: str,
  130. mention_agent_ids: list[str],
  131. topic_id: str | None = None,
  132. ) -> dict[str, object]: ...
  133. def create_gateway_interaction(
  134. self, run_id: str, **kwargs: object
  135. ) -> dict[str, object]: ...
  136. def expire_interaction(self, interaction_id: str) -> dict[str, object]: ...
  137. def begin_interaction_response(
  138. self, room_id: str, interaction_id: str, **kwargs: object
  139. ) -> dict[str, object]: ...
  140. def finish_interaction_response(
  141. self, request_id: str, **kwargs: object
  142. ) -> dict[str, object]: ...
  143. def fail_interaction_response(
  144. self, request_id: str, **kwargs: object
  145. ) -> dict[str, object]: ...
  146. class _Gateway(Protocol):
  147. transport: Any
  148. def create_session(
  149. self,
  150. profile: str,
  151. title: str,
  152. cwd: str,
  153. seed_messages: list[dict[str, Any]],
  154. configuration: dict[str, Any] | None = None,
  155. ) -> SessionIdentity: ...
  156. def resume_session(self, profile: str, stored_id: str) -> SessionIdentity: ...
  157. def submit_prompt(self, runtime_id: str, text: str) -> None: ...
  158. def interrupt(self, runtime_id: str) -> None: ...
  159. def close(self, runtime_id: str) -> bool: ...
  160. def respond_approval(self, runtime_id: str, choice: str) -> int: ...
  161. def respond_clarification(self, request_id: str, answer: str) -> str: ...
  162. def shutdown(self) -> None: ...
  163. GatewayFactory = Callable[[Callable[[str, str, dict[str, object]], None]], _Gateway]
  164. @dataclass(frozen=True)
  165. class _GatewayEvent:
  166. runtime_id: str
  167. generation: int
  168. event_type: str
  169. payload: dict[str, object]
  170. @dataclass
  171. class _RuntimeState:
  172. run_id: str
  173. room_id: str
  174. agent_id: str
  175. runtime_id: str
  176. generation: int
  177. expected_stored_id: str | None
  178. session_stored_id: str
  179. expected_context_seq: int
  180. through_seq: int
  181. profile: str = ""
  182. reply_mode: str = "mentioned"
  183. required_reply: bool = False
  184. automatic_published: bool = False
  185. had_visible_interaction: bool = False
  186. prompt_ready: asyncio.Event = field(default_factory=asyncio.Event)
  187. finished: asyncio.Event = field(default_factory=asyncio.Event)
  188. terminal_info_received: asyncio.Event = field(default_factory=asyncio.Event)
  189. finalize_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
  190. mutation_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
  191. pending_events: asyncio.Queue[_GatewayEvent | None] = field(
  192. default_factory=lambda: asyncio.Queue(maxsize=64)
  193. )
  194. event_task: asyncio.Task[None] | None = None
  195. grace_task: asyncio.Task[None] | None = None
  196. inflight_calls: set[asyncio.Task[object]] = field(default_factory=set)
  197. mutation_tasks: set[asyncio.Task[dict[str, object]]] = field(default_factory=set)
  198. content: str = ""
  199. reasoning: str = ""
  200. tool_state: list[dict[str, object]] = field(default_factory=list)
  201. tool_indexes: dict[str, int] = field(default_factory=dict)
  202. clarification_ids: dict[str, str] = field(default_factory=dict)
  203. pending_interaction_ids: set[str] = field(default_factory=set)
  204. stream_dirty: bool = False
  205. last_stream_flush: float = 0.0
  206. approval_ordinal: int = 0
  207. complete_seen: bool = False
  208. complete_index: int = 0
  209. event_index: int = 0
  210. terminal_status: str | None = None
  211. terminal_error: str = ""
  212. actual_model: str | None = None
  213. actual_provider: str | None = None
  214. actual_reasoning_effort: str | None = None
  215. actual_fast_mode: bool | None = None
  216. settle_error: BaseException | None = None
  217. prompt_committed: bool = False
  218. overflowed: bool = False
  219. finalizing: bool = False
  220. @dataclass
  221. class _AgentLockState:
  222. lock: asyncio.Lock = field(default_factory=asyncio.Lock)
  223. users: int = 0
  224. def _required_mapping(value: object, field_name: str) -> Mapping[str, object]:
  225. if not isinstance(value, Mapping):
  226. raise GroupOrchestratorError(f"{field_name} must be an object")
  227. return value
  228. def _required_string(value: object, field_name: str) -> str:
  229. if not isinstance(value, str) or not value:
  230. raise GroupOrchestratorError(f"{field_name} must be a non-empty string")
  231. return value
  232. def _required_string_allowing_empty(value: object, field_name: str) -> str:
  233. if not isinstance(value, str):
  234. raise GroupOrchestratorError(f"{field_name} must be a string")
  235. return value
  236. def _optional_string(value: object, field_name: str) -> str | None:
  237. if value is None:
  238. return None
  239. return _required_string(value, field_name)
  240. def _required_nonnegative_int(value: object, field_name: str) -> int:
  241. if isinstance(value, bool) or not isinstance(value, int) or value < 0:
  242. raise GroupOrchestratorError(f"{field_name} must be a non-negative integer")
  243. return value
  244. def _required_bool(value: object, field_name: str) -> bool:
  245. if not isinstance(value, bool):
  246. raise GroupOrchestratorError(f"{field_name} must be a boolean")
  247. return value
  248. def _strict_json_copy(value: object, field_name: str) -> object:
  249. try:
  250. encoded = json.dumps(
  251. value,
  252. ensure_ascii=False,
  253. sort_keys=True,
  254. separators=(",", ":"),
  255. allow_nan=False,
  256. )
  257. encoded.encode("utf-8")
  258. return json.loads(encoded)
  259. except (TypeError, ValueError, UnicodeEncodeError, json.JSONDecodeError) as error:
  260. raise GroupOrchestratorError(f"{field_name} must be strict JSON") from error
  261. def build_run_prompt(
  262. projection: Mapping[str, object],
  263. room_agents: list[Mapping[str, object]] | None = None,
  264. ) -> str:
  265. """Build a deterministic prompt from the Store's bounded projection."""
  266. run = _required_mapping(projection.get("run"), "projection.run")
  267. room = _required_mapping(projection.get("room"), "projection.room")
  268. agent = _required_mapping(projection.get("agent"), "projection.agent")
  269. messages = projection.get("messages")
  270. if not isinstance(messages, list) or not all(
  271. isinstance(message, Mapping) for message in messages
  272. ):
  273. raise GroupOrchestratorError("projection.messages must be a list of objects")
  274. envelope = {
  275. "agent": {
  276. "displayName": _required_string(
  277. agent.get("displayName"), "projection.agent.displayName"
  278. ),
  279. "id": _required_string(agent.get("id"), "projection.agent.id"),
  280. "description": _required_string_allowing_empty(
  281. agent.get("description", ""), "projection.agent.description"
  282. ),
  283. },
  284. "messages": _strict_json_copy(messages, "projection.messages"),
  285. "omittedSummary": _strict_json_copy(
  286. projection.get("omittedSummary"), "projection.omittedSummary"
  287. ),
  288. "room": {
  289. "id": _required_string(room.get("id"), "projection.room.id"),
  290. "name": _required_string(room.get("name"), "projection.room.name"),
  291. "maxReplyRounds": room.get("maxReplyRounds", 3),
  292. },
  293. "run": {
  294. "id": _required_string(run.get("id"), "projection.run.id"),
  295. "topicId": _required_string(
  296. run.get("topicId"), "projection.run.topicId"
  297. ),
  298. "rootMessageId": _required_string(
  299. run.get("rootMessageId"), "projection.run.rootMessageId"
  300. ),
  301. "triggerMessageId": _required_string(
  302. run.get("triggerMessageId"), "projection.run.triggerMessageId"
  303. ),
  304. "replyMode": _required_string(
  305. run.get("replyMode", "mentioned"), "projection.run.replyMode"
  306. ),
  307. "requiredReply": _required_bool(
  308. run.get("requiredReply", False), "projection.run.requiredReply"
  309. ),
  310. },
  311. "throughSeq": _required_nonnegative_int(
  312. projection.get("throughSeq"), "projection.throughSeq"
  313. ),
  314. "version": 1,
  315. }
  316. members: list[dict[str, str]] = []
  317. for candidate in room_agents or []:
  318. if candidate.get("enabled") is not True:
  319. continue
  320. members.append(
  321. {
  322. "displayName": _required_string(
  323. candidate.get("displayName"), "roomAgent.displayName"
  324. ),
  325. "id": _required_string(candidate.get("id"), "roomAgent.id"),
  326. }
  327. )
  328. members.sort(key=lambda item: (item["displayName"].casefold(), item["id"]))
  329. envelope["roomAgents"] = members
  330. encoded = json.dumps(
  331. envelope,
  332. ensure_ascii=False,
  333. sort_keys=True,
  334. separators=(",", ":"),
  335. allow_nan=False,
  336. )
  337. automatic_rule = ""
  338. if envelope["run"]["requiredReply"]:
  339. automatic_rule = (
  340. "你是本房间唯一主持人。该用户消息没有有效 @,你必须公开处理;"
  341. "禁止保持静默,禁止输出任何 YAOYAO_NO_REPLY 标记。必须三选一:"
  342. "直接回答;使用成员列表中的精确 @显示名 转派;或发起澄清请求。"
  343. )
  344. elif envelope["run"]["replyMode"] == "automatic":
  345. automatic_rule = (
  346. f"若触发内容与自己的职责无关,禁止调用工具、禁止@成员,且完整答复只能是"
  347. f"{_NO_REPLY_TOKEN};正常答复绝不能包含该标记。"
  348. )
  349. return (
  350. "你正在 Hermes 多设备群聊中回复。只输出要发送到公共房间的答复;"
  351. "仅在确实需要成员协作时,使用成员列表中的精确 @显示名 或 @all 发起下一轮。\n"
  352. f"{automatic_rule}\n"
  353. f"runId={envelope['run']['id']} roomId={envelope['room']['id']} "
  354. f"agentId={envelope['agent']['id']}\n"
  355. f"GROUP_CONTEXT_JSON={encoded}"
  356. )
  357. class GroupOrchestrator:
  358. """Drive Store-claimed runs through one process-local Gateway adapter."""
  359. def __init__(
  360. self,
  361. store: _Store,
  362. *,
  363. gateway_factory: GatewayFactory | None = None,
  364. work_enabled: Callable[[], bool] | None = None,
  365. event_queue_size: int = DEFAULT_EVENT_QUEUE_SIZE,
  366. completion_timeout: float = DEFAULT_COMPLETION_TIMEOUT,
  367. ) -> None:
  368. if (
  369. isinstance(event_queue_size, bool)
  370. or not isinstance(event_queue_size, int)
  371. or event_queue_size < 1
  372. ):
  373. raise ValueError("event_queue_size must be a positive integer")
  374. if (
  375. isinstance(completion_timeout, bool)
  376. or not isinstance(completion_timeout, (int, float))
  377. or not math.isfinite(completion_timeout)
  378. or completion_timeout <= 0
  379. ):
  380. raise ValueError("completion_timeout must be positive")
  381. self.store = store
  382. self._gateway_factory: GatewayFactory = (
  383. gateway_factory
  384. if gateway_factory is not None
  385. else lambda callback: GroupGatewayAdapter(on_event=callback)
  386. )
  387. if work_enabled is not None and not callable(work_enabled):
  388. raise ValueError("work_enabled must be callable")
  389. self._work_enabled = work_enabled if work_enabled is not None else lambda: True
  390. self._event_queue_size = event_queue_size
  391. self._completion_timeout = float(completion_timeout)
  392. self._loop: asyncio.AbstractEventLoop | None = None
  393. self._loop_thread_id: int | None = None
  394. self._gateway: _Gateway | None = None
  395. self._event_queue: asyncio.Queue[_GatewayEvent | None] | None = None
  396. self._event_task: asyncio.Task[None] | None = None
  397. self._scheduler_task: asyncio.Task[None] | None = None
  398. self._run_tasks: set[asyncio.Task[None]] = set()
  399. self._control_tasks: set[asyncio.Task[dict[str, object]]] = set()
  400. self._wake_event: asyncio.Event | None = None
  401. self._shutdown_event: asyncio.Event | None = None
  402. self._shutdown_complete: asyncio.Event | None = None
  403. self._shutdown_error: BaseException | None = None
  404. self._durable_failures: list[GroupOrchestratorError] = []
  405. self._run_failures: list[BaseException] = []
  406. self._failure_lock = threading.Lock()
  407. self._lifecycle_lock: asyncio.Lock | None = None
  408. self._started = False
  409. self._closing = False
  410. self._scheduler_busy = False
  411. self._mapping_lock = threading.RLock()
  412. self._runtime_states: dict[str, tuple[int, _RuntimeState]] = {}
  413. self._next_generation = 0
  414. self._agent_locks: dict[str, _AgentLockState] = {}
  415. self._submission_locks: dict[str, _AgentLockState] = {}
  416. self._room_locks: dict[str, _AgentLockState] = {}
  417. @property
  418. def event_queue_capacity(self) -> int:
  419. return self._event_queue_size
  420. @property
  421. def active_run_count(self) -> int:
  422. return len(self._run_tasks)
  423. async def start(self) -> None:
  424. """Initialize/recover durable state before exposing any scheduler work."""
  425. if self._lifecycle_lock is None:
  426. self._lifecycle_lock = asyncio.Lock()
  427. async with self._lifecycle_lock:
  428. if self._started:
  429. return
  430. if self._closing:
  431. raise GroupOrchestratorError("Orchestrator is shutting down")
  432. self._loop = asyncio.get_running_loop()
  433. self._loop_thread_id = threading.get_ident()
  434. try:
  435. await self._run_owned_start_worker(
  436. self.store.initialize,
  437. task_name="yaoyao-group-store-initialize",
  438. )
  439. await self._run_owned_start_worker(
  440. self.store.recover_after_restart,
  441. task_name="yaoyao-group-store-recover",
  442. )
  443. self._event_queue = asyncio.Queue(maxsize=self._event_queue_size)
  444. self._wake_event = asyncio.Event()
  445. self._shutdown_event = asyncio.Event()
  446. self._shutdown_complete = asyncio.Event()
  447. self._shutdown_error = None
  448. self._durable_failures = []
  449. self._run_failures = []
  450. await self._run_owned_start_worker(
  451. self._gateway_factory,
  452. self._gateway_event_callback,
  453. task_name="yaoyao-group-gateway-create",
  454. adopt=lambda gateway: setattr(self, "_gateway", gateway),
  455. )
  456. self._event_task = asyncio.create_task(
  457. self._consume_events(), name="yaoyao-group-events"
  458. )
  459. await asyncio.sleep(0)
  460. self._scheduler_task = asyncio.create_task(
  461. self._scheduler(), name="yaoyao-group-scheduler"
  462. )
  463. await asyncio.sleep(0)
  464. self._started = True
  465. self.wake()
  466. except BaseException:
  467. await self._rollback_failed_start()
  468. raise
  469. async def _run_owned_start_worker(
  470. self,
  471. callback: Callable[..., object],
  472. *args: object,
  473. task_name: str,
  474. adopt: Callable[[object], None] | None = None,
  475. ) -> object:
  476. """Wait out cancellation so a startup worker cannot outlive ownership."""
  477. worker = asyncio.get_running_loop().create_task(
  478. asyncio.to_thread(callback, *args), name=task_name
  479. )
  480. cancellation: asyncio.CancelledError | None = None
  481. while not worker.done():
  482. try:
  483. await asyncio.shield(worker)
  484. except asyncio.CancelledError as error:
  485. if cancellation is None:
  486. cancellation = error
  487. try:
  488. result = worker.result()
  489. except BaseException:
  490. if cancellation is not None:
  491. raise cancellation
  492. raise
  493. if adopt is not None:
  494. adopt(result)
  495. if cancellation is not None:
  496. raise cancellation
  497. return result
  498. async def _rollback_failed_start(self) -> None:
  499. """Reverse a partial start without invoking normal settlement paths."""
  500. for task in (self._scheduler_task, self._event_task):
  501. if task is not None and not task.done():
  502. task.cancel()
  503. tasks = tuple(
  504. task
  505. for task in (self._scheduler_task, self._event_task)
  506. if task is not None
  507. )
  508. if tasks:
  509. await asyncio.gather(*tasks, return_exceptions=True)
  510. gateway = self._gateway
  511. if gateway is not None:
  512. drain: Callable[[float | None], object] | None = None
  513. try:
  514. transport = getattr(gateway, "transport", None)
  515. candidate = getattr(transport, "drain", None)
  516. if callable(candidate):
  517. drain = candidate
  518. except BaseException as error: # noqa: BLE001 - shutdown still runs
  519. _log_failure_type("Failed to inspect a partial Gateway startup", error)
  520. if callable(drain):
  521. try:
  522. await self._run_owned_start_worker(
  523. drain,
  524. None,
  525. task_name="yaoyao-group-gateway-rollback-drain",
  526. )
  527. except BaseException as error: # noqa: BLE001 - keep primary
  528. _log_failure_type(
  529. "Failed to drain a partial Gateway startup", error
  530. )
  531. try:
  532. await self._run_owned_start_worker(
  533. gateway.shutdown,
  534. task_name="yaoyao-group-gateway-rollback-shutdown",
  535. )
  536. except BaseException as error: # noqa: BLE001 - preserve primary
  537. _log_failure_type(
  538. "Failed to roll back YaoYao group Gateway startup", error
  539. )
  540. self._started = False
  541. self._gateway = None
  542. self._event_task = None
  543. self._scheduler_task = None
  544. self._event_queue = None
  545. self._wake_event = None
  546. self._shutdown_event = None
  547. self._shutdown_complete = None
  548. self._loop = None
  549. self._loop_thread_id = None
  550. def wake(self) -> None:
  551. """Idempotently signal that queued durable work may now be runnable."""
  552. loop = self._loop
  553. wake_event = self._wake_event
  554. if loop is None or wake_event is None or self._closing:
  555. return
  556. if threading.get_ident() == self._loop_thread_id:
  557. wake_event.set()
  558. return
  559. try:
  560. loop.call_soon_threadsafe(wake_event.set)
  561. except RuntimeError:
  562. return
  563. async def drain_once(self, *, timeout: float = 5.0) -> None:
  564. """Wake the scheduler and wait until its current durable work is idle."""
  565. if not self._started:
  566. raise GroupOrchestratorError("Orchestrator is not started")
  567. if timeout <= 0:
  568. raise ValueError("timeout must be positive")
  569. self.wake()
  570. async def wait_idle() -> None:
  571. while True:
  572. wake_event = self._wake_event
  573. event_queue = self._event_queue
  574. if (
  575. wake_event is not None
  576. and event_queue is not None
  577. and not wake_event.is_set()
  578. and not self._scheduler_busy
  579. and not self._run_tasks
  580. and event_queue.empty()
  581. ):
  582. return
  583. await asyncio.sleep(0.001)
  584. await asyncio.wait_for(wait_idle(), timeout=timeout)
  585. async def _tracked_control(
  586. self,
  587. operation: Callable[[], Awaitable[dict[str, object]]],
  588. *,
  589. task_name: str,
  590. failure_message: str,
  591. ) -> dict[str, object]:
  592. """Shield one accepted mutation and expose it to orderly shutdown."""
  593. if not self._started or self._closing:
  594. raise GroupOrchestratorError("Orchestrator is not accepting controls")
  595. task = asyncio.create_task(operation(), name=task_name)
  596. self._control_tasks.add(task)
  597. def finished(done: asyncio.Task[dict[str, object]]) -> None:
  598. self._control_tasks.discard(done)
  599. if done.cancelled():
  600. return
  601. try:
  602. error = done.exception()
  603. except asyncio.CancelledError:
  604. return
  605. if error is not None:
  606. _log_failure_type(failure_message, error)
  607. task.add_done_callback(finished)
  608. return await asyncio.shield(task)
  609. @staticmethod
  610. def _public_command(value: Mapping[str, object]) -> dict[str, object]:
  611. command = _strict_json_copy(value, "command")
  612. if not isinstance(command, dict):
  613. raise GroupOrchestratorError("command must be an object")
  614. _required_string(command.get("requestId"), "command.requestId")
  615. return command
  616. async def create_room(self, *, command: Mapping[str, object]) -> dict[str, object]:
  617. safe_command = self._public_command(command)
  618. request_id = str(safe_command["requestId"])
  619. return await self._tracked_control(
  620. lambda: self._create_room(safe_command),
  621. task_name=f"yaoyao-group-create-room-{request_id}",
  622. failure_message="YaoYao group room creation failed",
  623. )
  624. async def _create_room(self, command: Mapping[str, object]) -> dict[str, object]:
  625. result = await asyncio.to_thread(self.store.create_room, command)
  626. if not isinstance(result, Mapping):
  627. raise GroupOrchestratorError("Created room is invalid")
  628. self.wake()
  629. return dict(result)
  630. async def update_room(
  631. self, *, room_id: str, command: Mapping[str, object]
  632. ) -> dict[str, object]:
  633. _required_string(room_id, "roomId")
  634. safe_command = self._public_command(command)
  635. request_id = str(safe_command["requestId"])
  636. return await self._tracked_control(
  637. lambda: self._update_room(room_id, safe_command),
  638. task_name=f"yaoyao-group-update-room-{request_id}",
  639. failure_message="YaoYao group room update failed",
  640. )
  641. async def _update_room(
  642. self, room_id: str, command: Mapping[str, object]
  643. ) -> dict[str, object]:
  644. async with self._room_serial(room_id):
  645. result = await asyncio.to_thread(self.store.update_room, room_id, command)
  646. if not isinstance(result, Mapping):
  647. raise GroupOrchestratorError("Updated room is invalid")
  648. self.wake()
  649. return dict(result)
  650. async def update_topic(
  651. self, *, room_id: str, topic_id: str, command: Mapping[str, object]
  652. ) -> dict[str, object]:
  653. _required_string(room_id, "roomId")
  654. _required_string(topic_id, "topicId")
  655. safe_command = self._public_command(command)
  656. request_id = str(safe_command["requestId"])
  657. return await self._tracked_control(
  658. lambda: self._update_topic(room_id, topic_id, safe_command),
  659. task_name=f"yaoyao-group-update-topic-{request_id}",
  660. failure_message="YaoYao group topic update failed",
  661. )
  662. async def _update_topic(
  663. self, room_id: str, topic_id: str, command: Mapping[str, object]
  664. ) -> dict[str, object]:
  665. async with self._room_serial(room_id):
  666. result = await asyncio.to_thread(
  667. self.store.update_topic, room_id, topic_id, command
  668. )
  669. if not isinstance(result, Mapping):
  670. raise GroupOrchestratorError("Updated topic is invalid")
  671. self.wake()
  672. return dict(result)
  673. async def mark_topic_read(
  674. self, *, room_id: str, topic_id: str, command: Mapping[str, object]
  675. ) -> dict[str, object]:
  676. _required_string(room_id, "roomId")
  677. _required_string(topic_id, "topicId")
  678. safe_command = self._public_command(command)
  679. request_id = str(safe_command["requestId"])
  680. return await self._tracked_control(
  681. lambda: self._mark_topic_read(room_id, topic_id, safe_command),
  682. task_name=f"yaoyao-group-mark-topic-read-{request_id}",
  683. failure_message="YaoYao group topic read update failed",
  684. )
  685. async def _mark_topic_read(
  686. self, room_id: str, topic_id: str, command: Mapping[str, object]
  687. ) -> dict[str, object]:
  688. async with self._room_serial(room_id):
  689. result = await asyncio.to_thread(
  690. self.store.mark_topic_read, room_id, topic_id, command
  691. )
  692. if not isinstance(result, Mapping):
  693. raise GroupOrchestratorError("Topic read response is invalid")
  694. self.wake()
  695. return dict(result)
  696. async def add_agent(
  697. self, *, room_id: str, command: Mapping[str, object]
  698. ) -> dict[str, object]:
  699. _required_string(room_id, "roomId")
  700. safe_command = self._public_command(command)
  701. request_id = str(safe_command["requestId"])
  702. return await self._tracked_control(
  703. lambda: self._add_agent(room_id, safe_command),
  704. task_name=f"yaoyao-group-add-agent-{request_id}",
  705. failure_message="YaoYao group Agent creation failed",
  706. )
  707. async def _add_agent(
  708. self, room_id: str, command: Mapping[str, object]
  709. ) -> dict[str, object]:
  710. async with self._room_serial(room_id):
  711. result = await asyncio.to_thread(self.store.add_agent, room_id, command)
  712. if not isinstance(result, Mapping):
  713. raise GroupOrchestratorError("Created Agent is invalid")
  714. self.wake()
  715. return dict(result)
  716. async def update_agent(
  717. self,
  718. *,
  719. room_id: str,
  720. agent_id: str,
  721. command: Mapping[str, object],
  722. ) -> dict[str, object]:
  723. _required_string(room_id, "roomId")
  724. _required_string(agent_id, "agentId")
  725. safe_command = self._public_command(command)
  726. request_id = str(safe_command["requestId"])
  727. return await self._tracked_control(
  728. lambda: self._mutate_agent(
  729. room_id=room_id,
  730. agent_id=agent_id,
  731. command=safe_command,
  732. operation="update",
  733. ),
  734. task_name=f"yaoyao-group-update-agent-{request_id}",
  735. failure_message="YaoYao group Agent update failed",
  736. )
  737. async def delete_agent(
  738. self,
  739. *,
  740. room_id: str,
  741. agent_id: str,
  742. request_id: str,
  743. ) -> dict[str, object]:
  744. _required_string(room_id, "roomId")
  745. _required_string(agent_id, "agentId")
  746. _required_string(request_id, "requestId")
  747. return await self._tracked_control(
  748. lambda: self._mutate_agent(
  749. room_id=room_id,
  750. agent_id=agent_id,
  751. command={"requestId": request_id},
  752. operation="delete",
  753. ),
  754. task_name=f"yaoyao-group-delete-agent-{request_id}",
  755. failure_message="YaoYao group Agent deletion failed",
  756. )
  757. async def _mutate_agent(
  758. self,
  759. *,
  760. room_id: str,
  761. agent_id: str,
  762. command: Mapping[str, object],
  763. operation: str,
  764. ) -> dict[str, object]:
  765. if operation == "update":
  766. store_method = self.store.update_agent_with_runtime_targets
  767. elif operation == "delete":
  768. store_method = self.store.delete_agent_with_runtime_targets
  769. else:
  770. raise GroupOrchestratorError("Unknown Agent mutation")
  771. async with self._room_serial(room_id):
  772. async with AsyncExitStack() as stack:
  773. await stack.enter_async_context(self._submission_serial(agent_id))
  774. with self._mapping_lock:
  775. states = sorted(
  776. (
  777. state
  778. for _generation, state in self._runtime_states.values()
  779. if state.room_id == room_id and state.agent_id == agent_id
  780. ),
  781. key=lambda state: (state.runtime_id, state.generation),
  782. )
  783. for state in states:
  784. await stack.enter_async_context(state.mutation_lock)
  785. current_states = self._current_owned_states(states)
  786. envelope = await self._run_states_blocking(
  787. current_states,
  788. store_method,
  789. room_id,
  790. agent_id,
  791. command,
  792. )
  793. result, runtime_ids = self._validated_lifecycle_envelope(
  794. envelope, f"agent.{operation}"
  795. )
  796. if result.get("id") != agent_id or result.get("roomId") != room_id:
  797. raise GroupOrchestratorError("Mutated Agent identity changed")
  798. if operation == "update":
  799. requested_enabled = command.get("enabled")
  800. if "enabled" in command and (
  801. not isinstance(requested_enabled, bool)
  802. or result.get("enabled") is not requested_enabled
  803. ):
  804. raise GroupOrchestratorError(
  805. "Updated Agent enabled state changed"
  806. )
  807. if requested_enabled is not False and runtime_ids:
  808. raise GroupOrchestratorError(
  809. "Non-disabling Agent update returned runtime targets"
  810. )
  811. self.wake()
  812. cleanup_errors = await self._cleanup_lifecycle_targets_locked(
  813. current_states,
  814. runtime_ids,
  815. room_id=room_id,
  816. agent_id=agent_id,
  817. )
  818. if cleanup_errors:
  819. raise GroupOrchestratorError(
  820. "Agent changed durably, but Gateway cleanup failed: "
  821. + "; ".join(cleanup_errors)
  822. )
  823. return result
  824. async def send_message(
  825. self,
  826. *,
  827. room_id: str,
  828. request_id: str,
  829. client_message_id: str,
  830. content: str,
  831. mention_agent_ids: list[str],
  832. topic_id: str | None = None,
  833. ) -> dict[str, object]:
  834. _required_string(room_id, "roomId")
  835. _required_string(request_id, "requestId")
  836. _required_string(client_message_id, "clientMessageId")
  837. if not isinstance(content, str):
  838. raise GroupOrchestratorError("content must be a string")
  839. if not isinstance(mention_agent_ids, list):
  840. raise GroupOrchestratorError("mentionAgentIds must be a list")
  841. if topic_id is not None:
  842. _required_string(topic_id, "topicId")
  843. safe_mentions = [
  844. _required_string(item, "mentionAgentIds") for item in mention_agent_ids
  845. ]
  846. return await self._tracked_control(
  847. lambda: self._send_message(
  848. room_id=room_id,
  849. request_id=request_id,
  850. client_message_id=client_message_id,
  851. content=content,
  852. mention_agent_ids=safe_mentions,
  853. topic_id=topic_id,
  854. ),
  855. task_name=f"yaoyao-group-send-{request_id}",
  856. failure_message="YaoYao group message submission failed",
  857. )
  858. async def _send_message(
  859. self,
  860. *,
  861. room_id: str,
  862. request_id: str,
  863. client_message_id: str,
  864. content: str,
  865. mention_agent_ids: list[str],
  866. topic_id: str | None,
  867. ) -> dict[str, object]:
  868. async with self._room_serial(room_id):
  869. async with AsyncExitStack() as stack:
  870. for agent_id in sorted(set(mention_agent_ids)):
  871. await stack.enter_async_context(self._submission_serial(agent_id))
  872. result = await asyncio.to_thread(
  873. self.store.create_human_message,
  874. room_id,
  875. request_id=request_id,
  876. client_message_id=client_message_id,
  877. content=content,
  878. mention_agent_ids=mention_agent_ids,
  879. topic_id=topic_id,
  880. )
  881. if not isinstance(result, Mapping):
  882. raise GroupOrchestratorError("Created message is invalid")
  883. self.wake()
  884. return dict(result)
  885. async def interrupt_agent(
  886. self,
  887. *,
  888. room_id: str,
  889. agent_id: str,
  890. request_id: str,
  891. ) -> dict[str, object]:
  892. """Durably interrupt one Agent, then clean only owned Gateway generations."""
  893. _required_string(room_id, "roomId")
  894. _required_string(agent_id, "agentId")
  895. _required_string(request_id, "requestId")
  896. if not self._started or self._closing:
  897. raise GroupOrchestratorError("Orchestrator is not accepting controls")
  898. task = asyncio.create_task(
  899. self._interrupt_agent(
  900. room_id=room_id,
  901. agent_id=agent_id,
  902. request_id=request_id,
  903. ),
  904. name=f"yaoyao-group-interrupt-{request_id}",
  905. )
  906. self._control_tasks.add(task)
  907. def finished(done: asyncio.Task[dict[str, object]]) -> None:
  908. self._control_tasks.discard(done)
  909. if done.cancelled():
  910. return
  911. try:
  912. error = done.exception()
  913. except asyncio.CancelledError:
  914. return
  915. if error is not None:
  916. _log_failure_type("YaoYao group Agent interrupt failed", error)
  917. task.add_done_callback(finished)
  918. return await asyncio.shield(task)
  919. async def archive_room(
  920. self,
  921. *,
  922. room_id: str,
  923. request_id: str,
  924. ) -> dict[str, object]:
  925. """Archive one room durably before cleaning its owned runtimes."""
  926. _required_string(room_id, "roomId")
  927. _required_string(request_id, "requestId")
  928. return await self._tracked_control(
  929. lambda: self._archive_room(room_id=room_id, request_id=request_id),
  930. task_name=f"yaoyao-group-archive-{request_id}",
  931. failure_message="YaoYao group room archive failed",
  932. )
  933. async def _archive_room(
  934. self,
  935. *,
  936. room_id: str,
  937. request_id: str,
  938. ) -> dict[str, object]:
  939. async with self._room_serial(room_id):
  940. room = await asyncio.to_thread(
  941. self.store.get_room, room_id, include_archived=True
  942. )
  943. agents = room.get("agents") if isinstance(room, Mapping) else None
  944. if not isinstance(agents, list):
  945. raise GroupOrchestratorError("room.agents must be a list")
  946. agent_ids = sorted(
  947. {
  948. _required_string(
  949. agent.get("id") if isinstance(agent, Mapping) else None,
  950. "room.agent.id",
  951. )
  952. for agent in agents
  953. }
  954. )
  955. async with AsyncExitStack() as stack:
  956. for agent_id in agent_ids:
  957. await stack.enter_async_context(self._submission_serial(agent_id))
  958. with self._mapping_lock:
  959. states = sorted(
  960. (
  961. state
  962. for _generation, state in self._runtime_states.values()
  963. if state.room_id == room_id
  964. ),
  965. key=lambda state: (
  966. state.agent_id,
  967. state.runtime_id,
  968. state.generation,
  969. ),
  970. )
  971. for state in states:
  972. await stack.enter_async_context(state.mutation_lock)
  973. current_states = self._current_owned_states(states)
  974. envelope = await self._run_states_blocking(
  975. current_states,
  976. self.store.archive_room_with_runtime_targets,
  977. room_id,
  978. {"requestId": request_id},
  979. )
  980. result, runtime_ids = self._validated_lifecycle_envelope(
  981. envelope, "archive"
  982. )
  983. if result.get("id") != room_id or result.get("archived") is not True:
  984. raise GroupOrchestratorError("Archived room identity changed")
  985. self.wake()
  986. cleanup_errors = await self._cleanup_lifecycle_targets_locked(
  987. current_states,
  988. runtime_ids,
  989. room_id=room_id,
  990. )
  991. if cleanup_errors:
  992. raise GroupOrchestratorError(
  993. "Room was archived durably, but Gateway cleanup failed: "
  994. + "; ".join(cleanup_errors)
  995. )
  996. return result
  997. def _current_owned_states(self, states: list[_RuntimeState]) -> list[_RuntimeState]:
  998. current_states: list[_RuntimeState] = []
  999. with self._mapping_lock:
  1000. for state in states:
  1001. if self._runtime_states.get(state.runtime_id) == (
  1002. state.generation,
  1003. state,
  1004. ):
  1005. current_states.append(state)
  1006. return current_states
  1007. @staticmethod
  1008. def _validated_lifecycle_envelope(
  1009. value: object, field_name: str
  1010. ) -> tuple[dict[str, object], tuple[str, ...]]:
  1011. envelope = _required_mapping(value, field_name)
  1012. result = _required_mapping(envelope.get("result"), f"{field_name}.result")
  1013. raw_runtime_ids = envelope.get("runtimeSessionIds")
  1014. if not isinstance(raw_runtime_ids, list) or len(raw_runtime_ids) > 8:
  1015. raise GroupOrchestratorError(
  1016. f"{field_name}.runtimeSessionIds must be a bounded list"
  1017. )
  1018. runtime_ids = tuple(
  1019. _required_string(item, f"{field_name}.runtimeSessionIds")
  1020. for item in raw_runtime_ids
  1021. )
  1022. if len(set(runtime_ids)) != len(runtime_ids):
  1023. raise GroupOrchestratorError(
  1024. f"{field_name}.runtimeSessionIds contains duplicates"
  1025. )
  1026. return dict(result), runtime_ids
  1027. async def _cleanup_lifecycle_targets_locked(
  1028. self,
  1029. states: list[_RuntimeState],
  1030. runtime_ids: tuple[str, ...],
  1031. *,
  1032. room_id: str,
  1033. agent_id: str | None = None,
  1034. ) -> list[str]:
  1035. owned = {
  1036. state.runtime_id: state
  1037. for state in states
  1038. if state.room_id == room_id
  1039. and (agent_id is None or state.agent_id == agent_id)
  1040. }
  1041. cleanup_errors: list[str] = []
  1042. replay_runtime_ids = set(runtime_ids)
  1043. ordered_runtime_ids = list(runtime_ids)
  1044. ordered_runtime_ids.extend(
  1045. runtime_id
  1046. for runtime_id in sorted(owned)
  1047. if runtime_id not in replay_runtime_ids
  1048. )
  1049. for runtime_id in ordered_runtime_ids:
  1050. state = owned.get(runtime_id)
  1051. if state is None:
  1052. continue
  1053. with self._mapping_lock:
  1054. current = self._runtime_states.get(runtime_id)
  1055. if current != (state.generation, state):
  1056. continue
  1057. try:
  1058. durable_run = await self._run_state_blocking(
  1059. state, self.store.get_run, state.run_id
  1060. )
  1061. except Exception: # noqa: BLE001 - never clean an unverified owner
  1062. cleanup_errors.append(f"verify {runtime_id}")
  1063. continue
  1064. if (
  1065. not isinstance(durable_run, Mapping)
  1066. or durable_run.get("status") not in _TERMINAL_RUN_STATUSES
  1067. or durable_run.get("runtimeSessionId") is not None
  1068. ):
  1069. # The idempotency replay may name a runtime id that has since
  1070. # been reused by a newer generation. Never clean that owner.
  1071. continue
  1072. error = await self._close_durably_interrupted_state_locked(state)
  1073. if error:
  1074. cleanup_errors.append(error)
  1075. return cleanup_errors
  1076. async def _interrupt_agent(
  1077. self,
  1078. *,
  1079. room_id: str,
  1080. agent_id: str,
  1081. request_id: str,
  1082. ) -> dict[str, object]:
  1083. async with self._submission_serial(agent_id):
  1084. with self._mapping_lock:
  1085. states = sorted(
  1086. (
  1087. state
  1088. for _generation, state in self._runtime_states.values()
  1089. if state.room_id == room_id and state.agent_id == agent_id
  1090. ),
  1091. key=lambda state: (state.runtime_id, state.generation),
  1092. )
  1093. async with AsyncExitStack() as stack:
  1094. for state in states:
  1095. await stack.enter_async_context(state.mutation_lock)
  1096. current_states: list[_RuntimeState] = []
  1097. with self._mapping_lock:
  1098. for state in states:
  1099. if self._runtime_states.get(state.runtime_id) == (
  1100. state.generation,
  1101. state,
  1102. ):
  1103. current_states.append(state)
  1104. if any(
  1105. state.finalizing and state.settle_error is not None
  1106. for state in current_states
  1107. ):
  1108. raise GroupOrchestratorError("Agent runtime is already finalizing")
  1109. envelope = await self._run_states_blocking(
  1110. current_states,
  1111. self.store.interrupt_agent_with_runtime_targets,
  1112. room_id,
  1113. agent_id,
  1114. {"requestId": request_id},
  1115. )
  1116. result, run_ids, runtime_ids = self._validated_interrupt_envelope(
  1117. envelope,
  1118. room_id=room_id,
  1119. agent_id=agent_id,
  1120. )
  1121. cleanup_errors: list[str] = []
  1122. for runtime_id in runtime_ids:
  1123. with self._mapping_lock:
  1124. current = self._runtime_states.get(runtime_id)
  1125. if current is None:
  1126. continue
  1127. generation, state = current
  1128. if (
  1129. state not in current_states
  1130. or generation != state.generation
  1131. or state.room_id != room_id
  1132. or state.agent_id != agent_id
  1133. or state.run_id not in run_ids
  1134. ):
  1135. # A replayed target must never affect a reused runtime id.
  1136. continue
  1137. error = await self._close_durably_interrupted_state_locked(state)
  1138. if error:
  1139. cleanup_errors.append(error)
  1140. if cleanup_errors:
  1141. raise GroupOrchestratorError(
  1142. "Agent was interrupted durably, but Gateway cleanup failed: "
  1143. + "; ".join(cleanup_errors)
  1144. )
  1145. return result
  1146. @staticmethod
  1147. def _validated_interrupt_envelope(
  1148. value: object,
  1149. *,
  1150. room_id: str,
  1151. agent_id: str,
  1152. ) -> tuple[dict[str, object], set[str], tuple[str, ...]]:
  1153. envelope = _required_mapping(value, "interrupt")
  1154. result_value = _required_mapping(envelope.get("result"), "interrupt.result")
  1155. if (
  1156. result_value.get("roomId") != room_id
  1157. or result_value.get("agentId") != agent_id
  1158. ):
  1159. raise GroupOrchestratorError("Interrupt scope changed")
  1160. def string_list(raw: object, field_name: str) -> tuple[str, ...]:
  1161. if not isinstance(raw, list) or len(raw) > 8:
  1162. raise GroupOrchestratorError(f"{field_name} must be a bounded list")
  1163. values = tuple(_required_string(item, field_name) for item in raw)
  1164. if len(set(values)) != len(values):
  1165. raise GroupOrchestratorError(f"{field_name} contains duplicates")
  1166. return values
  1167. run_ids = string_list(
  1168. result_value.get("interruptedRunIds"),
  1169. "interrupt.result.interruptedRunIds",
  1170. )
  1171. runtime_ids = string_list(
  1172. envelope.get("runtimeSessionIds"),
  1173. "interrupt.runtimeSessionIds",
  1174. )
  1175. return dict(result_value), set(run_ids), runtime_ids
  1176. async def _close_durably_interrupted_state_locked(
  1177. self, state: _RuntimeState
  1178. ) -> str | None:
  1179. async with state.finalize_lock:
  1180. with self._mapping_lock:
  1181. current = self._runtime_states.get(state.runtime_id)
  1182. if current != (state.generation, state):
  1183. return None
  1184. state.finalizing = True
  1185. failures: list[str] = []
  1186. gateway = self._required_gateway()
  1187. try:
  1188. await self._run_state_blocking(
  1189. state, gateway.interrupt, state.runtime_id
  1190. )
  1191. except Exception: # noqa: BLE001 - close is still required
  1192. failures.append(f"interrupt {state.runtime_id}")
  1193. closed = False
  1194. try:
  1195. closed = (
  1196. await self._run_state_blocking(
  1197. state, gateway.close, state.runtime_id
  1198. )
  1199. is True
  1200. )
  1201. if not closed:
  1202. failures.append(f"close {state.runtime_id}")
  1203. except Exception: # noqa: BLE001 - durable state already won
  1204. failures.append(f"close {state.runtime_id}")
  1205. if closed:
  1206. self._remove_runtime(state)
  1207. state.settle_error = None
  1208. state.finished.set()
  1209. state.terminal_info_received.set()
  1210. state.pending_interaction_ids.clear()
  1211. current_task = asyncio.current_task()
  1212. for helper in (state.event_task, state.grace_task):
  1213. if helper is None or helper is current_task:
  1214. continue
  1215. if not helper.done():
  1216. helper.cancel()
  1217. await asyncio.gather(helper, return_exceptions=True)
  1218. return ", ".join(failures) or None
  1219. async def respond_approval(
  1220. self,
  1221. *,
  1222. room_id: str,
  1223. interaction_id: str,
  1224. request_id: str,
  1225. choice: str,
  1226. permanent: bool,
  1227. ) -> dict[str, object]:
  1228. if not self._started or self._closing:
  1229. raise GroupOrchestratorError("Orchestrator is not accepting controls")
  1230. if choice not in _APPROVAL_CHOICES:
  1231. raise ValueError("choice must be once, session, always, or deny")
  1232. if not isinstance(permanent, bool) or ((choice == "always") != permanent):
  1233. raise ValueError("permanent must be true exactly for choice=always")
  1234. replay = await self._terminal_interaction_replay(
  1235. room_id=room_id,
  1236. interaction_id=interaction_id,
  1237. request_id=request_id,
  1238. kind="approval",
  1239. response={"choice": choice, "permanent": permanent},
  1240. )
  1241. if replay is not None:
  1242. return replay
  1243. state, interaction = await self._interaction_runtime(
  1244. room_id, interaction_id, "approval"
  1245. )
  1246. task = asyncio.create_task(
  1247. self._respond_approval_for_runtime(
  1248. state=state,
  1249. interaction=interaction,
  1250. room_id=room_id,
  1251. interaction_id=interaction_id,
  1252. request_id=request_id,
  1253. choice=choice,
  1254. permanent=permanent,
  1255. ),
  1256. name=f"yaoyao-group-approval-{request_id}",
  1257. )
  1258. self._track_mutation_task(state, task)
  1259. return await asyncio.shield(task)
  1260. async def _respond_approval_for_runtime(
  1261. self,
  1262. *,
  1263. state: _RuntimeState,
  1264. interaction: Mapping[str, object],
  1265. room_id: str,
  1266. interaction_id: str,
  1267. request_id: str,
  1268. choice: str,
  1269. permanent: bool,
  1270. ) -> dict[str, object]:
  1271. async with state.mutation_lock:
  1272. self._require_current_runtime(state)
  1273. claim = await self._run_state_blocking(
  1274. state,
  1275. self.store.begin_interaction_response,
  1276. room_id,
  1277. interaction_id,
  1278. request_id=request_id,
  1279. kind="approval",
  1280. response={"choice": choice, "permanent": permanent},
  1281. )
  1282. if not isinstance(claim, Mapping):
  1283. raise GroupOrchestratorError("Approval claim is invalid")
  1284. if claim.get("state") in {"replay", "failed"}:
  1285. response = claim.get("response")
  1286. return dict(response) if isinstance(response, Mapping) else dict(claim)
  1287. payload = interaction.get("payload")
  1288. choices = payload.get("choices") if isinstance(payload, Mapping) else None
  1289. if (
  1290. not isinstance(choices, list)
  1291. or choice not in choices
  1292. or not all(item in _APPROVAL_CHOICES for item in choices)
  1293. ):
  1294. await self._run_state_blocking(
  1295. state,
  1296. self.store.fail_interaction_response,
  1297. request_id,
  1298. reason="Approval choice is not allowed by the pending card",
  1299. uncertain=False,
  1300. )
  1301. raise ValueError("choice is not allowed by the pending approval")
  1302. try:
  1303. resolved = await self._run_state_blocking(
  1304. state,
  1305. self._required_gateway().respond_approval,
  1306. state.runtime_id,
  1307. choice,
  1308. )
  1309. except Exception:
  1310. await self._fail_unknown_interaction_response_locked(
  1311. state, request_id, "Gateway approval outcome is unknown"
  1312. )
  1313. raise GroupOrchestratorError("Gateway approval outcome is unknown")
  1314. if (
  1315. isinstance(resolved, bool)
  1316. or not isinstance(resolved, int)
  1317. or resolved not in {0, 1}
  1318. ):
  1319. await self._fail_unknown_interaction_response_locked(
  1320. state, request_id, "Gateway approval result is invalid"
  1321. )
  1322. raise GroupOrchestratorError("Gateway approval result is invalid")
  1323. public = {
  1324. "accepted": resolved == 1,
  1325. "interactionId": interaction_id,
  1326. }
  1327. try:
  1328. result = await self._store_retry(
  1329. state,
  1330. self.store.finish_interaction_response,
  1331. request_id,
  1332. response=public,
  1333. interaction_status=("resolved" if resolved == 1 else "cancelled"),
  1334. )
  1335. except Exception:
  1336. await self._finalize_state_locked(
  1337. state,
  1338. outcome="failed",
  1339. error="Durable approval response could not be completed",
  1340. interrupt=True,
  1341. )
  1342. raise
  1343. state.pending_interaction_ids.discard(interaction_id)
  1344. return dict(result)
  1345. async def respond_clarification(
  1346. self,
  1347. *,
  1348. room_id: str,
  1349. interaction_id: str,
  1350. request_id: str,
  1351. response: str,
  1352. ) -> dict[str, object]:
  1353. if not self._started or self._closing:
  1354. raise GroupOrchestratorError("Orchestrator is not accepting controls")
  1355. if not isinstance(response, str) or not response.strip():
  1356. raise ValueError("response must not be blank")
  1357. self._bounded_text(response, "clarification response")
  1358. replay = await self._terminal_interaction_replay(
  1359. room_id=room_id,
  1360. interaction_id=interaction_id,
  1361. request_id=request_id,
  1362. kind="clarification",
  1363. response={"response": response},
  1364. )
  1365. if replay is not None:
  1366. return replay
  1367. state, interaction = await self._interaction_runtime(
  1368. room_id, interaction_id, "clarification"
  1369. )
  1370. task = asyncio.create_task(
  1371. self._respond_clarification_for_runtime(
  1372. state=state,
  1373. interaction=interaction,
  1374. room_id=room_id,
  1375. interaction_id=interaction_id,
  1376. request_id=request_id,
  1377. response=response,
  1378. ),
  1379. name=f"yaoyao-group-clarification-{request_id}",
  1380. )
  1381. self._track_mutation_task(state, task)
  1382. return await asyncio.shield(task)
  1383. @staticmethod
  1384. def _track_mutation_task(
  1385. state: _RuntimeState, task: asyncio.Task[dict[str, object]]
  1386. ) -> None:
  1387. state.mutation_tasks.add(task)
  1388. def finished(done: asyncio.Task[dict[str, object]]) -> None:
  1389. state.mutation_tasks.discard(done)
  1390. if done.cancelled():
  1391. return
  1392. try:
  1393. done.exception()
  1394. except asyncio.CancelledError:
  1395. return
  1396. task.add_done_callback(finished)
  1397. async def _respond_clarification_for_runtime(
  1398. self,
  1399. *,
  1400. state: _RuntimeState,
  1401. interaction: Mapping[str, object],
  1402. room_id: str,
  1403. interaction_id: str,
  1404. request_id: str,
  1405. response: str,
  1406. ) -> dict[str, object]:
  1407. async with state.mutation_lock:
  1408. self._require_current_runtime(state)
  1409. claim = await self._run_state_blocking(
  1410. state,
  1411. self.store.begin_interaction_response,
  1412. room_id,
  1413. interaction_id,
  1414. request_id=request_id,
  1415. kind="clarification",
  1416. response={"response": response},
  1417. )
  1418. if not isinstance(claim, Mapping):
  1419. raise GroupOrchestratorError("Clarification claim is invalid")
  1420. if claim.get("state") in {"replay", "failed"}:
  1421. replay = claim.get("response")
  1422. return dict(replay) if isinstance(replay, Mapping) else dict(claim)
  1423. payload = interaction.get("payload")
  1424. gateway_request_id = (
  1425. payload.get("gatewayRequestId")
  1426. if isinstance(payload, Mapping)
  1427. else None
  1428. )
  1429. gateway_request_id = _required_string(
  1430. gateway_request_id, "interaction.payload.gatewayRequestId"
  1431. )
  1432. try:
  1433. status = await self._run_state_blocking(
  1434. state,
  1435. self._required_gateway().respond_clarification,
  1436. gateway_request_id,
  1437. response,
  1438. )
  1439. except Exception:
  1440. await self._fail_unknown_interaction_response_locked(
  1441. state, request_id, "Gateway clarification outcome is unknown"
  1442. )
  1443. raise GroupOrchestratorError("Gateway clarification outcome is unknown")
  1444. if status not in {"ok", "expired"}:
  1445. await self._fail_unknown_interaction_response_locked(
  1446. state, request_id, "Gateway clarification result is invalid"
  1447. )
  1448. raise GroupOrchestratorError("Gateway clarification result is invalid")
  1449. public = {
  1450. "accepted": status == "ok",
  1451. "interactionId": interaction_id,
  1452. }
  1453. try:
  1454. result = await self._store_retry(
  1455. state,
  1456. self.store.finish_interaction_response,
  1457. request_id,
  1458. response=public,
  1459. interaction_status=("resolved" if status == "ok" else "cancelled"),
  1460. )
  1461. except Exception:
  1462. await self._finalize_state_locked(
  1463. state,
  1464. outcome="failed",
  1465. error="Durable clarification response could not be completed",
  1466. interrupt=True,
  1467. )
  1468. raise
  1469. state.pending_interaction_ids.discard(interaction_id)
  1470. return dict(result)
  1471. async def _terminal_interaction_replay(
  1472. self,
  1473. *,
  1474. room_id: str,
  1475. interaction_id: str,
  1476. request_id: str,
  1477. kind: str,
  1478. response: Mapping[str, object],
  1479. ) -> dict[str, object] | None:
  1480. interaction = await asyncio.to_thread(
  1481. self.store.get_interaction, interaction_id
  1482. )
  1483. if (
  1484. not isinstance(interaction, Mapping)
  1485. or interaction.get("roomId") != room_id
  1486. or interaction.get("kind") != kind
  1487. ):
  1488. raise GroupOrchestratorError("Interaction is not in this room")
  1489. if interaction.get("status") == "pending":
  1490. return None
  1491. result = await asyncio.to_thread(
  1492. self.store.begin_interaction_response,
  1493. room_id,
  1494. interaction_id,
  1495. request_id=request_id,
  1496. kind=kind,
  1497. response=response,
  1498. )
  1499. if not isinstance(result, Mapping):
  1500. raise GroupOrchestratorError("Interaction replay is invalid")
  1501. replay = result.get("response")
  1502. if result.get("state") == "replay" and isinstance(replay, Mapping):
  1503. return dict(replay)
  1504. if result.get("state") == "failed":
  1505. return dict(result)
  1506. raise GroupOrchestratorError("Interaction is already terminal")
  1507. async def _interaction_runtime(
  1508. self, room_id: str, interaction_id: str, kind: str
  1509. ) -> tuple[_RuntimeState, Mapping[str, object]]:
  1510. interaction = await asyncio.to_thread(
  1511. self.store.get_interaction, interaction_id
  1512. )
  1513. if (
  1514. not isinstance(interaction, Mapping)
  1515. or interaction.get("roomId") != room_id
  1516. or interaction.get("kind") != kind
  1517. or interaction.get("status") != "pending"
  1518. ):
  1519. raise GroupOrchestratorError("Interaction is not pending in this room")
  1520. run_id = _required_string(interaction.get("runId"), "interaction.runId")
  1521. run = await asyncio.to_thread(self.store.get_run, run_id)
  1522. runtime_id = _required_string(
  1523. run.get("runtimeSessionId") if isinstance(run, Mapping) else None,
  1524. "run.runtimeSessionId",
  1525. )
  1526. with self._mapping_lock:
  1527. current = self._runtime_states.get(runtime_id)
  1528. if current is None or current[1].run_id != run_id:
  1529. raise GroupOrchestratorError("Interaction runtime is not active")
  1530. return current[1], interaction
  1531. def _require_current_runtime(self, state: _RuntimeState) -> None:
  1532. with self._mapping_lock:
  1533. current = self._runtime_states.get(state.runtime_id)
  1534. if (
  1535. current != (state.generation, state)
  1536. or state.finished.is_set()
  1537. or state.finalizing
  1538. ):
  1539. raise GroupOrchestratorError("Interaction runtime generation changed")
  1540. async def _fail_unknown_interaction_response_locked(
  1541. self, state: _RuntimeState, request_id: str, reason: str
  1542. ) -> None:
  1543. try:
  1544. await self._store_retry(
  1545. state,
  1546. self.store.fail_interaction_response,
  1547. request_id,
  1548. reason=reason,
  1549. uncertain=True,
  1550. )
  1551. except Exception:
  1552. logger.error("Failed to persist unknown interaction outcome")
  1553. await self._finalize_state_locked(
  1554. state, outcome="failed", error=reason, interrupt=True
  1555. )
  1556. async def shutdown(self) -> None:
  1557. """Stop claims, settle every owned runtime, drain, then close Gateway."""
  1558. if self._lifecycle_lock is None:
  1559. self._lifecycle_lock = asyncio.Lock()
  1560. wait_for_owner: asyncio.Event | None = None
  1561. completed_shutdown = False
  1562. async with self._lifecycle_lock:
  1563. if not self._started:
  1564. self._closing = True
  1565. completed_shutdown = bool(
  1566. self._shutdown_complete is not None
  1567. and self._shutdown_complete.is_set()
  1568. )
  1569. if not completed_shutdown:
  1570. return
  1571. scheduler = None
  1572. elif self._closing:
  1573. wait_for_owner = self._shutdown_complete
  1574. scheduler = None
  1575. else:
  1576. self._closing = True
  1577. if self._shutdown_event is not None:
  1578. self._shutdown_event.set()
  1579. if self._wake_event is not None:
  1580. self._wake_event.set()
  1581. scheduler = self._scheduler_task
  1582. if completed_shutdown:
  1583. if self._shutdown_error is not None:
  1584. raise self._shutdown_error
  1585. return
  1586. if wait_for_owner is not None:
  1587. await wait_for_owner.wait()
  1588. if self._shutdown_error is not None:
  1589. raise self._shutdown_error
  1590. return
  1591. first_error: BaseException | None = None
  1592. def remember(error: BaseException) -> None:
  1593. nonlocal first_error
  1594. if first_error is None:
  1595. first_error = error
  1596. if scheduler is not None:
  1597. try:
  1598. await scheduler
  1599. except BaseException as error: # noqa: BLE001 - cleanup continues
  1600. remember(error)
  1601. while self._run_tasks:
  1602. tasks = tuple(self._run_tasks)
  1603. try:
  1604. results = await asyncio.gather(*tasks, return_exceptions=True)
  1605. except BaseException as error: # noqa: BLE001 - cleanup continues
  1606. remember(error)
  1607. else:
  1608. self._remember_task_results(results, remember)
  1609. while self._control_tasks:
  1610. tasks = tuple(self._control_tasks)
  1611. try:
  1612. results = await asyncio.gather(*tasks, return_exceptions=True)
  1613. except BaseException as error: # noqa: BLE001 - cleanup continues
  1614. remember(error)
  1615. else:
  1616. self._remember_task_results(results, remember)
  1617. with self._failure_lock:
  1618. run_failures = tuple(self._run_failures)
  1619. durable_failures = tuple(self._durable_failures)
  1620. for error in (*run_failures, *durable_failures):
  1621. remember(error)
  1622. with self._mapping_lock:
  1623. remaining_states = [
  1624. state for _generation, state in self._runtime_states.values()
  1625. ]
  1626. for state in remaining_states:
  1627. if state.settle_error is not None:
  1628. remember(state.settle_error)
  1629. current_task = asyncio.current_task()
  1630. for helper in (state.event_task, state.grace_task):
  1631. if helper is None or helper is current_task:
  1632. continue
  1633. if not helper.done():
  1634. helper.cancel()
  1635. try:
  1636. results = await asyncio.gather(helper, return_exceptions=True)
  1637. except BaseException as error: # noqa: BLE001 - cleanup continues
  1638. remember(error)
  1639. else:
  1640. self._remember_task_results(results, remember)
  1641. pending_calls = tuple(state.inflight_calls)
  1642. if pending_calls:
  1643. try:
  1644. results = await asyncio.gather(
  1645. *pending_calls, return_exceptions=True
  1646. )
  1647. except BaseException as error: # noqa: BLE001 - cleanup continues
  1648. remember(error)
  1649. else:
  1650. self._remember_task_results(results, remember)
  1651. self._remove_runtime(state)
  1652. try:
  1653. await asyncio.to_thread(
  1654. self._required_gateway().close, state.runtime_id
  1655. )
  1656. except BaseException as error: # noqa: BLE001 - cleanup continues
  1657. remember(error)
  1658. state.finished.set()
  1659. gateway = self._gateway
  1660. if gateway is not None:
  1661. drain: Callable[[float | None], object] | None = None
  1662. try:
  1663. transport = getattr(gateway, "transport", None)
  1664. candidate = getattr(transport, "drain", None)
  1665. if callable(candidate):
  1666. drain = candidate
  1667. except BaseException as error: # noqa: BLE001 - shutdown still runs
  1668. remember(error)
  1669. if drain is not None:
  1670. try:
  1671. await asyncio.to_thread(drain, None)
  1672. except BaseException as error: # noqa: BLE001 - cleanup continues
  1673. remember(error)
  1674. try:
  1675. await asyncio.to_thread(gateway.shutdown)
  1676. except BaseException as error: # noqa: BLE001 - cleanup continues
  1677. remember(error)
  1678. try:
  1679. event_queue = self._event_queue
  1680. event_task = self._event_task
  1681. if event_queue is not None:
  1682. try:
  1683. await event_queue.join()
  1684. await event_queue.put(None)
  1685. except BaseException as error: # noqa: BLE001 - completion must signal
  1686. remember(error)
  1687. if event_task is not None:
  1688. try:
  1689. await event_task
  1690. except BaseException as error: # noqa: BLE001 - completion must signal
  1691. remember(error)
  1692. finally:
  1693. async with self._lifecycle_lock:
  1694. self._started = False
  1695. self._gateway = None
  1696. self._scheduler_task = None
  1697. self._event_task = None
  1698. self._shutdown_error = first_error
  1699. if self._shutdown_complete is not None:
  1700. self._shutdown_complete.set()
  1701. if first_error is not None:
  1702. raise first_error
  1703. @staticmethod
  1704. def _remember_task_results(
  1705. results: list[object], remember: Callable[[BaseException], None]
  1706. ) -> None:
  1707. for result in results:
  1708. if isinstance(result, BaseException) and not isinstance(
  1709. result, asyncio.CancelledError
  1710. ):
  1711. remember(result)
  1712. async def _scheduler(self) -> None:
  1713. wake_event = self._wake_event
  1714. shutdown_event = self._shutdown_event
  1715. if wake_event is None or shutdown_event is None:
  1716. raise GroupOrchestratorError("Scheduler started without lifecycle state")
  1717. claim_failure_count = 0
  1718. while True:
  1719. await wake_event.wait()
  1720. wake_event.clear()
  1721. if shutdown_event.is_set():
  1722. return
  1723. self._scheduler_busy = True
  1724. try:
  1725. if not await self._work_is_enabled():
  1726. if await self._wait_for_disabled_work(shutdown_event, wake_event):
  1727. return
  1728. continue
  1729. try:
  1730. more_cascades = await self._drain_pending_cascades()
  1731. except Exception: # noqa: BLE001 - durable ledger remains pending
  1732. claim_failure_count += 1
  1733. delay = min(
  1734. _STORE_RETRY_DELAYS[0]
  1735. * (2 ** min(claim_failure_count - 1, 10)),
  1736. _CLAIM_RETRY_MAX_SECONDS,
  1737. )
  1738. logger.error("Pending YaoYao group cascade temporarily failed")
  1739. try:
  1740. await asyncio.wait_for(shutdown_event.wait(), timeout=delay)
  1741. except asyncio.TimeoutError:
  1742. wake_event.set()
  1743. continue
  1744. return
  1745. if more_cascades:
  1746. wake_event.set()
  1747. continue
  1748. while not shutdown_event.is_set():
  1749. if not await self._work_is_enabled():
  1750. if await self._wait_for_disabled_work(
  1751. shutdown_event, wake_event
  1752. ):
  1753. return
  1754. break
  1755. try:
  1756. claimed = await asyncio.to_thread(
  1757. self.store.claim_next_runnable_run
  1758. )
  1759. except Exception: # noqa: BLE001 - keep scheduler alive
  1760. claim_failure_count += 1
  1761. delay = min(
  1762. _STORE_RETRY_DELAYS[0]
  1763. * (2 ** min(claim_failure_count - 1, 10)),
  1764. _CLAIM_RETRY_MAX_SECONDS,
  1765. )
  1766. logger.error("YaoYao group run claim temporarily failed")
  1767. try:
  1768. await asyncio.wait_for(shutdown_event.wait(), timeout=delay)
  1769. except asyncio.TimeoutError:
  1770. continue
  1771. return
  1772. claim_failure_count = 0
  1773. if claimed is None:
  1774. break
  1775. task = asyncio.create_task(
  1776. self._run_claimed(claimed),
  1777. name=f"yaoyao-group-run-{claimed.get('id', 'invalid')}",
  1778. )
  1779. self._run_tasks.add(task)
  1780. task.add_done_callback(self._run_finished)
  1781. finally:
  1782. self._scheduler_busy = False
  1783. async def _work_is_enabled(self) -> bool:
  1784. try:
  1785. return await asyncio.to_thread(self._work_enabled) is True
  1786. except Exception as error: # noqa: BLE001 - fail closed on config errors
  1787. _log_failure_type("YaoYao group work gate temporarily failed", error)
  1788. return False
  1789. @staticmethod
  1790. async def _wait_for_disabled_work(
  1791. shutdown_event: asyncio.Event,
  1792. wake_event: asyncio.Event,
  1793. ) -> bool:
  1794. try:
  1795. await asyncio.wait_for(
  1796. shutdown_event.wait(), timeout=_WORK_DISABLED_RETRY_SECONDS
  1797. )
  1798. except asyncio.TimeoutError:
  1799. wake_event.set()
  1800. return False
  1801. return True
  1802. async def _drain_pending_cascades(self) -> bool:
  1803. page = await asyncio.to_thread(self.store.list_pending_cascades, limit=32)
  1804. if not isinstance(page, Mapping):
  1805. raise GroupOrchestratorError("Pending cascade page is invalid")
  1806. items = page.get("items")
  1807. if not isinstance(items, list) or not all(
  1808. isinstance(item, Mapping) for item in items
  1809. ):
  1810. raise GroupOrchestratorError("Pending cascade items are invalid")
  1811. created = False
  1812. for item in items:
  1813. source_run_id = _required_string(
  1814. item.get("sourceRunId"), "cascade.sourceRunId"
  1815. )
  1816. result = await asyncio.to_thread(self.store.complete_cascade, source_run_id)
  1817. if not isinstance(result, Mapping):
  1818. raise GroupOrchestratorError("Cascade completion is invalid")
  1819. run_count = result.get("runCount")
  1820. if isinstance(run_count, bool) or not isinstance(run_count, int):
  1821. raise GroupOrchestratorError("Cascade run count is invalid")
  1822. created = created or run_count > 0
  1823. if created:
  1824. self.wake()
  1825. return page.get("nextCursor") is not None
  1826. def _run_finished(self, task: asyncio.Task[None]) -> None:
  1827. self._run_tasks.discard(task)
  1828. try:
  1829. error = task.exception()
  1830. except asyncio.CancelledError:
  1831. error = None
  1832. if error is not None:
  1833. with self._failure_lock:
  1834. self._run_failures.append(error)
  1835. _log_failure_type("YaoYao group run task failed", error)
  1836. self.wake()
  1837. async def _run_claimed(self, claimed: Mapping[str, object]) -> None:
  1838. run_id_value = claimed.get("id")
  1839. if not isinstance(run_id_value, str) or not run_id_value:
  1840. logger.error("Claimed YaoYao group run has no usable identity")
  1841. return
  1842. run_id = run_id_value
  1843. try:
  1844. claimed_agent_id = _required_string(claimed.get("agentId"), "run.agentId")
  1845. async with self._agent_serial(claimed_agent_id):
  1846. try:
  1847. await self._run_claimed_for_agent(claimed, run_id)
  1848. except Exception: # noqa: BLE001 - never strand a durable claim
  1849. logger.exception("YaoYao group claimed run preparation failed")
  1850. await self._fail_claim_without_projection(run_id)
  1851. except Exception: # noqa: BLE001 - malformed claim still has run ID
  1852. logger.exception("YaoYao group claimed run identity is invalid")
  1853. await self._fail_claim_without_projection(run_id)
  1854. @asynccontextmanager
  1855. async def _agent_serial(self, agent_id: str):
  1856. async with self._keyed_serial(self._agent_locks, agent_id):
  1857. yield
  1858. @asynccontextmanager
  1859. async def _submission_serial(self, agent_id: str):
  1860. async with self._keyed_serial(self._submission_locks, agent_id):
  1861. yield
  1862. @asynccontextmanager
  1863. async def _room_serial(self, room_id: str):
  1864. async with self._keyed_serial(self._room_locks, room_id):
  1865. yield
  1866. @staticmethod
  1867. @asynccontextmanager
  1868. async def _keyed_serial(locks: dict[str, _AgentLockState], key: str):
  1869. entry = locks.get(key)
  1870. if entry is None:
  1871. entry = _AgentLockState()
  1872. locks[key] = entry
  1873. entry.users += 1
  1874. try:
  1875. async with entry.lock:
  1876. yield
  1877. finally:
  1878. entry.users -= 1
  1879. if entry.users == 0 and locks.get(key) is entry:
  1880. locks.pop(key, None)
  1881. async def _run_claimed_for_agent(
  1882. self, claimed: Mapping[str, object], run_id: str
  1883. ) -> None:
  1884. try:
  1885. projection = await asyncio.to_thread(self.store.read_run_projection, run_id)
  1886. except Exception: # noqa: BLE001 - durable claim must be terminal
  1887. logger.exception("Failed to read YaoYao group run projection")
  1888. await self._fail_claim_without_projection(run_id)
  1889. return
  1890. run = _required_mapping(projection.get("run"), "projection.run")
  1891. room = _required_mapping(projection.get("room"), "projection.room")
  1892. agent = _required_mapping(projection.get("agent"), "projection.agent")
  1893. room_id = _required_string(room.get("id"), "projection.room.id")
  1894. agent_id = _required_string(agent.get("id"), "projection.agent.id")
  1895. if _required_string(run.get("id"), "projection.run.id") != run_id:
  1896. raise GroupOrchestratorError("projection run identity changed")
  1897. if agent_id != _required_string(claimed.get("agentId"), "run.agentId"):
  1898. raise GroupOrchestratorError("projection Agent identity changed")
  1899. configuration = {
  1900. "model": agent.get("model"),
  1901. "provider": agent.get("provider"),
  1902. "reasoning_effort": agent.get("reasoningEffort"),
  1903. "fast": agent.get("fastMode"),
  1904. }
  1905. try:
  1906. rotated = await asyncio.to_thread(
  1907. self.store.prepare_run_session_configuration,
  1908. run_id,
  1909. configuration,
  1910. )
  1911. if rotated:
  1912. projection = await asyncio.to_thread(
  1913. self.store.read_run_projection, run_id
  1914. )
  1915. run = _required_mapping(projection.get("run"), "projection.run")
  1916. room = _required_mapping(projection.get("room"), "projection.room")
  1917. agent = _required_mapping(projection.get("agent"), "projection.agent")
  1918. except Exception:
  1919. logger.exception("Failed to prepare YaoYao group Agent configuration")
  1920. await self._fail_claim_without_projection(run_id)
  1921. return
  1922. profile = _required_string(agent.get("profile"), "projection.agent.profile")
  1923. reply_mode = _required_string(run.get("replyMode"), "projection.run.replyMode")
  1924. if reply_mode not in {"mentioned", "automatic"}:
  1925. raise GroupOrchestratorError("projection.run.replyMode is invalid")
  1926. required_reply = _required_bool(
  1927. run.get("requiredReply", False), "projection.run.requiredReply"
  1928. )
  1929. if required_reply and reply_mode != "automatic":
  1930. raise GroupOrchestratorError(
  1931. "projection.run.requiredReply requires automatic replyMode"
  1932. )
  1933. display_name = _required_string(
  1934. agent.get("displayName"), "projection.agent.displayName"
  1935. )
  1936. room_name = _required_string(room.get("name"), "projection.room.name")
  1937. cwd_value = room.get("cwd", "")
  1938. if not isinstance(cwd_value, str):
  1939. raise GroupOrchestratorError("projection.room.cwd must be a string")
  1940. expected_stored_id = _optional_string(
  1941. agent.get("storedSessionId"), "projection.agent.storedSessionId"
  1942. )
  1943. expected_context_seq = _required_nonnegative_int(
  1944. agent.get("lastContextMessageSeq"),
  1945. "projection.agent.lastContextMessageSeq",
  1946. )
  1947. through_seq = _required_nonnegative_int(
  1948. projection.get("throughSeq"), "projection.throughSeq"
  1949. )
  1950. room_detail = await asyncio.to_thread(self.store.get_room, room_id)
  1951. room_agents = room_detail.get("agents")
  1952. if not isinstance(room_agents, list) or not all(
  1953. isinstance(item, Mapping) for item in room_agents
  1954. ):
  1955. raise GroupOrchestratorError("room.agents must be a list of objects")
  1956. prompt = build_run_prompt(projection, room_agents)
  1957. gateway = self._required_gateway()
  1958. try:
  1959. if expected_stored_id is None:
  1960. identity = await asyncio.to_thread(
  1961. gateway.create_session,
  1962. profile,
  1963. f"[群聊] {room_name} · {display_name}",
  1964. cwd_value,
  1965. [],
  1966. configuration,
  1967. )
  1968. else:
  1969. identity = await asyncio.to_thread(
  1970. gateway.resume_session, profile, expected_stored_id
  1971. )
  1972. except Exception: # noqa: BLE001 - never create a fallback session
  1973. logger.error("YaoYao group session create/resume failed")
  1974. await self._settle_unbound_failed(
  1975. run_id,
  1976. expected_stored_id=expected_stored_id,
  1977. stored_session_id=expected_stored_id,
  1978. error="Agent session setup failed",
  1979. )
  1980. return
  1981. identity = self._validated_identity(identity)
  1982. if identity.running:
  1983. # A busy resume may belong to another consumer. Persist rotation,
  1984. # but never adopt, submit to, interrupt, or close that runtime.
  1985. await self._settle_unbound_failed(
  1986. run_id,
  1987. expected_stored_id=expected_stored_id,
  1988. stored_session_id=identity.stored_id,
  1989. error="Agent stored session is already running",
  1990. )
  1991. return
  1992. bound = False
  1993. for delay in (*_STORE_RETRY_DELAYS, None):
  1994. try:
  1995. await asyncio.to_thread(
  1996. self.store.bind_run_runtime, run_id, identity.runtime_id
  1997. )
  1998. bound = True
  1999. break
  2000. except Exception: # noqa: BLE001 - exact bind replay is idempotent
  2001. if delay is None:
  2002. try:
  2003. durable = await asyncio.to_thread(self.store.get_run, run_id)
  2004. except Exception: # noqa: BLE001 - identity remains unknown
  2005. durable = None
  2006. bound = (
  2007. isinstance(durable, Mapping)
  2008. and durable.get("status") in {"running", "awaiting_input"}
  2009. and durable.get("runtimeSessionId") == identity.runtime_id
  2010. )
  2011. break
  2012. await asyncio.sleep(delay)
  2013. if not bound:
  2014. logger.error("Failed to bind YaoYao group runtime")
  2015. await self._close_unbound_runtime(identity.runtime_id)
  2016. await self._settle_unbound_failed(
  2017. run_id,
  2018. expected_stored_id=expected_stored_id,
  2019. stored_session_id=identity.stored_id,
  2020. error="Agent runtime binding failed",
  2021. )
  2022. return
  2023. async with self._room_serial(room_id):
  2024. async with self._submission_serial(agent_id):
  2025. durable = None
  2026. for delay in (*_STORE_RETRY_DELAYS, None):
  2027. try:
  2028. durable = await asyncio.to_thread(self.store.get_run, run_id)
  2029. break
  2030. except Exception: # noqa: BLE001 - unknown owner cannot submit
  2031. if delay is None:
  2032. break
  2033. await asyncio.sleep(delay)
  2034. if (
  2035. not isinstance(durable, Mapping)
  2036. or durable.get("status") not in {"running", "awaiting_input"}
  2037. or durable.get("runtimeSessionId") != identity.runtime_id
  2038. ):
  2039. logger.error("Bound YaoYao group runtime lost durable ownership")
  2040. await self._close_unbound_runtime(identity.runtime_id)
  2041. if not (
  2042. isinstance(durable, Mapping)
  2043. and durable.get("status") in _TERMINAL_RUN_STATUSES
  2044. ):
  2045. await self._settle_unbound_failed(
  2046. run_id,
  2047. expected_stored_id=expected_stored_id,
  2048. stored_session_id=identity.stored_id,
  2049. error="Agent runtime ownership check failed",
  2050. )
  2051. return
  2052. state = self._install_runtime(
  2053. run_id=run_id,
  2054. room_id=room_id,
  2055. agent_id=agent_id,
  2056. identity=identity,
  2057. expected_stored_id=expected_stored_id,
  2058. expected_context_seq=expected_context_seq,
  2059. through_seq=through_seq,
  2060. profile=profile,
  2061. reply_mode=reply_mode,
  2062. required_reply=required_reply,
  2063. )
  2064. if self._closing:
  2065. state.prompt_ready.set()
  2066. await self._finalize_state(
  2067. state,
  2068. outcome="interrupted",
  2069. error="Group orchestrator stopped",
  2070. interrupt=True,
  2071. )
  2072. return
  2073. try:
  2074. await self._run_state_blocking(
  2075. state, gateway.submit_prompt, identity.runtime_id, prompt
  2076. )
  2077. except Exception: # noqa: BLE001 - ACK is intentionally unknown
  2078. logger.error("YaoYao group prompt submission failed")
  2079. state.prompt_ready.set()
  2080. await self._finalize_state(
  2081. state,
  2082. outcome="failed",
  2083. error="Prompt submission result is unknown",
  2084. interrupt=True,
  2085. )
  2086. return
  2087. try:
  2088. for delay in (*_STORE_RETRY_DELAYS, None):
  2089. try:
  2090. await self._run_state_blocking(
  2091. state,
  2092. self.store.commit_prompt_submission,
  2093. run_id,
  2094. expected_stored_session_id=expected_stored_id,
  2095. stored_session_id=identity.stored_id,
  2096. runtime_session_id=identity.runtime_id,
  2097. expected_context_seq=expected_context_seq,
  2098. through_seq=through_seq,
  2099. )
  2100. state.prompt_committed = True
  2101. break
  2102. except Exception:
  2103. if delay is None:
  2104. raise
  2105. await asyncio.sleep(delay)
  2106. except Exception: # noqa: BLE001 - prompts are never retried
  2107. logger.error("Failed to commit YaoYao group prompt ACK")
  2108. state.prompt_ready.set()
  2109. await self._finalize_state(
  2110. state,
  2111. outcome="failed",
  2112. error="Prompt acknowledgement could not be committed",
  2113. interrupt=True,
  2114. )
  2115. return
  2116. finally:
  2117. state.prompt_ready.set()
  2118. if state.overflowed:
  2119. await self._finalize_state(
  2120. state,
  2121. outcome="failed",
  2122. error="Runtime control event buffer exceeded capacity",
  2123. interrupt=True,
  2124. )
  2125. return
  2126. shutdown_event = self._shutdown_event
  2127. if shutdown_event is None:
  2128. raise GroupOrchestratorError("Run started without shutdown state")
  2129. finished_wait = asyncio.create_task(state.finished.wait())
  2130. shutdown_wait = asyncio.create_task(shutdown_event.wait())
  2131. try:
  2132. done, pending = await asyncio.wait(
  2133. {finished_wait, shutdown_wait},
  2134. return_when=asyncio.FIRST_COMPLETED,
  2135. )
  2136. for waiter in pending:
  2137. waiter.cancel()
  2138. if finished_wait in done and state.finished.is_set():
  2139. return
  2140. if shutdown_wait in done and shutdown_event.is_set():
  2141. outcome = "interrupted"
  2142. error = "Group orchestrator stopped"
  2143. await self._finalize_state(
  2144. state, outcome=outcome, error=error, interrupt=True
  2145. )
  2146. finally:
  2147. for waiter in (finished_wait, shutdown_wait):
  2148. if not waiter.done():
  2149. waiter.cancel()
  2150. async def _fail_claim_without_projection(self, run_id: str) -> None:
  2151. for delay in (*_STORE_RETRY_DELAYS, None):
  2152. try:
  2153. await asyncio.to_thread(
  2154. self.store.transition_run,
  2155. run_id,
  2156. "failed",
  2157. error="Agent context projection failed",
  2158. )
  2159. return
  2160. except Exception: # noqa: BLE001 - reconcile before retrying
  2161. if await self._run_is_durably_terminal(run_id):
  2162. return
  2163. if delay is None:
  2164. logger.exception("Failed to settle malformed YaoYao group claim")
  2165. self._record_durable_failure(
  2166. "Durable claimed run settlement failed"
  2167. )
  2168. return
  2169. await asyncio.sleep(delay)
  2170. async def _settle_unbound_failed(
  2171. self,
  2172. run_id: str,
  2173. *,
  2174. expected_stored_id: str | None,
  2175. stored_session_id: str | None,
  2176. error: str,
  2177. ) -> None:
  2178. for delay in (*_STORE_RETRY_DELAYS, None):
  2179. try:
  2180. await asyncio.to_thread(
  2181. self.store.settle_run,
  2182. run_id,
  2183. runtime_session_id=None,
  2184. expected_stored_session_id=expected_stored_id,
  2185. stored_session_id=stored_session_id,
  2186. outcome="failed",
  2187. error=self._bounded_error(error),
  2188. )
  2189. return
  2190. except Exception: # noqa: BLE001 - reconcile before retrying
  2191. if await self._run_is_durably_terminal(run_id):
  2192. return
  2193. if delay is None:
  2194. logger.exception("Failed to settle unbound YaoYao group run")
  2195. self._record_durable_failure(
  2196. "Durable unbound run settlement failed"
  2197. )
  2198. return
  2199. await asyncio.sleep(delay)
  2200. def _record_durable_failure(self, message: str) -> None:
  2201. failure = GroupOrchestratorError(message)
  2202. with self._failure_lock:
  2203. self._durable_failures.append(failure)
  2204. async def _run_is_durably_terminal(self, run_id: str) -> bool:
  2205. try:
  2206. durable = await asyncio.to_thread(self.store.get_run, run_id)
  2207. except Exception: # noqa: BLE001 - missing is safe only for local cleanup
  2208. return False
  2209. return (
  2210. isinstance(durable, Mapping)
  2211. and durable.get("status") in _TERMINAL_RUN_STATUSES
  2212. )
  2213. async def _close_unbound_runtime(self, runtime_id: str) -> None:
  2214. try:
  2215. await asyncio.to_thread(self._required_gateway().close, runtime_id)
  2216. except Exception: # noqa: BLE001 - best effort after failed durable bind
  2217. logger.exception("Failed to close unbound YaoYao group runtime")
  2218. @staticmethod
  2219. def _validated_identity(identity: object) -> SessionIdentity:
  2220. stored_id = _required_string(
  2221. getattr(identity, "stored_id", None), "session.stored_id"
  2222. )
  2223. runtime_id = _required_string(
  2224. getattr(identity, "runtime_id", None), "session.runtime_id"
  2225. )
  2226. running = getattr(identity, "running", None)
  2227. if not isinstance(running, bool):
  2228. raise GroupOrchestratorError("session.running must be a boolean")
  2229. return SessionIdentity(stored_id, runtime_id, running)
  2230. def _install_runtime(
  2231. self,
  2232. *,
  2233. run_id: str,
  2234. room_id: str,
  2235. agent_id: str,
  2236. identity: SessionIdentity,
  2237. expected_stored_id: str | None,
  2238. expected_context_seq: int,
  2239. through_seq: int,
  2240. profile: str,
  2241. reply_mode: str,
  2242. required_reply: bool,
  2243. ) -> _RuntimeState:
  2244. with self._mapping_lock:
  2245. if identity.runtime_id in self._runtime_states:
  2246. raise GroupOrchestratorError(
  2247. "Gateway runtime is already mapped to an active generation"
  2248. )
  2249. self._next_generation += 1
  2250. state = _RuntimeState(
  2251. run_id=run_id,
  2252. room_id=room_id,
  2253. agent_id=agent_id,
  2254. runtime_id=identity.runtime_id,
  2255. generation=self._next_generation,
  2256. expected_stored_id=expected_stored_id,
  2257. session_stored_id=identity.stored_id,
  2258. expected_context_seq=expected_context_seq,
  2259. through_seq=through_seq,
  2260. profile=profile,
  2261. reply_mode=reply_mode,
  2262. required_reply=required_reply,
  2263. automatic_published=required_reply,
  2264. )
  2265. self._runtime_states[identity.runtime_id] = (
  2266. state.generation,
  2267. state,
  2268. )
  2269. state.event_task = asyncio.create_task(
  2270. self._consume_runtime_events(state),
  2271. name=f"yaoyao-group-runtime-{state.generation}",
  2272. )
  2273. return state
  2274. def _remove_runtime(self, state: _RuntimeState) -> None:
  2275. with self._mapping_lock:
  2276. current = self._runtime_states.get(state.runtime_id)
  2277. if current == (state.generation, state):
  2278. self._runtime_states.pop(state.runtime_id, None)
  2279. def _gateway_event_callback(
  2280. self,
  2281. runtime_id: str,
  2282. event_type: str,
  2283. payload: dict[str, object],
  2284. ) -> None:
  2285. """Bridge the Gateway callback thread into one bounded asyncio queue."""
  2286. if event_type not in _PERSISTED_EVENTS:
  2287. return
  2288. with self._mapping_lock:
  2289. current = self._runtime_states.get(runtime_id)
  2290. if current is None:
  2291. logger.debug("Ignored event for an unmapped group runtime")
  2292. return
  2293. generation, _state = current
  2294. loop = self._loop
  2295. queue = self._event_queue
  2296. if loop is None or queue is None or loop.is_closed():
  2297. return
  2298. try:
  2299. safe_payload = _strict_json_copy(payload, "gateway.payload")
  2300. except GroupOrchestratorError:
  2301. logger.warning("Ignored malformed group Gateway payload")
  2302. return
  2303. if not isinstance(safe_payload, dict):
  2304. return
  2305. envelope = _GatewayEvent(
  2306. runtime_id=runtime_id,
  2307. generation=generation,
  2308. event_type=event_type,
  2309. payload=safe_payload,
  2310. )
  2311. if threading.get_ident() == self._loop_thread_id:
  2312. # The production transport uses a dedicated worker. Supporting a
  2313. # loop-local test callback is still safe while the queue has room.
  2314. try:
  2315. queue.put_nowait(envelope)
  2316. except asyncio.QueueFull:
  2317. logger.error("Loop-local group Gateway callback exceeded capacity")
  2318. return
  2319. try:
  2320. future = asyncio.run_coroutine_threadsafe(queue.put(envelope), loop)
  2321. future.result()
  2322. except (RuntimeError, asyncio.CancelledError):
  2323. return
  2324. async def _consume_events(self) -> None:
  2325. queue = self._event_queue
  2326. if queue is None:
  2327. raise GroupOrchestratorError("Event consumer has no queue")
  2328. while True:
  2329. event = await queue.get()
  2330. try:
  2331. if event is None:
  2332. return
  2333. await self._handle_event(event)
  2334. except Exception: # noqa: BLE001 - isolate all Gateway callbacks
  2335. logger.exception("Failed to consume YaoYao group Gateway event")
  2336. finally:
  2337. queue.task_done()
  2338. async def _handle_event(self, event: _GatewayEvent) -> None:
  2339. if event.event_type not in _PERSISTED_EVENTS:
  2340. return
  2341. with self._mapping_lock:
  2342. current = self._runtime_states.get(event.runtime_id)
  2343. if current is None or current[0] != event.generation:
  2344. logger.debug("Ignored stale group Gateway generation")
  2345. return
  2346. state = current[1]
  2347. if state.finished.is_set() or state.finalizing:
  2348. return
  2349. try:
  2350. state.pending_events.put_nowait(event)
  2351. except asyncio.QueueFull:
  2352. # The transport already applies global backpressure. A per-runtime
  2353. # overflow is an invariant breach: fail that run without blocking
  2354. # unrelated sessions in the one global router.
  2355. state.overflowed = True
  2356. state.prompt_ready.set()
  2357. async def _consume_runtime_events(self, state: _RuntimeState) -> None:
  2358. """Serialize one runtime without head-of-line blocking other sessions."""
  2359. await state.prompt_ready.wait()
  2360. while not state.finished.is_set():
  2361. if state.finalizing:
  2362. return
  2363. if state.overflowed and state.prompt_committed:
  2364. await self._finalize_state(
  2365. state,
  2366. outcome="failed",
  2367. error="Runtime control event buffer exceeded capacity",
  2368. interrupt=True,
  2369. )
  2370. return
  2371. timeout: float | None = None
  2372. if state.stream_dirty and not (
  2373. state.reply_mode == "automatic" and not state.automatic_published
  2374. ):
  2375. timeout = max(
  2376. 0.0,
  2377. STREAM_FLUSH_INTERVAL
  2378. - (asyncio.get_running_loop().time() - state.last_stream_flush),
  2379. )
  2380. try:
  2381. if timeout is None:
  2382. event = await state.pending_events.get()
  2383. else:
  2384. event = await asyncio.wait_for(
  2385. state.pending_events.get(), timeout=timeout
  2386. )
  2387. except asyncio.TimeoutError:
  2388. async with state.mutation_lock:
  2389. if state.finished.is_set() or state.finalizing:
  2390. return
  2391. with self._mapping_lock:
  2392. current = self._runtime_states.get(state.runtime_id)
  2393. if current != (state.generation, state):
  2394. continue
  2395. try:
  2396. await self._flush_streaming(state)
  2397. except Exception: # noqa: BLE001 - fail this runtime durably
  2398. logger.error("Failed to persist YaoYao group streaming message")
  2399. await self._finalize_state_locked(
  2400. state,
  2401. outcome="failed",
  2402. error="Streaming Agent message could not be persisted",
  2403. interrupt=True,
  2404. )
  2405. continue
  2406. try:
  2407. if event is None:
  2408. return
  2409. if not state.prompt_committed:
  2410. continue
  2411. with self._mapping_lock:
  2412. current = self._runtime_states.get(event.runtime_id)
  2413. if current != (event.generation, state):
  2414. continue
  2415. state.event_index += 1
  2416. async with state.mutation_lock:
  2417. try:
  2418. await self._apply_runtime_event(state, event)
  2419. except Exception: # noqa: BLE001 - fail this runtime only
  2420. logger.error("Failed to process YaoYao group runtime event")
  2421. await self._finalize_state_locked(
  2422. state,
  2423. outcome="failed",
  2424. error="Gateway control event could not be persisted",
  2425. interrupt=True,
  2426. )
  2427. except Exception: # noqa: BLE001 - isolate this runtime only
  2428. logger.error("Failed to lock YaoYao group runtime event")
  2429. await self._finalize_state(
  2430. state,
  2431. outcome="failed",
  2432. error="Gateway control event could not be persisted",
  2433. interrupt=True,
  2434. )
  2435. finally:
  2436. state.pending_events.task_done()
  2437. async def _apply_runtime_event(
  2438. self, state: _RuntimeState, event: _GatewayEvent
  2439. ) -> None:
  2440. event_type = event.event_type
  2441. payload = event.payload
  2442. if state.finalizing:
  2443. return
  2444. self._require_safe_control_payload(event_type, payload)
  2445. if state.complete_seen and event_type != "session.info":
  2446. return
  2447. if event_type == "message.start":
  2448. if state.reply_mode == "mentioned" or state.automatic_published:
  2449. await self._persist_message_snapshot(state, status="streaming")
  2450. return
  2451. if event_type in {"message.delta", "reasoning.delta"}:
  2452. text = payload.get("text")
  2453. if not isinstance(text, str):
  2454. raise GroupOrchestratorError("Gateway delta text is invalid")
  2455. if event_type == "message.delta":
  2456. state.content = self._append_bounded_text(
  2457. state.content, text, "content"
  2458. )
  2459. else:
  2460. state.reasoning = self._append_bounded_text(
  2461. state.reasoning, text, "reasoning"
  2462. )
  2463. state.stream_dirty = True
  2464. now = asyncio.get_running_loop().time()
  2465. if state.last_stream_flush == 0.0 or (
  2466. now - state.last_stream_flush >= STREAM_FLUSH_INTERVAL
  2467. ):
  2468. await self._flush_streaming(state)
  2469. return
  2470. if event_type.startswith("tool."):
  2471. await self._publish_automatic_state(state)
  2472. await self._flush_streaming(state)
  2473. self._merge_tool_event(state, event_type, payload)
  2474. await self._persist_message_snapshot(state, status="streaming")
  2475. return
  2476. if event_type == "approval.request":
  2477. await self._publish_automatic_state(state)
  2478. await self._flush_streaming(state)
  2479. await self._persist_approval(state, payload)
  2480. return
  2481. if event_type == "clarify.request":
  2482. await self._publish_automatic_state(state)
  2483. await self._flush_streaming(state)
  2484. await self._persist_clarification(state, payload)
  2485. return
  2486. if event_type == "clarify.expire":
  2487. await self._publish_automatic_state(state)
  2488. await self._flush_streaming(state)
  2489. await self._expire_clarification(state, payload)
  2490. return
  2491. if event_type == "message.complete":
  2492. if state.complete_seen:
  2493. return
  2494. await self._flush_streaming(state)
  2495. await self._expire_pending_interactions(state)
  2496. status = payload.get("status")
  2497. outcomes = {
  2498. "complete": "completed",
  2499. "error": "failed",
  2500. "interrupted": "interrupted",
  2501. }
  2502. outcome = outcomes.get(status) if isinstance(status, str) else None
  2503. if outcome is None:
  2504. outcome = "failed"
  2505. state.terminal_error = "Invalid message.complete status"
  2506. else:
  2507. raw_error = payload.get("error")
  2508. if outcome != "completed" and isinstance(raw_error, str):
  2509. state.terminal_error = self._bounded_error(raw_error)
  2510. for tool in state.tool_state:
  2511. if tool.get("status") == "running":
  2512. tool["status"] = "completed" if outcome == "completed" else "failed"
  2513. final_text = payload.get("text", payload.get("content"))
  2514. if isinstance(final_text, str):
  2515. state.content = self._bounded_text(final_text, "content")
  2516. final_reasoning = payload.get("reasoning")
  2517. if isinstance(final_reasoning, str):
  2518. state.reasoning = self._bounded_text(final_reasoning, "reasoning")
  2519. automatic_unpublished = (
  2520. state.reply_mode == "automatic" and not state.automatic_published
  2521. )
  2522. if state.reply_mode == "automatic":
  2523. state.content = _sanitize_no_reply_text(state.content)
  2524. state.reasoning = _sanitize_no_reply_text(state.reasoning)
  2525. if (
  2526. state.required_reply
  2527. and outcome == "completed"
  2528. and not state.content.strip()
  2529. and not state.had_visible_interaction
  2530. ):
  2531. state.content = _HOST_FALLBACK_REPLY
  2532. suppress_automatic = automatic_unpublished and (
  2533. outcome != "completed" or not state.content.strip()
  2534. )
  2535. if suppress_automatic:
  2536. state.content = ""
  2537. state.reasoning = ""
  2538. state.tool_state = []
  2539. elif automatic_unpublished:
  2540. state.automatic_published = True
  2541. # GroupStore only accepts completed as a direct running-run
  2542. # terminal upsert. Failure/interruption must first persist the
  2543. # partial body as streaming; settle_run atomically maps both run
  2544. # and response message to the final failed/interrupted status.
  2545. message_status = "completed" if outcome == "completed" else "streaming"
  2546. try:
  2547. await self._persist_message_snapshot(
  2548. state,
  2549. status=message_status,
  2550. error=state.terminal_error,
  2551. publish=not suppress_automatic,
  2552. )
  2553. state.stream_dirty = False
  2554. except Exception:
  2555. await self._finalize_state_locked(
  2556. state,
  2557. outcome="failed",
  2558. error="Terminal Agent message could not be persisted",
  2559. interrupt=True,
  2560. )
  2561. return
  2562. state.terminal_status = outcome
  2563. state.complete_seen = True
  2564. state.complete_index = state.event_index
  2565. state.grace_task = asyncio.create_task(
  2566. self._await_terminal_info(state),
  2567. name=f"yaoyao-group-grace-{state.generation}",
  2568. )
  2569. return
  2570. if event_type == "session.info":
  2571. await self._flush_streaming(state)
  2572. if (
  2573. payload.get("running") is not False
  2574. or not state.complete_seen
  2575. or state.event_index <= state.complete_index
  2576. ):
  2577. return
  2578. profile_name = payload.get("profile_name")
  2579. if profile_name != state.profile:
  2580. await self._finalize_state_locked(
  2581. state,
  2582. outcome="failed",
  2583. error="Final session profile is missing",
  2584. stored_session_id=state.session_stored_id,
  2585. interrupt=True,
  2586. )
  2587. return
  2588. rotated = payload.get("stored_session_id")
  2589. if not isinstance(rotated, str) or not rotated:
  2590. await self._finalize_state_locked(
  2591. state,
  2592. outcome="failed",
  2593. error="Final session identity is missing",
  2594. stored_session_id=state.session_stored_id,
  2595. interrupt=True,
  2596. )
  2597. return
  2598. actual_model = payload.get("model")
  2599. actual_provider = payload.get("provider")
  2600. if (
  2601. isinstance(actual_model, str)
  2602. and actual_model.strip()
  2603. and isinstance(actual_provider, str)
  2604. and actual_provider.strip()
  2605. ):
  2606. state.actual_model = actual_model.strip()
  2607. state.actual_provider = actual_provider.strip()
  2608. actual_reasoning = payload.get("reasoning_effort")
  2609. if isinstance(actual_reasoning, str) and actual_reasoning.strip():
  2610. state.actual_reasoning_effort = actual_reasoning.strip()
  2611. actual_fast = payload.get("fast")
  2612. if isinstance(actual_fast, bool):
  2613. state.actual_fast_mode = actual_fast
  2614. state.terminal_info_received.set()
  2615. await self._finalize_state_locked(
  2616. state,
  2617. outcome=state.terminal_status or "failed",
  2618. error=state.terminal_error,
  2619. stored_session_id=rotated,
  2620. interrupt=False,
  2621. )
  2622. return
  2623. if event_type == "error":
  2624. await self._flush_streaming(state)
  2625. await self._finalize_state_locked(
  2626. state,
  2627. outcome="failed",
  2628. error="Agent Gateway reported a failure",
  2629. interrupt=True,
  2630. )
  2631. @staticmethod
  2632. def _require_safe_control_payload(
  2633. event_type: str, payload: Mapping[str, object]
  2634. ) -> None:
  2635. truncated = payload.get("truncatedFields")
  2636. invalid = payload.get("invalidFields")
  2637. truncated_fields = (
  2638. {item for item in truncated if isinstance(item, str)}
  2639. if isinstance(truncated, list)
  2640. else set()
  2641. )
  2642. invalid_fields = (
  2643. {item for item in invalid if isinstance(item, str)}
  2644. if isinstance(invalid, list)
  2645. else set()
  2646. )
  2647. if payload.get("transportInvalidPayload") is True and not invalid_fields:
  2648. raise GroupOrchestratorError("Gateway control payload is invalid")
  2649. critical: set[str]
  2650. if event_type == "message.complete":
  2651. critical = {"status", "text", "reasoning"}
  2652. elif event_type == "session.info":
  2653. critical = {"running", "stored_session_id", "profile_name"}
  2654. elif event_type in {"approval.request", "clarify.request"}:
  2655. if payload.get("transportTruncated") is True:
  2656. raise GroupOrchestratorError(
  2657. "Gateway interaction request was truncated"
  2658. )
  2659. critical = {
  2660. "request_id",
  2661. "question",
  2662. "command",
  2663. "choices",
  2664. "tool",
  2665. "name",
  2666. }
  2667. elif event_type == "clarify.expire":
  2668. critical = {"request_id"}
  2669. elif event_type.startswith("tool."):
  2670. critical = {"tool_id", "id"}
  2671. if event_type == "tool.output_risk":
  2672. critical |= {"risk", "redacted"}
  2673. elif event_type == "error":
  2674. critical = {"message", "error", "reason"}
  2675. else:
  2676. critical = set()
  2677. if critical & (truncated_fields | invalid_fields):
  2678. raise GroupOrchestratorError(
  2679. "Gateway control payload lost a critical field"
  2680. )
  2681. if payload.get("transportInvalidPayload") is True:
  2682. # Known-invalid non-critical diagnostics never justify trusting an
  2683. # approval/tool/terminal control frame as authoritative.
  2684. if event_type != "session.info":
  2685. raise GroupOrchestratorError("Gateway control payload is invalid")
  2686. if (
  2687. payload.get("transportTruncated") is True
  2688. and not truncated_fields
  2689. and event_type
  2690. in {
  2691. "message.complete",
  2692. "session.info",
  2693. "approval.request",
  2694. "clarify.request",
  2695. "clarify.expire",
  2696. }
  2697. ):
  2698. raise GroupOrchestratorError("Gateway control payload was replaced")
  2699. async def _flush_streaming(self, state: _RuntimeState) -> None:
  2700. if not state.stream_dirty:
  2701. return
  2702. if state.reply_mode == "automatic" and not state.automatic_published:
  2703. return
  2704. await self._persist_message_snapshot(state, status="streaming")
  2705. state.stream_dirty = False
  2706. state.last_stream_flush = asyncio.get_running_loop().time()
  2707. async def _persist_message_snapshot(
  2708. self,
  2709. state: _RuntimeState,
  2710. *,
  2711. status: str,
  2712. error: str = "",
  2713. publish: bool | None = None,
  2714. ) -> None:
  2715. should_publish = (
  2716. state.reply_mode == "mentioned" or state.automatic_published
  2717. if publish is None
  2718. else publish
  2719. )
  2720. content = (
  2721. _sanitize_no_reply_text(state.content)
  2722. if state.reply_mode == "automatic"
  2723. else state.content
  2724. )
  2725. reasoning = (
  2726. _sanitize_no_reply_text(state.reasoning)
  2727. if state.reply_mode == "automatic"
  2728. else state.reasoning
  2729. )
  2730. persisted = False
  2731. for delay in (*_STORE_RETRY_DELAYS, None):
  2732. try:
  2733. await self._run_state_blocking(
  2734. state,
  2735. self.store.upsert_agent_message,
  2736. state.run_id,
  2737. content=content,
  2738. reasoning=reasoning,
  2739. tool_state=state.tool_state,
  2740. status=status,
  2741. error=error,
  2742. publish=should_publish,
  2743. )
  2744. persisted = True
  2745. break
  2746. except Exception: # noqa: BLE001 - reconcile a lost Store reply
  2747. try:
  2748. persisted = bool(
  2749. await self._run_state_blocking(
  2750. state,
  2751. self._terminal_message_is_durable,
  2752. state,
  2753. content=content,
  2754. reasoning=reasoning,
  2755. tool_state=state.tool_state,
  2756. status=status,
  2757. error=error,
  2758. )
  2759. )
  2760. except Exception: # noqa: BLE001 - retry the exact snapshot
  2761. persisted = False
  2762. if persisted:
  2763. break
  2764. if delay is None:
  2765. break
  2766. await asyncio.sleep(delay)
  2767. if not persisted:
  2768. raise GroupOrchestratorError("Agent message snapshot is not durable")
  2769. async def _publish_automatic_state(self, state: _RuntimeState) -> None:
  2770. if state.reply_mode != "automatic" or state.automatic_published:
  2771. return
  2772. state.automatic_published = True
  2773. await self._persist_message_snapshot(state, status="streaming", publish=True)
  2774. state.stream_dirty = False
  2775. state.last_stream_flush = asyncio.get_running_loop().time()
  2776. @staticmethod
  2777. def _bounded_text(value: str, field_name: str) -> str:
  2778. try:
  2779. size = len(value.encode("utf-8"))
  2780. except UnicodeEncodeError as error:
  2781. raise GroupOrchestratorError(
  2782. f"Gateway {field_name} is not valid UTF-8"
  2783. ) from error
  2784. if size > MAX_MESSAGE_BYTES:
  2785. raise GroupOrchestratorError(
  2786. f"Gateway {field_name} exceeds the group message limit"
  2787. )
  2788. return value
  2789. @classmethod
  2790. def _append_bounded_text(cls, existing: str, delta: str, field_name: str) -> str:
  2791. return cls._bounded_text(existing + delta, field_name)
  2792. def _merge_tool_event(
  2793. self,
  2794. state: _RuntimeState,
  2795. event_type: str,
  2796. payload: Mapping[str, object],
  2797. ) -> None:
  2798. if payload.get("transportInvalidPayload") is True:
  2799. raise GroupOrchestratorError("Gateway tool payload is invalid")
  2800. tool_id = payload.get("tool_id", payload.get("id"))
  2801. tool_id = _required_string(tool_id, "tool.tool_id")
  2802. if tool_id == _TOOL_TRUNCATION_ID:
  2803. raise GroupOrchestratorError("Gateway tool id is reserved")
  2804. if len(tool_id.encode("utf-8")) > _TOOL_TEXT_LIMIT:
  2805. raise GroupOrchestratorError("Gateway tool id exceeds the limit")
  2806. name_value = payload.get("name")
  2807. name = (
  2808. self._bounded_tool_text(name_value)
  2809. if isinstance(name_value, str) and name_value
  2810. else None
  2811. )
  2812. existing = self._tool_state_for_id(state, tool_id)
  2813. tool: dict[str, object] = (
  2814. dict(existing)
  2815. if existing is not None
  2816. else {"id": tool_id, "name": name or "unknown", "status": "running"}
  2817. )
  2818. if name is not None:
  2819. tool["name"] = name
  2820. if "args" in payload:
  2821. tool["arguments"] = self._bounded_tool_value(payload["args"], "tool.args")
  2822. elif "args_text" in payload:
  2823. args_text = payload["args_text"]
  2824. if not isinstance(args_text, str):
  2825. raise GroupOrchestratorError("Gateway tool args_text is invalid")
  2826. tool["arguments"] = self._bounded_tool_text(args_text)
  2827. if event_type == "tool.output_risk":
  2828. output_risk, risk_preview = self._project_output_risk(payload)
  2829. tool["outputRisk"] = output_risk
  2830. tool["preview"] = risk_preview
  2831. preview = next(
  2832. (
  2833. value
  2834. for key in ("summary", "context", "result_text", "args_text")
  2835. if isinstance((value := payload.get(key)), str) and value
  2836. ),
  2837. None,
  2838. )
  2839. if preview is not None and event_type != "tool.output_risk":
  2840. tool["preview"] = self._bounded_tool_text(preview)
  2841. if "result" in payload:
  2842. tool["result"] = self._bounded_tool_value(payload["result"], "tool.result")
  2843. error = payload.get("error")
  2844. if isinstance(error, str) and error:
  2845. tool["error"] = self._bounded_tool_text(error)
  2846. duration = payload.get("duration_s")
  2847. if (
  2848. isinstance(duration, (int, float))
  2849. and not isinstance(duration, bool)
  2850. and math.isfinite(duration)
  2851. and duration >= 0
  2852. ):
  2853. tool["durationSeconds"] = float(duration)
  2854. if event_type == "tool.complete":
  2855. raw_status = payload.get("status")
  2856. tool["status"] = (
  2857. "failed"
  2858. if tool.get("error")
  2859. or raw_status in {"error", "failed", "cancelled", "canceled"}
  2860. else "completed"
  2861. )
  2862. elif existing is None:
  2863. tool["status"] = "running"
  2864. self._upsert_tool_state(state, tool)
  2865. @staticmethod
  2866. def _bounded_tool_text(value: str, limit: int = _TOOL_TEXT_LIMIT) -> str:
  2867. try:
  2868. encoded = value.encode("utf-8")
  2869. except UnicodeEncodeError as error:
  2870. raise GroupOrchestratorError(
  2871. "Gateway tool text is not valid UTF-8"
  2872. ) from error
  2873. if len(encoded) <= limit:
  2874. return value
  2875. suffix = "..."
  2876. prefix = encoded[: limit - len(suffix)]
  2877. while prefix:
  2878. try:
  2879. return prefix.decode("utf-8") + suffix
  2880. except UnicodeDecodeError:
  2881. prefix = prefix[:-1]
  2882. return suffix
  2883. @classmethod
  2884. def _bounded_tool_value(cls, value: object, field_name: str) -> object:
  2885. copied = _strict_json_copy(value, field_name)
  2886. encoded = json.dumps(
  2887. copied,
  2888. ensure_ascii=False,
  2889. sort_keys=True,
  2890. separators=(",", ":"),
  2891. allow_nan=False,
  2892. ).encode("utf-8")
  2893. if len(encoded) <= _TOOL_VALUE_LIMIT:
  2894. return copied
  2895. return {"originalBytes": len(encoded), "truncated": True}
  2896. @classmethod
  2897. def _project_output_risk(
  2898. cls, payload: Mapping[str, object]
  2899. ) -> tuple[dict[str, object], str]:
  2900. risk_value = payload.get("risk")
  2901. findings_value = payload.get("findings")
  2902. redacted = payload.get("redacted")
  2903. if not isinstance(risk_value, str) or not risk_value:
  2904. raise GroupOrchestratorError("Gateway tool output risk is invalid")
  2905. if not isinstance(findings_value, list) or not all(
  2906. isinstance(item, str) for item in findings_value
  2907. ):
  2908. raise GroupOrchestratorError("Gateway tool output findings are invalid")
  2909. if not isinstance(redacted, bool):
  2910. raise GroupOrchestratorError("Gateway tool output redaction is invalid")
  2911. risk = cls._bounded_tool_text(risk_value, 256)
  2912. findings = [
  2913. cls._bounded_tool_text(item, 512)
  2914. for item in findings_value[:_TOOL_FINDING_LIMIT]
  2915. ]
  2916. truncated = (
  2917. len(findings_value) > len(findings)
  2918. or risk != risk_value
  2919. or any(
  2920. projected != original
  2921. for projected, original in zip(findings, findings_value)
  2922. )
  2923. )
  2924. transport_truncated = payload.get("truncatedFields")
  2925. if isinstance(transport_truncated, list) and "findings" in transport_truncated:
  2926. truncated = True
  2927. output_risk: dict[str, object] = {
  2928. "risk": risk,
  2929. "findings": findings,
  2930. "redacted": redacted,
  2931. }
  2932. if truncated:
  2933. output_risk["truncated"] = True
  2934. preview = f"Output risk: {risk}"
  2935. if findings:
  2936. preview += "; findings: " + ", ".join(findings)
  2937. preview += f"; redacted: {'true' if redacted else 'false'}"
  2938. return output_risk, cls._bounded_tool_text(preview)
  2939. @staticmethod
  2940. def _tool_state_for_id(
  2941. state: _RuntimeState, tool_id: str
  2942. ) -> dict[str, object] | None:
  2943. index = state.tool_indexes.get(tool_id)
  2944. return None if index is None else state.tool_state[index]
  2945. @classmethod
  2946. def _upsert_tool_state(cls, state: _RuntimeState, tool: dict[str, object]) -> None:
  2947. tool_id = str(tool["id"])
  2948. index = state.tool_indexes.get(tool_id)
  2949. candidate = [dict(item) for item in state.tool_state]
  2950. if index is None:
  2951. candidate.append(dict(tool))
  2952. else:
  2953. candidate[index] = dict(tool)
  2954. candidate = cls._fit_tool_state(candidate, newest_tool_id=tool_id)
  2955. state.tool_state = candidate
  2956. state.tool_indexes = {
  2957. str(item["id"]): offset for offset, item in enumerate(candidate)
  2958. }
  2959. @classmethod
  2960. def _fit_tool_state(
  2961. cls,
  2962. candidate: list[dict[str, object]],
  2963. *,
  2964. newest_tool_id: str,
  2965. ) -> list[dict[str, object]]:
  2966. def encoded_size(items: list[dict[str, object]]) -> int:
  2967. try:
  2968. return len(
  2969. json.dumps(
  2970. items,
  2971. ensure_ascii=False,
  2972. sort_keys=True,
  2973. separators=(",", ":"),
  2974. allow_nan=False,
  2975. ).encode("utf-8")
  2976. )
  2977. except (TypeError, ValueError, UnicodeEncodeError) as error:
  2978. raise GroupOrchestratorError("Gateway tool state is invalid") from error
  2979. if (
  2980. len(candidate) <= _TOOL_COUNT_LIMIT
  2981. and encoded_size(candidate) <= _TOOL_STATE_INTERNAL_LIMIT
  2982. ):
  2983. return candidate
  2984. newest = next(
  2985. (item for item in candidate if item.get("id") == newest_tool_id), None
  2986. )
  2987. if newest is None:
  2988. raise GroupOrchestratorError("Gateway tool state merge lost its tool")
  2989. retained = [
  2990. item
  2991. for item in candidate
  2992. if item.get("id") not in {newest_tool_id, _TOOL_TRUNCATION_ID}
  2993. ]
  2994. marker = {
  2995. "id": _TOOL_TRUNCATION_ID,
  2996. "name": "hermes",
  2997. "status": "completed",
  2998. "preview": "Earlier tool state omitted by the group size limit",
  2999. }
  3000. fitted = [*retained, marker, newest]
  3001. while (
  3002. len(fitted) > _TOOL_COUNT_LIMIT
  3003. or encoded_size(fitted) > _TOOL_STATE_INTERNAL_LIMIT
  3004. ):
  3005. if not retained:
  3006. break
  3007. retained.pop(0)
  3008. fitted = [*retained, marker, newest]
  3009. if (
  3010. len(fitted) > _TOOL_COUNT_LIMIT
  3011. or encoded_size(fitted) > _TOOL_STATE_INTERNAL_LIMIT
  3012. ):
  3013. compact_newest: dict[str, object] = {
  3014. "id": newest_tool_id,
  3015. "name": cls._bounded_tool_text(
  3016. str(newest.get("name") or "unknown"), 256
  3017. ),
  3018. "status": str(newest.get("status") or "running"),
  3019. "preview": cls._bounded_tool_text(
  3020. str(newest.get("preview") or "Tool state truncated"), 4096
  3021. ),
  3022. }
  3023. output_risk = newest.get("outputRisk")
  3024. if isinstance(output_risk, Mapping):
  3025. compact_newest["outputRisk"] = {
  3026. "risk": cls._bounded_tool_text(
  3027. str(output_risk.get("risk") or "unknown"), 256
  3028. ),
  3029. "findings": [
  3030. cls._bounded_tool_text(item, 512)
  3031. for item in output_risk.get("findings", [])[
  3032. :_TOOL_FINDING_LIMIT
  3033. ]
  3034. if isinstance(item, str)
  3035. ],
  3036. "redacted": output_risk.get("redacted") is True,
  3037. "truncated": True,
  3038. }
  3039. fitted = [marker, compact_newest]
  3040. if encoded_size(fitted) > MAX_TOOL_STATE_BYTES:
  3041. raise GroupOrchestratorError("Gateway tool state exceeds the limit")
  3042. return fitted
  3043. async def _store_retry(
  3044. self,
  3045. state: _RuntimeState,
  3046. function: Callable[..., object],
  3047. *args: object,
  3048. **kwargs: object,
  3049. ) -> object:
  3050. last_error: Exception | None = None
  3051. for delay in (*_STORE_RETRY_DELAYS, None):
  3052. try:
  3053. return await self._run_state_blocking(state, function, *args, **kwargs)
  3054. except Exception as error: # noqa: BLE001 - exact Store APIs replay
  3055. last_error = error
  3056. if delay is None:
  3057. break
  3058. await asyncio.sleep(delay)
  3059. if last_error is None:
  3060. raise GroupOrchestratorError("Store operation failed")
  3061. raise last_error
  3062. async def _persist_approval(
  3063. self, state: _RuntimeState, payload: Mapping[str, object]
  3064. ) -> None:
  3065. if payload.get("transportInvalidPayload") is True:
  3066. raise GroupOrchestratorError("Gateway approval payload is invalid")
  3067. raw_choices = payload.get("choices")
  3068. if raw_choices is None:
  3069. choices = ["deny"]
  3070. elif (
  3071. not isinstance(raw_choices, list)
  3072. or not raw_choices
  3073. or not all(
  3074. isinstance(choice, str) and choice in _APPROVAL_CHOICES
  3075. for choice in raw_choices
  3076. )
  3077. ):
  3078. raise GroupOrchestratorError("Gateway approval choices are invalid")
  3079. else:
  3080. choices = list(dict.fromkeys(raw_choices))
  3081. safe_payload = dict(payload)
  3082. safe_payload["choices"] = choices
  3083. state.approval_ordinal += 1
  3084. interaction_id = "approval-" + str(
  3085. uuid.uuid5(
  3086. uuid.NAMESPACE_URL,
  3087. f"{state.run_id}:{state.generation}:approval:{state.approval_ordinal}",
  3088. )
  3089. )
  3090. interaction = await self._store_retry(
  3091. state,
  3092. self.store.create_gateway_interaction,
  3093. state.run_id,
  3094. kind="approval",
  3095. gateway_interaction_id=interaction_id,
  3096. payload=safe_payload,
  3097. )
  3098. if not isinstance(interaction, Mapping):
  3099. raise GroupOrchestratorError("Stored approval is invalid")
  3100. local_id = _required_string(interaction.get("id"), "interaction.id")
  3101. state.had_visible_interaction = True
  3102. state.pending_interaction_ids.add(local_id)
  3103. async def _persist_clarification(
  3104. self, state: _RuntimeState, payload: Mapping[str, object]
  3105. ) -> None:
  3106. if payload.get("transportInvalidPayload") is True:
  3107. raise GroupOrchestratorError("Gateway clarification payload is invalid")
  3108. request_id = _required_string(
  3109. payload.get("request_id"), "clarification.request_id"
  3110. )
  3111. interaction = await self._store_retry(
  3112. state,
  3113. self.store.create_gateway_interaction,
  3114. state.run_id,
  3115. kind="clarification",
  3116. gateway_interaction_id=request_id,
  3117. payload=dict(payload),
  3118. )
  3119. if not isinstance(interaction, Mapping):
  3120. raise GroupOrchestratorError("Stored clarification is invalid")
  3121. local_id = _required_string(interaction.get("id"), "interaction.id")
  3122. state.had_visible_interaction = True
  3123. state.clarification_ids[request_id] = local_id
  3124. state.pending_interaction_ids.add(local_id)
  3125. async def _expire_clarification(
  3126. self, state: _RuntimeState, payload: Mapping[str, object]
  3127. ) -> None:
  3128. request_id = _required_string(
  3129. payload.get("request_id"), "clarification.request_id"
  3130. )
  3131. local_id = state.clarification_ids.get(request_id)
  3132. if local_id is None:
  3133. return
  3134. await self._store_retry(state, self.store.expire_interaction, local_id)
  3135. state.pending_interaction_ids.discard(local_id)
  3136. async def _expire_pending_interactions(self, state: _RuntimeState) -> None:
  3137. for interaction_id in tuple(sorted(state.pending_interaction_ids)):
  3138. await self._store_retry(
  3139. state, self.store.expire_interaction, interaction_id
  3140. )
  3141. state.pending_interaction_ids.discard(interaction_id)
  3142. def _terminal_message_is_durable(
  3143. self,
  3144. state: _RuntimeState,
  3145. *,
  3146. content: str,
  3147. reasoning: str,
  3148. tool_state: list[dict[str, object]],
  3149. status: str,
  3150. error: str,
  3151. ) -> bool:
  3152. """Reconcile a lost Store reply without accepting another run's write."""
  3153. run = self.store.get_run(state.run_id)
  3154. if not isinstance(run, Mapping):
  3155. return False
  3156. response_message_id = run.get("responseMessageId")
  3157. if not isinstance(response_message_id, str) or not response_message_id:
  3158. return False
  3159. if (
  3160. run.get("id") != state.run_id
  3161. or run.get("roomId") != state.room_id
  3162. or run.get("agentId") != state.agent_id
  3163. or run.get("runtimeSessionId") != state.runtime_id
  3164. or run.get("status") not in {"running", "awaiting_input"}
  3165. ):
  3166. return False
  3167. message = self.store.get_message(response_message_id)
  3168. if not isinstance(message, Mapping):
  3169. return False
  3170. return (
  3171. message.get("id") == response_message_id
  3172. and message.get("roomId") == state.room_id
  3173. and message.get("senderKind") == "agent"
  3174. and message.get("senderId") == state.agent_id
  3175. and message.get("content") == content
  3176. and message.get("reasoning") == reasoning
  3177. and message.get("toolState") == tool_state
  3178. and message.get("status") == status
  3179. and message.get("error") == error
  3180. )
  3181. async def _run_state_blocking(
  3182. self,
  3183. state: _RuntimeState,
  3184. function: Callable[..., object],
  3185. *args: object,
  3186. **kwargs: object,
  3187. ) -> object:
  3188. """Track a blocking call so shutdown never abandons its worker thread."""
  3189. return await self._run_states_blocking([state], function, *args, **kwargs)
  3190. @staticmethod
  3191. async def _run_states_blocking(
  3192. states: list[_RuntimeState],
  3193. function: Callable[..., object],
  3194. *args: object,
  3195. **kwargs: object,
  3196. ) -> object:
  3197. """Shield one worker call and expose it to every affected runtime."""
  3198. task: asyncio.Task[object] = asyncio.create_task(
  3199. asyncio.to_thread(function, *args, **kwargs)
  3200. )
  3201. unique_states = tuple({id(state): state for state in states}.values())
  3202. for state in unique_states:
  3203. state.inflight_calls.add(task)
  3204. def finished(done: asyncio.Task[object]) -> None:
  3205. for state in unique_states:
  3206. state.inflight_calls.discard(done)
  3207. task.add_done_callback(finished)
  3208. return await asyncio.shield(task)
  3209. async def _await_terminal_info(self, state: _RuntimeState) -> None:
  3210. """Apply the grace timeout only after a terminal message.complete."""
  3211. terminal_wait = asyncio.create_task(state.terminal_info_received.wait())
  3212. finished_wait = asyncio.create_task(state.finished.wait())
  3213. try:
  3214. done, pending = await asyncio.wait(
  3215. {terminal_wait, finished_wait},
  3216. timeout=self._completion_timeout,
  3217. return_when=asyncio.FIRST_COMPLETED,
  3218. )
  3219. for waiter in pending:
  3220. waiter.cancel()
  3221. if terminal_wait in done or finished_wait in done:
  3222. return
  3223. await self._finalize_state(
  3224. state,
  3225. outcome=(
  3226. state.terminal_status
  3227. if state.terminal_status in {"failed", "interrupted"}
  3228. else "interrupted"
  3229. ),
  3230. error=state.terminal_error or "Final session state timed out",
  3231. interrupt=True,
  3232. )
  3233. finally:
  3234. for waiter in (terminal_wait, finished_wait):
  3235. if not waiter.done():
  3236. waiter.cancel()
  3237. async def _finalize_state(
  3238. self,
  3239. state: _RuntimeState,
  3240. *,
  3241. outcome: str,
  3242. error: str,
  3243. interrupt: bool,
  3244. stored_session_id: str | None = None,
  3245. ) -> None:
  3246. async with state.mutation_lock:
  3247. await self._finalize_state_locked(
  3248. state,
  3249. outcome=outcome,
  3250. error=error,
  3251. interrupt=interrupt,
  3252. stored_session_id=stored_session_id,
  3253. )
  3254. async def _finalize_state_locked(
  3255. self,
  3256. state: _RuntimeState,
  3257. *,
  3258. outcome: str,
  3259. error: str,
  3260. interrupt: bool,
  3261. stored_session_id: str | None = None,
  3262. ) -> None:
  3263. async with state.finalize_lock:
  3264. if state.finished.is_set():
  3265. return
  3266. state.finalizing = True
  3267. current_task = asyncio.current_task()
  3268. pending_calls = tuple(state.inflight_calls)
  3269. if pending_calls:
  3270. await asyncio.gather(*pending_calls, return_exceptions=True)
  3271. gateway = self._required_gateway()
  3272. if interrupt:
  3273. try:
  3274. await self._run_state_blocking(
  3275. state, gateway.interrupt, state.runtime_id
  3276. )
  3277. except Exception: # noqa: BLE001 - settle remains mandatory
  3278. logger.exception("Failed to interrupt YaoYao group runtime")
  3279. expected_stored = (
  3280. state.session_stored_id
  3281. if state.prompt_committed
  3282. else state.expected_stored_id
  3283. )
  3284. rotated_stored = stored_session_id or state.session_stored_id
  3285. settled = False
  3286. for delay in (*_STORE_RETRY_DELAYS, None):
  3287. try:
  3288. await self._run_state_blocking(
  3289. state,
  3290. self.store.settle_run,
  3291. state.run_id,
  3292. runtime_session_id=state.runtime_id,
  3293. expected_stored_session_id=expected_stored,
  3294. stored_session_id=rotated_stored,
  3295. outcome=outcome,
  3296. actual_model=state.actual_model,
  3297. actual_provider=state.actual_provider,
  3298. actual_reasoning_effort=state.actual_reasoning_effort,
  3299. actual_fast_mode=state.actual_fast_mode,
  3300. error=self._bounded_error(error),
  3301. )
  3302. settled = True
  3303. break
  3304. except Exception: # noqa: BLE001 - reconcile before retrying
  3305. if await self._run_is_durably_terminal(state.run_id):
  3306. settled = True
  3307. break
  3308. if delay is None:
  3309. logger.exception("Failed to settle YaoYao group runtime")
  3310. break
  3311. await asyncio.sleep(delay)
  3312. if not settled:
  3313. state.settle_error = GroupOrchestratorError(
  3314. "Durable runtime settlement failed"
  3315. )
  3316. return
  3317. state.settle_error = None
  3318. self._remove_runtime(state)
  3319. try:
  3320. await self._run_state_blocking(state, gateway.close, state.runtime_id)
  3321. except Exception: # noqa: BLE001 - durable state is already terminal
  3322. logger.exception("Failed to close settled YaoYao group runtime")
  3323. if outcome == "completed":
  3324. try:
  3325. cascade = await self._run_state_blocking(
  3326. state, self.store.complete_cascade, state.run_id
  3327. )
  3328. except Exception: # noqa: BLE001 - source is already durably complete
  3329. logger.error("Pending YaoYao group cascade remains durable")
  3330. self.wake()
  3331. else:
  3332. if (
  3333. isinstance(cascade, Mapping)
  3334. and isinstance(cascade.get("runCount"), int)
  3335. and not isinstance(cascade.get("runCount"), bool)
  3336. and cascade["runCount"] > 0
  3337. ):
  3338. self.wake()
  3339. state.finished.set()
  3340. state.terminal_info_received.set()
  3341. if state.event_task is not None and state.event_task is not current_task:
  3342. if not state.event_task.done():
  3343. state.event_task.cancel()
  3344. await asyncio.gather(state.event_task, return_exceptions=True)
  3345. if state.grace_task is not None and state.grace_task is not current_task:
  3346. if not state.grace_task.done():
  3347. state.grace_task.cancel()
  3348. await asyncio.gather(state.grace_task, return_exceptions=True)
  3349. def _required_gateway(self) -> _Gateway:
  3350. if self._gateway is None:
  3351. raise GroupOrchestratorError("Gateway adapter is not started")
  3352. return self._gateway
  3353. @staticmethod
  3354. def _bounded_error(value: str) -> str:
  3355. encoded = value.encode("utf-8", errors="replace")
  3356. if len(encoded) <= _ERROR_TEXT_LIMIT:
  3357. return value
  3358. shortened = encoded[: _ERROR_TEXT_LIMIT - 3]
  3359. while shortened:
  3360. try:
  3361. return shortened.decode("utf-8") + "..."
  3362. except UnicodeDecodeError:
  3363. shortened = shortened[:-1]
  3364. return "..."
  3365. __all__ = [
  3366. "DEFAULT_COMPLETION_TIMEOUT",
  3367. "DEFAULT_EVENT_QUEUE_SIZE",
  3368. "GroupOrchestrator",
  3369. "GroupOrchestratorError",
  3370. "build_run_prompt",
  3371. ]