streaming.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370
  1. """WeCom native streaming mixin (``msgtype: stream`` via aibot_respond_msg): per-turn state, per-req_id
  2. ack tracking (official replyStreamNonBlocking semantics), keep-alive heartbeat, finalize clock fallback."""
  3. from __future__ import annotations
  4. import asyncio
  5. import logging
  6. import time
  7. import uuid
  8. from dataclasses import dataclass
  9. from typing import Any, Dict, Optional
  10. logger = logging.getLogger("plugins.platforms.wecom.adapter")
  11. APP_CMD_RESPONSE = "aibot_respond_msg"
  12. # Each reply stream lives ~6 minutes (the connection ping does NOT refresh it); afterwards
  13. # 846608 (stream window) / 846604 (req_id window) mean the reply flow is dead. 846609 = ws
  14. # lost its subscription. 6000 = finalize raced a newer frame (bubble already replaced: benign).
  15. STREAM_EXPIRED_ERRCODE = 846608
  16. STREAM_REQUEST_EXPIRED_ERRCODE = 846604
  17. STREAM_NOT_SUBSCRIBED_ERRCODE = 846609
  18. STREAM_VERSION_CONFLICT_ERRCODE = 6000
  19. MAX_STREAM_CONTENT_LENGTH = 20480 # WeCom server-enforced byte limit per frame
  20. # SDK queue is 100 frames per reqId; cap intermediates (openclaw uses 85) so finalize has room.
  21. MAX_INTERMEDIATE_FRAMES = 85
  22. # Two defences against the 6-min window (docs/wecom-stream-keepalive-*.md): Layer 2 clock
  23. # fallback (always on) declines finish=true past STREAM_SAFE_DURATION_SECONDS so the consumer's
  24. # send() delivers; Layer 1 keep-alive (OFF by default) re-sends accumulated text as finish=false
  25. # every interval — off because an extra frame widens the ack race double-send relies on.
  26. STREAM_SAFE_DURATION_SECONDS = 330.0
  27. STREAM_KEEPALIVE_INTERVAL_SECONDS = 120.0
  28. STREAM_KEEPALIVE_ENABLED_DEFAULT = False
  29. class WeComStreamExpiredError(RuntimeError):
  30. """Raised on errcode 846608/846604: the stream/req_id reply flow is dead; fall back to ``aibot_send_msg``."""
  31. def __init__(self, errcode: int = STREAM_EXPIRED_ERRCODE, errmsg: str = ""):
  32. super().__init__(f"WeCom stream expired (errcode={errcode}): {errmsg or 'no detail'}")
  33. self.errcode, self.errmsg = errcode, errmsg
  34. @dataclass
  35. class ReplyFrame:
  36. """A reply frame awaiting its aibot_respond_msg ack (FIFO per req_id)."""
  37. body: dict[str, Any]
  38. future: asyncio.Future
  39. is_final: bool = False
  40. sent_at: Optional[float] = None
  41. class ReplyQueue:
  42. """Per-req_id pending-ack tracker: intermediates skip while an ack is pending, finals wait."""
  43. def __init__(self, req_id: str):
  44. self.req_id, self.pending_ack = req_id, None # pending_ack: Optional[ReplyFrame]
  45. class StreamTurn:
  46. """Per-turn stream state so concurrent messages never share a stream."""
  47. def __init__(self, chat_id: str, req_id: str):
  48. self.chat_id, self.req_id, self.stream_id = chat_id, req_id, f"stream_{uuid.uuid4().hex[:12]}"
  49. self.accumulated_text = ""
  50. self.finalized = self.seeded = self.expired = False # seeded prevents a double seed (errcode 6000)
  51. self.start_time = time.monotonic()
  52. self.last_sent_content: str = "" # content ACTUALLY sent; final frame must differ or WeCom drops it
  53. self._intermediate_frames_sent: int = 0
  54. self.keepalive_handle: Optional[asyncio.TimerHandle] = None # cancel on EVERY turn-exit path
  55. def _stream_of(body: dict[str, Any]) -> dict[str, Any]:
  56. return body.get("stream", {}) if isinstance(body.get("stream"), dict) else {}
  57. def _stream_desc(body: dict[str, Any]) -> tuple:
  58. stream = _stream_of(body)
  59. return stream.get("id", "N/A"), stream.get("finish", "N/A")
  60. def _elapsed(since: Optional[float]) -> float:
  61. return time.monotonic() - (since or time.monotonic())
  62. class WeComStreamMixin:
  63. """Native streaming mixed into WeComAdapter (uses its ws transport, registries and ``_stream_*`` config)."""
  64. MAX_STREAM_CONTENT_LENGTH = MAX_STREAM_CONTENT_LENGTH
  65. _REPLY_ACK_TIMEOUT = 15.0 # official REPLY_SEND_TIMEOUT_MS; shorter widened the double-send race
  66. async def _send_reply_queued(self, reply_req_id: str, body: dict[str, Any], *, is_final: bool = False, skip_if_pending: bool = False) -> dict[str, Any]:
  67. """aibot_respond_msg with per-req_id ack tracking: is_final drains the pending ack then awaits its own;
  68. skip_if_pending returns ``{"skipped": True}`` while a prior ack is pending."""
  69. self._require_ws()
  70. normalized = self._require_reply_req_id(reply_req_id)
  71. queue = self._reply_queues.setdefault(normalized, ReplyQueue(normalized))
  72. if skip_if_pending and queue.pending_ack is not None:
  73. return {"skipped": True, "errcode": 0, "errmsg": "pending_ack"}
  74. if is_final and queue.pending_ack is not None:
  75. await self._drain_pending_ack(queue, normalized)
  76. future: asyncio.Future = asyncio.get_running_loop().create_future()
  77. frame = ReplyFrame(body=body, future=future, is_final=is_final, sent_at=time.monotonic())
  78. # Register BEFORE sending so a mid-send ack routes; re-attach `queue` because the drain
  79. # above may have let the intermediate ack pop it out of _reply_queues (orphan → timeout).
  80. self._reply_queues[normalized] = queue
  81. queue.pending_ack = frame
  82. logger.debug(
  83. "[%s] _send_reply_queued: req_id=%s is_final=%s skip_if_pending=%s stream_id=%s finish=%s content_len=%d", self.name, normalized, is_final, skip_if_pending, *_stream_desc(body), len(_stream_of(body).get("content", "") or ""),
  84. )
  85. try:
  86. await self._send_json({"cmd": APP_CMD_RESPONSE, "headers": {"req_id": normalized}, "body": body})
  87. except Exception:
  88. # Nobody awaits the future here — cancel it rather than log "exception never retrieved".
  89. self._release_pending(queue, normalized, frame)
  90. if not future.done():
  91. future.cancel()
  92. raise
  93. if not is_final: # fire-and-forget; pending_ack stays registered so later frames can skip
  94. return {"errcode": 0, "errmsg": "sent_nonblocking"}
  95. try:
  96. return await asyncio.wait_for(future, timeout=self._REPLY_ACK_TIMEOUT)
  97. except TimeoutError:
  98. # Bytes went out, ack is late — WeCom already rendered it; raising caused duplicates.
  99. logger.warning("[%s] Final frame ack timeout (req_id=%s) — treating as delivered (matches official wecom-openclaw-plugin behaviour). No fallback send.", self.name, normalized)
  100. return {"errcode": 0, "errmsg": "ack_timeout_assumed_delivered", "ack_pending": True}
  101. finally:
  102. self._release_pending(queue, normalized, frame)
  103. async def _drain_pending_ack(self, queue: ReplyQueue, req_id: str) -> None:
  104. """Before a final frame: wait (bounded) for the pending intermediate's ack, then clear it."""
  105. pending_frame = queue.pending_ack
  106. pending_desc = (self.name, req_id, *_stream_desc(pending_frame.body))
  107. logger.debug("[%s] _send_reply_queued: final waiting for pending ack drain — req_id=%s pending_stream_id=%s pending_finish=%s pending_sent_at=%.1fs_ago", *pending_desc, _elapsed(pending_frame.sent_at))
  108. try:
  109. await asyncio.wait_for(asyncio.shield(pending_frame.future), timeout=self._REPLY_ACK_TIMEOUT)
  110. except TimeoutError:
  111. logger.warning(
  112. "[%s] Reply ack timeout waiting for pending (req_id=%s) — pending_stream_id=%s pending_finish=%s elapsed=%.1fs. Possible causes: ack cmd filtered, ack req_id mismatch, or WeCom did not ack.",
  113. *pending_desc, _elapsed(pending_frame.sent_at),
  114. )
  115. except Exception:
  116. pass
  117. queue.pending_ack = None # resolved or timed out either way
  118. def _release_pending(self, queue: ReplyQueue, req_id: str, frame: ReplyFrame) -> None:
  119. """Clear ``frame`` if it is still the pending ack; drop the queue once empty."""
  120. if queue.pending_ack is frame:
  121. queue.pending_ack = None
  122. if queue.pending_ack is None:
  123. self._reply_queues.pop(req_id, None)
  124. def _resolve_reply_ack(self, req_id: str, payload: dict[str, Any]) -> bool:
  125. """Resolve a pending reply ack. Returns True if handled."""
  126. queue = self._reply_queues.get(req_id)
  127. if queue is None or queue.pending_ack is None:
  128. return False
  129. frame = queue.pending_ack
  130. if not frame.future.done():
  131. _body = payload.get("body", {}) if isinstance(payload.get("body"), dict) else {}
  132. logger.debug("[%s] _resolve_reply_ack: resolved req_id=%s is_final=%s elapsed=%.2fs errcode=%s", self.name, req_id, frame.is_final, _elapsed(frame.sent_at), _body.get("errcode", "N/A"))
  133. frame.future.set_result(payload)
  134. self._release_pending(queue, req_id, frame)
  135. return True
  136. def _fail_reply_queues(self, error: Exception) -> None:
  137. for queue in list(self._reply_queues.values()):
  138. if queue.pending_ack and not queue.pending_ack.future.done():
  139. queue.pending_ack.future.set_exception(error)
  140. self._reply_queues.clear()
  141. def _resolve_stream_req_id(self, chat_id: str, reply_to: Optional[str]) -> Optional[str]:
  142. """Explicit ``reply_to`` (cached message id) → last inbound req_id for the chat → None."""
  143. return self._reply_req_id_for_message(reply_to) or self._last_chat_req_ids.get(str(chat_id or "").strip()) or None
  144. @staticmethod
  145. def _cancel_keepalive(turn: StreamTurn) -> None:
  146. handle, turn.keepalive_handle = turn.keepalive_handle, None
  147. if handle is not None:
  148. try:
  149. handle.cancel()
  150. except Exception:
  151. pass
  152. def _retire_turn(self, turn: StreamTurn, turn_id: Optional[str]) -> None:
  153. """Single choke point for "turn is dead": cancel the timer, then drop it from the registry."""
  154. self._cancel_keepalive(turn)
  155. self._stream_turns.pop(f"{turn.chat_id}:{turn_id or turn.req_id}", None)
  156. def _expire_turn(self, turn: StreamTurn, turn_id: Optional[str]) -> None:
  157. turn.expired = True
  158. self._retire_turn(turn, turn_id)
  159. self._stream_expired_chats.add(turn.chat_id)
  160. def _find_active_turn_for_chat(self, chat_id: str) -> Optional[StreamTurn]:
  161. return next((t for t in self._stream_turns.values() if t.chat_id == chat_id and not t.finalized), None)
  162. def _arm_keepalive(self, turn: StreamTurn, *, turn_id: Optional[str]) -> None:
  163. """Arm the keep-alive timer if enabled and not already armed (idempotent)."""
  164. if not self._stream_keepalive_enabled or turn.finalized or turn.expired or turn.keepalive_handle is not None:
  165. return
  166. try:
  167. turn.keepalive_handle = asyncio.get_running_loop().call_later(self._stream_keepalive_interval_seconds, self._on_keepalive_fire, turn, turn_id)
  168. except RuntimeError:
  169. pass
  170. def _on_keepalive_fire(self, turn: StreamTurn, turn_id: Optional[str]) -> None:
  171. turn.keepalive_handle = None
  172. if not (turn.finalized or turn.expired):
  173. try:
  174. asyncio.ensure_future(self._keepalive_send(turn, turn_id))
  175. except RuntimeError:
  176. pass
  177. async def _keepalive_send(self, turn: StreamTurn, turn_id: Optional[str]) -> None:
  178. """Re-send accumulated text as finish=false to refresh the window, then re-arm. Never a placeholder
  179. (empty text skips); on 846604/846608 the turn is retired for Layer 2."""
  180. if turn.finalized or turn.expired or turn._intermediate_frames_sent >= MAX_INTERMEDIATE_FRAMES:
  181. return # cap reached: no room for intermediates; let finalize / Layer 2 run
  182. content = turn.accumulated_text or ""
  183. if not content.strip():
  184. self._arm_keepalive(turn, turn_id=turn_id)
  185. return
  186. try:
  187. await self._send_stream_reply(turn.req_id, turn.stream_id, content, finish=False)
  188. except WeComStreamExpiredError:
  189. self._expire_turn(turn, turn_id)
  190. return
  191. except Exception as exc:
  192. logger.debug("[%s] keep-alive send failed (chat=%s, turn=%s): %s", self.name, turn.chat_id, turn.stream_id, exc)
  193. self._arm_keepalive(turn, turn_id=turn_id) # transient — retry next interval
  194. return
  195. turn.last_sent_content = content
  196. self._arm_keepalive(turn, turn_id=turn_id)
  197. @staticmethod
  198. def _truncate_stream_content(content: str, limit: int) -> str:
  199. """Truncate to ``limit`` UTF-8 bytes (WeCom caps frames by bytes, not codepoints)."""
  200. encoded = content.encode("utf-8")
  201. return content if len(encoded) <= limit else encoded[:limit].decode("utf-8", errors="ignore")
  202. async def _send_stream_reply(self, reply_req_id: str, stream_id: str, content: str, finish: bool = False) -> dict[str, Any]:
  203. """Send one ``msgtype: "stream"`` frame: intermediates non-blocking/skip-if-pending, the final frame awaits
  204. its ack so 846608/6000 are detected. Raises WeComStreamExpiredError on expiry."""
  205. truncated = self._truncate_stream_content(content or "", self.MAX_STREAM_CONTENT_LENGTH)
  206. if len(content or "") != len(truncated):
  207. logger.warning("[%s] Stream content truncated for stream_id=%s", self.name, stream_id)
  208. body: dict[str, Any] = {"msgtype": "stream", "stream": {"id": stream_id, "finish": bool(finish), "content": truncated}}
  209. if not finish:
  210. return await self._send_reply_queued(reply_req_id, body, is_final=False, skip_if_pending=True)
  211. response = await self._send_reply_queued(reply_req_id, body, is_final=True, skip_if_pending=False)
  212. errcode = response.get("errcode", 0)
  213. if errcode in (STREAM_EXPIRED_ERRCODE, STREAM_REQUEST_EXPIRED_ERRCODE):
  214. raise WeComStreamExpiredError(errcode=errcode, errmsg=str(response.get("errmsg") or ""))
  215. if errcode == STREAM_VERSION_CONFLICT_ERRCODE:
  216. # Content is already on screen; raising would pop the turn and duplicate via send().
  217. logger.info("[%s] finalize hit errcode 6000 (version conflict) — bubble already replaced by a newer frame; treating as delivered.", self.name)
  218. return response
  219. self._raise_for_wecom_error(response, "send stream reply")
  220. return response
  221. async def send_stream_frame(self, text: str, *, finalize: bool = False, chat_id: Optional[str] = None, reply_to: Optional[str] = None, **kwargs) -> bool:
  222. """Gateway streaming entry point: first call seeds the turn, later calls push cumulative text, ``finalize=True``
  223. closes it; ``turn_id`` kwarg keys concurrent turns. Returns False when unavailable — caller falls back to send()."""
  224. chat = (chat_id or "").strip()
  225. if not chat:
  226. logger.warning("[%s] send_stream_frame: chat_id required", self.name)
  227. return False
  228. turn_id = kwargs.get("turn_id")
  229. # Chat-level expiry only blocks NEW turn creation; a known turn_id may still finalize.
  230. if not turn_id and chat in self._stream_expired_chats:
  231. return False
  232. inner = lambda: self._send_stream_frame_inner(text, chat=chat, reply_to=reply_to, finalize=finalize, turn_id=turn_id)
  233. # Finalize counts toward 30/min → control lane; intermediates are unmetered (no queue).
  234. return await self._enqueue_chat_send(chat, inner, is_control=True) if finalize else await inner()
  235. def _locate_turn(self, chat: str, reply_to: Optional[str], finalize: bool, turn_id: Optional[str]) -> Optional[StreamTurn]:
  236. """Find or create the StreamTurn (None = unavailable); a turn locks to its creation req_id."""
  237. if turn_id:
  238. turn = self._stream_turns.get(f"{chat}:{turn_id}")
  239. if turn:
  240. return turn
  241. if finalize: # never create a turn on finalize: caller must fall back, not seed+finish
  242. logger.debug("[%s] send_stream_frame: cannot finalize non-existent turn (turn_id=%s, chat=%s)", self.name, turn_id, chat)
  243. return None
  244. elif existing_turn := self._find_active_turn_for_chat(chat): # direct callers without turn_id reuse the chat's active (unfinalized) turn
  245. logger.debug("[%s] send_stream_frame: reusing existing turn %s for chat %s", self.name, existing_turn.stream_id, chat)
  246. return existing_turn
  247. suffix = f" (turn_id={turn_id})" if turn_id else ""
  248. req_id = None if chat in self._stream_expired_chats else self._resolve_stream_req_id(chat, reply_to)
  249. if not req_id:
  250. why = "chat %s is expired, cannot create new turn%s" if chat in self._stream_expired_chats else "no req_id available for chat %s%s"
  251. logger.debug("[%s] send_stream_frame: " + why, self.name, chat, suffix)
  252. return None
  253. key = f"{chat}:{turn_id or req_id}"
  254. turn = (None if turn_id else self._stream_turns.get(key)) or StreamTurn(chat, req_id)
  255. self._stream_turns[key] = turn
  256. logger.debug("[%s] send_stream_frame: created new turn %s (%s) for chat %s", self.name, turn.stream_id, f"turn_id={turn_id}, req_id={req_id}" if turn_id else f"req_id={req_id}", chat)
  257. return turn
  258. async def _finalize_turn(self, turn: StreamTurn, text: str, chat: str, turn_id: Optional[str]) -> bool:
  259. """Send the finish=true frame (or decline via the Layer 2 clock fallback)."""
  260. # Layer 2: an old stream would hit 846604/846608 on finish=true, so decline up front and let
  261. # send() deliver once. Skipped with Layer 1 on: the heartbeat refreshed the window.
  262. if not self._stream_keepalive_enabled:
  263. stream_age = time.monotonic() - turn.start_time
  264. if stream_age >= self._stream_safe_duration_seconds:
  265. logger.info(
  266. "[%s] Stream age %.0fs >= safe duration %.0fs for chat %s — declining finalize frame, falling back to proactive send (Layer 2 clock fallback).",
  267. self.name, stream_age, self._stream_safe_duration_seconds, chat,
  268. )
  269. self._expire_turn(turn, turn_id)
  270. return False
  271. self._cancel_keepalive(turn)
  272. # A final frame identical to the last intermediate is silently dropped — differ via ZWSP.
  273. final_text = text + "\u200b" if text and text == turn.last_sent_content else text
  274. await self._send_stream_reply(turn.req_id, turn.stream_id, final_text, finish=True)
  275. turn.finalized = True
  276. self._stream_turns.pop(f"{chat}:{turn_id or turn.req_id}", None)
  277. return True
  278. async def _send_stream_frame_inner(self, text: str, *, chat: str, reply_to: Optional[str] = None, finalize: bool = False, turn_id: Optional[str] = None) -> bool:
  279. turn: Optional[StreamTurn] = None
  280. try:
  281. turn = self._locate_turn(chat, reply_to, finalize, turn_id)
  282. if turn is None or turn.expired:
  283. return False
  284. if not turn.seeded and not turn.finalized:
  285. # Official THINKING_MESSAGE seed; `seeded` prevents a double seed (6000).
  286. await self._send_stream_reply(turn.req_id, turn.stream_id, "<think></think>", finish=False)
  287. turn.seeded = True
  288. self._arm_keepalive(turn, turn_id=turn_id)
  289. if not text and not finalize:
  290. return True # consumer's explicit seed call — nothing more to send
  291. if finalize:
  292. return await self._finalize_turn(turn, text, chat, turn_id)
  293. # Fire-and-forget: the gateway decides when to push (identity dedup in stream_consumer.py).
  294. turn.accumulated_text = text
  295. if turn._intermediate_frames_sent >= MAX_INTERMEDIATE_FRAMES or text == turn.last_sent_content:
  296. return True # cap reached (finalize drains the rest) or nothing new
  297. await self._send_stream_reply(turn.req_id, turn.stream_id, text, finish=False)
  298. turn._intermediate_frames_sent += 1
  299. turn.last_sent_content = text
  300. return True
  301. except WeComStreamExpiredError:
  302. # Intermediates are overwritten by the next frame anyway; expiring here would duplicate.
  303. if not finalize:
  304. logger.info("[%s] Intermediate stream frame expired (errcode=%d) for chat %s — dropping frame, stream stays live", self.name, STREAM_EXPIRED_ERRCODE, chat)
  305. return True
  306. logger.info("[%s] Stream expired (errcode=%d) for chat %s — switching to proactive send", self.name, STREAM_EXPIRED_ERRCODE, chat)
  307. if turn is None:
  308. self._stream_expired_chats.add(chat)
  309. else:
  310. self._expire_turn(turn, turn_id)
  311. except Exception as exc:
  312. if not finalize: # same intermediate/final split as above
  313. logger.info("[%s] Intermediate stream frame failed (chat=%s): %s — dropping frame, stream stays live", self.name, chat, exc)
  314. return True
  315. logger.warning("[%s] Stream frame failed (chat=%s): %s", self.name, chat, exc)
  316. if turn is not None:
  317. self._retire_turn(turn, turn_id)
  318. return False
  319. def supports_native_streaming(self, chat_type: Optional[str] = None, metadata: Optional[dict[str, Any]] = None) -> bool:
  320. """Stream frames work in DMs and groups alike (groups just need a cached inbound req_id)."""
  321. del chat_type, metadata
  322. return True
  323. async def send_typing(self, chat_id: str, metadata=None) -> None:
  324. """No-op: the stream consumer's seed frame triggers WeCom typing; repeated calls would open orphan streams."""
  325. del chat_id, metadata