media.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358
  1. """WeCom media mixin: inbound attachment caching; outbound chunked ``aibot_upload_media_*`` upload then
  2. native send (image/video/voice/file)."""
  3. from __future__ import annotations
  4. import asyncio
  5. import base64
  6. import hashlib
  7. import logging
  8. import mimetypes
  9. import re
  10. import uuid
  11. from pathlib import Path
  12. from typing import Any, Dict, List, Optional, Tuple
  13. from urllib.parse import unquote, urlparse
  14. from agent.i18n import t
  15. from gateway.platforms.base import SendResult, cache_document_from_bytes_async, cache_image_from_bytes_async
  16. logger = logging.getLogger("plugins.platforms.wecom.adapter")
  17. APP_CMD_SEND = "aibot_send_msg"
  18. APP_CMD_UPLOAD_MEDIA_INIT = "aibot_upload_media_init"
  19. APP_CMD_UPLOAD_MEDIA_CHUNK = "aibot_upload_media_chunk"
  20. APP_CMD_UPLOAD_MEDIA_FINISH = "aibot_upload_media_finish"
  21. IMAGE_MAX_BYTES = 10 * 1024 * 1024
  22. VIDEO_MAX_BYTES = 10 * 1024 * 1024
  23. VOICE_MAX_BYTES = 2 * 1024 * 1024
  24. FILE_MAX_BYTES = 20 * 1024 * 1024
  25. ABSOLUTE_MAX_BYTES = FILE_MAX_BYTES
  26. UPLOAD_CHUNK_SIZE = 512 * 1024
  27. MAX_UPLOAD_CHUNKS = 100
  28. VOICE_SUPPORTED_MIMES = {"audio/amr"}
  29. _IMAGE_MAGIC = ((b"\x89PNG\r\n\x1a\n", ".png"), (b"\xff\xd8\xff", ".jpg"), ((b"GIF87a", b"GIF89a"), ".gif"))
  30. _MIME_PREFIX_KINDS = (("image/", "image"), ("video/", "video"), ("audio/", "voice"))
  31. # type -> (max bytes, catalog key of the kind label, human cap) for the "downgrade to file" notice;
  32. # the label resolves through ``t()`` at verdict time. Notices shipped in Chinese before i18n
  33. # (zh.yaml keeps that text).
  34. _TYPE_LIMITS = {"image": (IMAGE_MAX_BYTES, "platform.wecom.media.kind_image", "10MB"),
  35. "video": (VIDEO_MAX_BYTES, "platform.wecom.media.kind_video", "10MB"),
  36. "voice": (VOICE_MAX_BYTES, "platform.wecom.media.kind_voice", "2MB")}
  37. def _size_verdict(final_type: str, *, reject: Optional[str] = None, downgrade: Optional[str] = None) -> dict[str, Any]:
  38. return {"final_type": final_type, "rejected": reject is not None, "reject_reason": reject, "downgraded": downgrade is not None, "downgrade_note": downgrade}
  39. def _dict_at(container: dict[str, Any], key: str) -> dict[str, Any]:
  40. return container.get(key) if isinstance(container.get(key), dict) else {}
  41. def _media_body(media_type: str, media_id: str) -> dict[str, Any]:
  42. return {"msgtype": media_type, media_type: {"media_id": media_id}}
  43. class WeComMediaMixin:
  44. """Media helpers mixed into WeComAdapter (uses its transport, req_id cache and stream registry)."""
  45. async def _extract_media(self, body: dict[str, Any]) -> tuple[list[str], list[str]]:
  46. refs: list[tuple[str, dict[str, Any]]] = []
  47. msgtype = str(body.get("msgtype") or "").lower()
  48. def _ref(kind: str, container: dict[str, Any]) -> bool:
  49. found = isinstance(container.get(kind), dict)
  50. if found:
  51. refs.append((kind, container[kind]))
  52. return found
  53. if msgtype == "mixed":
  54. mixed = _dict_at(body, "mixed")
  55. for item in mixed.get("msg_item") if isinstance(mixed.get("msg_item"), list) else []:
  56. if isinstance(item, dict) and str(item.get("msgtype") or "").lower() == "image":
  57. _ref("image", item)
  58. else:
  59. _ref("image", body)
  60. if msgtype == "file":
  61. _ref("file", body)
  62. if msgtype == "appmsg" and isinstance(body.get("appmsg"), dict): # AI Bot attachments (PDF/Word/Excel)
  63. _ref("file", body["appmsg"]) or _ref("image", body["appmsg"])
  64. quote = _dict_at(body, "quote")
  65. quote_type = str(quote.get("msgtype") or "").lower()
  66. if quote_type in ("image", "file"):
  67. _ref(quote_type, quote)
  68. cached = [c for c in [await self._cache_media(kind, ref) for kind, ref in refs] if c]
  69. return [c[0] for c in cached], [c[1] for c in cached]
  70. async def _cache_media(self, kind: str, media: dict[str, Any]) -> Optional[tuple[str, str]]:
  71. """Cache an inbound image/file reference (inline base64 or URL) to local storage."""
  72. if media.get("base64"):
  73. try:
  74. raw = self._decode_base64(media["base64"])
  75. except Exception as exc:
  76. logger.debug("[%s] Failed to decode %s base64 media: %s", self.name, kind, exc)
  77. return None
  78. filename = str(media.get("filename") or media.get("name") or "wecom_file")
  79. return await self._store_media(kind, raw, self._detect_image_ext(raw), "", filename, mimetypes.guess_type(filename)[0] or "application/octet-stream", "")
  80. url = str(media.get("url") or "").strip()
  81. if not url:
  82. return None
  83. # The key arrives URL-encoded in some payloads (%2F, %3D); base64-decode the real value.
  84. aes_key = unquote(str(media.get("aeskey") or "").strip())
  85. try:
  86. step = "download"
  87. raw, headers = await self._download_remote_bytes(url, max_bytes=ABSOLUTE_MAX_BYTES)
  88. step = "decrypt"
  89. raw = self._decrypt_file_bytes(raw, aes_key) if aes_key else raw
  90. except Exception as exc:
  91. logger.debug("[%s] Failed to %s %s from %s: %s", self.name, step, kind, url, exc)
  92. return None
  93. content_type = str(headers.get("content-type") or "").split(";", 1)[0].strip() or "application/octet-stream"
  94. ext = self._guess_extension(url, content_type, fallback=self._detect_image_ext(raw))
  95. # Images: never forward the CDN's generic octet-stream label — downstream classifiers
  96. # reject non-image MIMEs; derive it from the resolved extension instead.
  97. image_mime = content_type if content_type.startswith("image/") else ""
  98. return await self._store_media(kind, raw, ext, image_mime, self._guess_filename(url, headers.get("content-disposition"), content_type), content_type, f" from {url}")
  99. async def _store_media(self, kind, raw, ext, image_mime, filename, doc_mime, origin) -> Optional[tuple[str, str]]:
  100. """Cache bytes as an image (``kind == "image"``) or a document; returns (path, mime)."""
  101. if kind != "image":
  102. return await cache_document_from_bytes_async(raw, filename), doc_mime
  103. try:
  104. return await cache_image_from_bytes_async(raw, ext), image_mime or self._mime_for_ext(ext, fallback="image/jpeg")
  105. except ValueError as exc:
  106. logger.warning("[%s] Rejected non-image bytes%s: %s", self.name, origin, exc)
  107. return None
  108. @staticmethod
  109. def _decode_base64(data: str) -> bytes:
  110. return base64.b64decode(data.split(",", 1)[-1].strip())
  111. @staticmethod
  112. def _detect_image_ext(data: bytes) -> str:
  113. webp = ".webp" if data.startswith(b"RIFF") and data[8:12] == b"WEBP" else ".jpg"
  114. return next((ext for magic, ext in _IMAGE_MAGIC if data.startswith(magic)), webp)
  115. @staticmethod
  116. def _mime_for_ext(ext: str, fallback: str = "application/octet-stream") -> str:
  117. return mimetypes.types_map.get(ext.lower(), fallback)
  118. @staticmethod
  119. def _guess_extension(url: str, content_type: str, fallback: str) -> str:
  120. # WeCom's CDN labels images application/octet-stream; mimetypes maps that to ".bin",
  121. # which is truthy and used to win over the magic-byte fallback (#10085).
  122. ext = mimetypes.guess_extension(content_type) if content_type else None
  123. if ext == ".bin":
  124. ext = None
  125. return ext or Path(urlparse(url).path).suffix or fallback
  126. @staticmethod
  127. def _guess_filename(url: str, content_disposition: Optional[str], content_type: str) -> str:
  128. match = re.search(r'filename="?([^";]+)"?', content_disposition or "")
  129. if match:
  130. return match.group(1)
  131. name = Path(urlparse(url).path).name or "document"
  132. return name if "." in name else f"{name}{mimetypes.guess_extension(content_type) or '.bin'}"
  133. @staticmethod
  134. def _decrypt_file_bytes(encrypted_data: bytes, aes_key: str) -> bytes:
  135. if not encrypted_data:
  136. raise ValueError("encrypted_data is empty")
  137. if not aes_key:
  138. raise ValueError("aes_key is required")
  139. key = base64.b64decode(aes_key + '=' * ((4 - len(aes_key) % 4) % 4)) # WeCom doesn't pad base64 keys
  140. if len(key) != 32:
  141. raise ValueError(f"Invalid WeCom AES key length: expected 32 bytes, got {len(key)}")
  142. try:
  143. from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
  144. except ImportError as exc: # pragma: no cover - dependency is environment-specific
  145. raise RuntimeError("cryptography is required for WeCom media decryption") from exc
  146. decryptor = Cipher(algorithms.AES(key), modes.CBC(key[:16])).decryptor()
  147. decrypted = decryptor.update(encrypted_data) + decryptor.finalize()
  148. pad_len = decrypted[-1]
  149. if pad_len < 1 or pad_len > 32 or pad_len > len(decrypted):
  150. raise ValueError(f"Invalid PKCS#7 padding value: {pad_len}")
  151. if any(byte != pad_len for byte in decrypted[-pad_len:]):
  152. raise ValueError("Invalid PKCS#7 padding: padding bytes mismatch")
  153. return decrypted[:-pad_len]
  154. async def _download_remote_bytes(self, url: str, max_bytes: int) -> tuple[bytes, dict[str, str]]:
  155. from gateway.platforms.base import ssrf_redirect_guard
  156. from tools.url_safety import create_ssrf_safe_async_client, is_safe_url
  157. from plugins.platforms.wecom import adapter as _adapter_mod
  158. if not is_safe_url(url):
  159. raise ValueError(f"Blocked unsafe URL (SSRF protection): {url[:80]}")
  160. if not _adapter_mod.HTTPX_AVAILABLE:
  161. raise RuntimeError("httpx is required for WeCom media download")
  162. client = self._http_client or create_ssrf_safe_async_client(timeout=30.0, follow_redirects=True, event_hooks={"response": [ssrf_redirect_guard]})
  163. try:
  164. async with client.stream("GET", url, headers={"User-Agent": "HermesAgent/1.0", "Accept": "*/*"}) as response:
  165. response.raise_for_status()
  166. headers = {key.lower(): value for key, value in response.headers.items()}
  167. content_length = headers.get("content-length")
  168. if content_length and content_length.isdigit() and int(content_length) > max_bytes:
  169. raise ValueError(f"Remote media exceeds WeCom limit: {int(content_length)} bytes > {max_bytes} bytes")
  170. data = bytearray()
  171. async for chunk in response.aiter_bytes():
  172. data.extend(chunk)
  173. if len(data) > max_bytes:
  174. raise ValueError(f"Remote media exceeds WeCom limit while downloading: {len(data)} bytes > {max_bytes} bytes")
  175. return bytes(data), headers
  176. finally:
  177. if client is not self._http_client:
  178. await client.aclose()
  179. @staticmethod
  180. def _guess_mime_type(filename: str) -> str:
  181. return mimetypes.guess_type(filename)[0] or ("audio/amr" if Path(filename).suffix.lower() == ".amr" else "application/octet-stream")
  182. @staticmethod
  183. def _normalize_content_type(content_type: str, filename: str) -> str:
  184. normalized = str(content_type or "").split(";", 1)[0].strip().lower()
  185. return normalized if normalized and normalized not in {"application/octet-stream", "text/plain"} else WeComMediaMixin._guess_mime_type(filename)
  186. @staticmethod
  187. def _detect_wecom_media_type(content_type: str) -> str:
  188. mime_type = str(content_type or "").strip().lower()
  189. return "voice" if mime_type == "application/ogg" else next((kind for prefix, kind in _MIME_PREFIX_KINDS if mime_type.startswith(prefix)), "file")
  190. @staticmethod
  191. def _apply_file_size_limits(file_size: int, detected_type: str, content_type: Optional[str] = None) -> dict[str, Any]:
  192. file_size_mb = file_size / (1024 * 1024)
  193. normalized_type = str(detected_type or "file").lower()
  194. normalized_content_type = str(content_type or "").strip().lower()
  195. if file_size > ABSOLUTE_MAX_BYTES:
  196. return _size_verdict(normalized_type, reject=t("platform.wecom.media.file_too_large", size_mb=f"{file_size_mb:.2f}"))
  197. if normalized_type == "voice" and normalized_content_type and normalized_content_type not in VOICE_SUPPORTED_MIMES:
  198. return _size_verdict("file", downgrade=t("platform.wecom.media.voice_format_downgraded", content_type=normalized_content_type))
  199. max_bytes, label_key, cap = _TYPE_LIMITS.get(normalized_type, (None, "", ""))
  200. if max_bytes is not None and file_size > max_bytes:
  201. return _size_verdict("file", downgrade=t("platform.wecom.media.size_downgraded",
  202. label=t(label_key), size_mb=f"{file_size_mb:.2f}", cap=cap))
  203. return _size_verdict(normalized_type)
  204. @staticmethod
  205. def _looks_like_url(media_source: str) -> bool:
  206. return urlparse(str(media_source or "")).scheme in {"http", "https"}
  207. async def _load_outbound_media(self, media_source: str, file_name: Optional[str] = None) -> tuple[bytes, str, str]:
  208. source = str(media_source or "").strip()
  209. if not source:
  210. raise ValueError("media source is required")
  211. if re.fullmatch(r"<[^>\n]+>", source):
  212. raise ValueError(f"Media placeholder was not replaced with a real file path: {source}")
  213. parsed = urlparse(source)
  214. if parsed.scheme in {"http", "https"}:
  215. data, headers = await self._download_remote_bytes(source, max_bytes=ABSOLUTE_MAX_BYTES)
  216. resolved_name = file_name or self._guess_filename(source, headers.get("content-disposition"), headers.get("content-type", ""))
  217. return data, self._normalize_content_type(headers.get("content-type", ""), resolved_name), resolved_name
  218. local_path = Path(unquote(parsed.path) if parsed.scheme == "file" else source).expanduser()
  219. local_path = local_path if local_path.is_absolute() else (Path.cwd() / local_path).resolve()
  220. if not local_path.is_file():
  221. raise FileNotFoundError(f"Media file not found: {local_path}")
  222. resolved_name = file_name or local_path.name
  223. return local_path.read_bytes(), self._normalize_content_type("", resolved_name), resolved_name
  224. async def _prepare_outbound_media(self, media_source: str, file_name: Optional[str] = None) -> dict[str, Any]:
  225. data, content_type, resolved_name = await self._load_outbound_media(media_source, file_name=file_name)
  226. detected_type = self._detect_wecom_media_type(content_type)
  227. return {"data": data, "content_type": content_type, "file_name": resolved_name, "detected_type": detected_type, **self._apply_file_size_limits(len(data), detected_type, content_type)}
  228. async def _checked_request(self, cmd: str, body: dict[str, Any], operation: str) -> dict[str, Any]:
  229. self._raise_for_wecom_error(response := await self._send_request(cmd, body), operation)
  230. return response
  231. async def _upload_media_bytes(self, data: bytes, media_type: str, filename: str) -> dict[str, Any]:
  232. if not data:
  233. raise ValueError("Cannot upload empty media")
  234. total_size, total_chunks = len(data), (len(data) + UPLOAD_CHUNK_SIZE - 1) // UPLOAD_CHUNK_SIZE
  235. if total_chunks > MAX_UPLOAD_CHUNKS:
  236. raise ValueError(f"File too large: {total_chunks} chunks exceeds maximum of {MAX_UPLOAD_CHUNKS} chunks")
  237. init_payload = {"type": media_type, "filename": filename, "total_size": total_size, "total_chunks": total_chunks, "md5": hashlib.md5(data).hexdigest()}
  238. init_response = await self._checked_request(APP_CMD_UPLOAD_MEDIA_INIT, init_payload, "media upload init")
  239. upload_id = str(_dict_at(init_response, "body").get("upload_id") or "").strip()
  240. if not upload_id:
  241. raise RuntimeError(f"media upload init failed: missing upload_id in response {init_response}")
  242. for chunk_index, start in enumerate(range(0, total_size, UPLOAD_CHUNK_SIZE)): # official SDK uses 0-based chunk indexes
  243. chunk_b64 = base64.b64encode(data[start : start + UPLOAD_CHUNK_SIZE]).decode("ascii")
  244. await self._checked_request(APP_CMD_UPLOAD_MEDIA_CHUNK, {"upload_id": upload_id, "chunk_index": chunk_index, "base64_data": chunk_b64}, f"media upload chunk {chunk_index}")
  245. finish_response = await self._checked_request(APP_CMD_UPLOAD_MEDIA_FINISH, {"upload_id": upload_id}, "media upload finish")
  246. finish_body = _dict_at(finish_response, "body")
  247. media_id = str(finish_body.get("media_id") or "").strip()
  248. if not media_id:
  249. raise RuntimeError(f"media upload finish failed: missing media_id in response {finish_response}")
  250. return {"type": str(finish_body.get("type") or media_type), "media_id": media_id, "created_at": finish_body.get("created_at")}
  251. async def _send_media_message(self, chat_id: str, media_type: str, media_id: str) -> dict[str, Any]:
  252. return await self._checked_request(APP_CMD_SEND, {"chatid": chat_id, **_media_body(media_type, media_id)}, "send media message")
  253. async def _send_followup_markdown(self, chat_id: str, content: str, reply_to: Optional[str] = None) -> Optional[SendResult]:
  254. if not content:
  255. return None
  256. result = await self.send(chat_id=chat_id, content=content, reply_to=reply_to)
  257. if not result.success:
  258. logger.warning("[%s] Follow-up markdown send failed: %s", self.name, result.error)
  259. return result
  260. async def _send_media_source(self, chat_id: str, media_source: str, caption: Optional[str] = None, file_name: Optional[str] = None, reply_to: Optional[str] = None) -> SendResult:
  261. if not chat_id:
  262. return SendResult(success=False, error="chat_id is required")
  263. try:
  264. prepared = await self._prepare_outbound_media(media_source, file_name=file_name)
  265. except Exception as exc:
  266. if not isinstance(exc, FileNotFoundError):
  267. logger.error("[%s] Failed to prepare outbound media %s: %s", self.name, media_source, exc)
  268. return SendResult(success=False, error=str(exc))
  269. if prepared["rejected"]:
  270. text = self.warning_text(f"⚠️ {prepared['reject_reason']}", caption or "")
  271. if text:
  272. await self._send_followup_markdown(chat_id, text, reply_to=reply_to)
  273. return SendResult(success=False, error=prepared["reject_reason"])
  274. reply_req_id = self._cached_reply_req_id(chat_id, reply_to)
  275. # Active/expired stream owns the req_id (passive replyMedia is never acked): go proactive.
  276. if self._find_active_turn_for_chat(chat_id) or chat_id in self._stream_expired_chats:
  277. reply_req_id = None
  278. try:
  279. upload_result = await self._upload_media_bytes(prepared["data"], prepared["final_type"], prepared["file_name"])
  280. logger.info("[%s] upload_media_bytes OK: media_id=%s type=%s", self.name, upload_result.get("media_id"), prepared["final_type"])
  281. if reply_req_id: # passive reply when a req_id is usable, else proactive APP_CMD_SEND
  282. media_response = await self._send_reply_request(reply_req_id, _media_body(prepared["final_type"], upload_result["media_id"]))
  283. self._raise_for_wecom_error(media_response, "send reply media message")
  284. else:
  285. media_response = await self._send_media_message(chat_id, prepared["final_type"], upload_result["media_id"])
  286. logger.info("[%s] %s OK: %s", self.name, "send_reply_media" if reply_req_id else "send_media_message", media_response)
  287. except TimeoutError:
  288. logger.error("[%s] TIMEOUT in _send_media_source for %s", self.name, media_source)
  289. return SendResult(success=False, error="Timeout sending media to WeCom")
  290. except Exception as exc:
  291. logger.error("[%s] Failed to send media %s: %s", self.name, media_source, exc)
  292. return SendResult(success=False, error=str(exc))
  293. raw: dict[str, Any] = {"upload": upload_result, "media": media_response}
  294. downgrade = self.warning_text(f"ℹ️ {prepared['downgrade_note']}") if prepared["downgraded"] and prepared["downgrade_note"] else None
  295. for key, text in (("caption", caption), ("downgrade", downgrade or None)):
  296. followup = await self._send_followup_markdown(chat_id, text, reply_to=reply_to) if text else None
  297. raw[key] = followup.raw_response if followup else None
  298. raw[f"{key}_error"] = followup.error if followup and not followup.success else None
  299. return SendResult(success=True, message_id=self._payload_req_id(media_response) or uuid.uuid4().hex[:12], raw_response=raw)
  300. async def send_image(self, chat_id: str, image_url: str, caption: Optional[str] = None, reply_to: Optional[str] = None, metadata: Optional[dict[str, Any]] = None) -> SendResult:
  301. result = await self._send_media_source(chat_id=chat_id, media_source=image_url, caption=caption, reply_to=reply_to)
  302. if result.success or not self._looks_like_url(image_url):
  303. return result
  304. logger.warning("[%s] Falling back to text send for image URL %s: %s", self.name, image_url, result.error)
  305. return await self.send(chat_id=chat_id, content=f"{caption}\n{image_url}" if caption else image_url, reply_to=reply_to)
  306. async def send_image_file(self, chat_id: str, image_path: str, caption: Optional[str] = None, reply_to: Optional[str] = None, **kwargs) -> SendResult:
  307. return await self._send_media_source(chat_id=chat_id, media_source=image_path, caption=caption, reply_to=reply_to)
  308. async def send_document(self, chat_id: str, file_path: str, caption: Optional[str] = None, file_name: Optional[str] = None, reply_to: Optional[str] = None, **kwargs) -> SendResult:
  309. logger.info("[%s] send_document called: chat=%s file=%s", self.name, chat_id, file_path)
  310. return await self._send_media_source(chat_id=chat_id, media_source=file_path, caption=caption, file_name=file_name, reply_to=reply_to)
  311. async def send_voice(self, chat_id: str, audio_path: str, caption: Optional[str] = None, reply_to: Optional[str] = None, **kwargs) -> SendResult:
  312. return await self._send_media_source(chat_id=chat_id, media_source=audio_path, caption=caption, reply_to=reply_to)
  313. async def send_video(self, chat_id: str, video_path: str, caption: Optional[str] = None, reply_to: Optional[str] = None, **kwargs) -> SendResult:
  314. return await self._send_media_source(chat_id=chat_id, media_source=video_path, caption=caption, reply_to=reply_to)