chunked_upload.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292
  1. """QQ Bot chunked upload flow (files 10 MB..~100 MB; inline uploads cap at ~10 MB):
  2. 1. POST .../upload_prepare → upload_id, block_size, pre-signed COS part URLs;
  3. 2. per part: PUT bytes to COS, then POST .../upload_part_finish;
  4. 3. POST .../files with {"upload_id"} → ``file_info`` for a RichMedia message.
  5. biz_code 40093001 = part_finish retryable until ``retry_timeout``; 40093002 = daily
  6. quota (UploadDailyLimitExceededError); other API/I/O failures raise RuntimeError.
  7. Ported from WideLee's qqbot-agent-sdk v1.2.2 (authorship via Co-authored-by).
  8. """
  9. from __future__ import annotations
  10. import asyncio
  11. import hashlib
  12. import logging
  13. from dataclasses import dataclass
  14. from pathlib import Path
  15. from typing import Any, Awaitable, Callable, Dict, List
  16. from .constants import FILE_UPLOAD_TIMEOUT
  17. logger = logging.getLogger(__name__)
  18. _BIZ_CODE_DAILY_LIMIT = 40093002 # upload_prepare: daily cumulative limit
  19. _BIZ_CODE_PART_RETRYABLE = 40093001 # upload_part_finish: transient
  20. _DEFAULT_CONCURRENT_PARTS = 1
  21. _MAX_CONCURRENT_PARTS = 10
  22. _PART_UPLOAD_TIMEOUT = 300.0 # 5 minutes per COS PUT
  23. _PART_UPLOAD_MAX_RETRIES = 2
  24. _PART_FINISH_RETRY_INTERVAL = 1.0
  25. _PART_FINISH_DEFAULT_TIMEOUT = 120.0
  26. _PART_FINISH_MAX_TIMEOUT = 600.0
  27. _COMPLETE_UPLOAD_MAX_RETRIES = 2
  28. _COMPLETE_UPLOAD_BASE_DELAY = 2.0
  29. _MD5_10M_SIZE = 10_002_432 # first N bytes used for the ``md5_10m`` hash (per QQ API spec)
  30. class _UploadError(Exception):
  31. def __init__(self, file_name: str, file_size: int, message: str) -> None:
  32. self.file_name, self.file_size = file_name, file_size
  33. super().__init__(message)
  34. @property
  35. def file_size_human(self) -> str:
  36. return format_size(self.file_size)
  37. class UploadDailyLimitExceededError(_UploadError):
  38. """Raised when ``upload_prepare`` returns biz_code 40093002 (daily quota hit)."""
  39. def __init__(self, file_name: str, file_size: int, message: str = "") -> None:
  40. super().__init__(file_name, file_size, message or f"Daily upload limit exceeded for {file_name!r}")
  41. class UploadFileTooLargeError(_UploadError):
  42. """Raised when a file exceeds the platform per-file size limit."""
  43. def __init__(self, file_name: str, file_size: int, limit_bytes: int = 0, message: str = "") -> None:
  44. self.limit_bytes = limit_bytes
  45. limit_str = f" ({format_size(limit_bytes)})" if limit_bytes else ""
  46. super().__init__(file_name, file_size,
  47. message or f"File {file_name!r} ({format_size(file_size)}) exceeds platform limit{limit_str}")
  48. @property
  49. def limit_human(self) -> str:
  50. return format_size(self.limit_bytes) if self.limit_bytes else "unknown"
  51. @dataclass
  52. class _PreparePart:
  53. index: int
  54. presigned_url: str
  55. block_size: int = 0
  56. @dataclass
  57. class _PrepareResult:
  58. upload_id: str
  59. block_size: int
  60. parts: list[_PreparePart]
  61. concurrency: int = _DEFAULT_CONCURRENT_PARTS
  62. retry_timeout: float = 0.0
  63. def _parse_prepare_response(raw: dict[str, Any]) -> _PrepareResult:
  64. """Parse upload_prepare response (either bare or wrapped in ``data``)."""
  65. src = raw.get("data") if isinstance(raw.get("data"), dict) else raw
  66. if not (upload_id := str(src.get("upload_id", ""))):
  67. raise ValueError(f"upload_prepare response missing upload_id: {str(raw)[:200]}")
  68. block_size = int(src.get("block_size", 0))
  69. raw_parts = src.get("parts") or src.get("part_list") or []
  70. if not isinstance(raw_parts, list) or not raw_parts:
  71. raise ValueError(f"upload_prepare response missing parts: {str(raw)[:200]}")
  72. parts = [_PreparePart(index=int(p.get("part_index") or p.get("index") or 0),
  73. presigned_url=str(p.get("presigned_url") or p.get("url") or ""),
  74. block_size=int(p.get("block_size", 0)))
  75. for p in raw_parts if isinstance(p, dict)]
  76. return _PrepareResult(
  77. upload_id=upload_id, block_size=block_size, parts=parts,
  78. concurrency=int(src.get("concurrency", _DEFAULT_CONCURRENT_PARTS)) or _DEFAULT_CONCURRENT_PARTS,
  79. retry_timeout=float(src.get("retry_timeout", 0.0) or 0.0))
  80. def _api_path(chat_type: str, target_id: str, endpoint: str) -> str:
  81. return f"{'/v2/users' if chat_type == 'c2c' else '/v2/groups'}/{target_id}/{endpoint}"
  82. @dataclass
  83. class _Job:
  84. """Per-upload state shared by the part workers."""
  85. chat_type: str
  86. target_id: str
  87. file_path: str
  88. file_size: int
  89. upload_id: str = ""
  90. block_size: int = 0
  91. total_parts: int = 0
  92. retry_timeout: float = 0.0
  93. completed: int = 0
  94. def path(self, endpoint: str) -> str:
  95. return _api_path(self.chat_type, self.target_id, endpoint)
  96. class ChunkedUploader:
  97. """Run the prepare → PUT parts → complete sequence. ``api_request`` is the adapter's
  98. bound ``_api_request(method, path, body=..., timeout=...)`` (injected to avoid a
  99. circular import; must raise RuntimeError with the biz_code in the message);
  100. ``http_put`` is ``(url, data, headers) -> httpx-like response`` for COS PUTs."""
  101. def __init__(self, api_request: Callable[..., Awaitable[dict[str, Any]]],
  102. http_put: Callable[..., Awaitable[Any]], log_tag: str = "QQBot") -> None:
  103. self._api_request = api_request
  104. self._http_put = http_put
  105. self._log_tag = log_tag
  106. async def _post(self, job: _Job, endpoint: str, body: dict[str, Any]) -> dict[str, Any]:
  107. return await self._api_request("POST", job.path(endpoint), body=body, timeout=FILE_UPLOAD_TIMEOUT)
  108. async def upload(self, chat_type: str, target_id: str, file_path: str, file_type: int, file_name: str) -> dict[str, Any]:
  109. """Run the full chunked upload (``chat_type`` 'c2c'|'group', ``file_type`` MEDIA_TYPE_*)
  110. and return the raw ``complete_upload`` response (contains ``file_info``).
  111. Raises UploadDailyLimitExceededError (40093002), UploadFileTooLargeError, RuntimeError."""
  112. if chat_type not in {"c2c", "group"}:
  113. raise ValueError(f"ChunkedUploader: unsupported chat_type {chat_type!r}")
  114. job = _Job(chat_type, target_id, file_path, Path(file_path).stat().st_size)
  115. logger.info("[%s] Chunked upload start: file=%s size=%s type=%d", self._log_tag, file_name,
  116. format_size(job.file_size), file_type)
  117. # Hashing is blocking I/O → executor.
  118. hashes = await asyncio.get_running_loop().run_in_executor(None, _compute_file_hashes, file_path, job.file_size)
  119. prepare = await self._prepare(job, file_type, file_name, hashes)
  120. max_concurrent = min(prepare.concurrency, _MAX_CONCURRENT_PARTS)
  121. job.upload_id, job.block_size, job.total_parts = prepare.upload_id, prepare.block_size, len(prepare.parts)
  122. job.retry_timeout = min(prepare.retry_timeout if prepare.retry_timeout > 0 else _PART_FINISH_DEFAULT_TIMEOUT,
  123. _PART_FINISH_MAX_TIMEOUT)
  124. logger.info("[%s] Prepared: upload_id=%s block_size=%s parts=%d concurrency=%d", self._log_tag,
  125. job.upload_id, format_size(job.block_size), job.total_parts, max_concurrent)
  126. sem = asyncio.Semaphore(max(max_concurrent, 1))
  127. async def _run(part: _PreparePart) -> None:
  128. async with sem:
  129. await self._upload_one_part(job, part)
  130. await asyncio.gather(*(_run(p) for p in prepare.parts))
  131. logger.info("[%s] All %d parts uploaded, completing…", self._log_tag, job.total_parts)
  132. return await self._complete(job)
  133. async def _prepare(self, job: _Job, file_type: int, file_name: str, hashes: dict[str, str]) -> _PrepareResult:
  134. body = {"file_type": file_type, "file_name": file_name, "file_size": job.file_size,
  135. "md5": hashes["md5"], "sha1": hashes["sha1"], "md5_10m": hashes["md5_10m"]}
  136. try:
  137. raw = await self._post(job, "upload_prepare", body)
  138. except RuntimeError as exc:
  139. if f"{_BIZ_CODE_DAILY_LIMIT}" in str(exc):
  140. raise UploadDailyLimitExceededError(file_name, job.file_size, str(exc)) from exc
  141. raise
  142. return _parse_prepare_response(raw)
  143. async def _upload_one_part(self, job: _Job, part: _PreparePart) -> None:
  144. """PUT one part to COS, then call ``upload_part_finish``."""
  145. part_index, total_parts = part.index, job.total_parts
  146. offset = (part_index - 1) * job.block_size
  147. # Per-part block_size wins; fall back to the response-level value.
  148. length = min(part.block_size if part.block_size > 0 else job.block_size, job.file_size - offset)
  149. data = await asyncio.get_running_loop().run_in_executor(None, _read_file_chunk, job.file_path, offset, length)
  150. md5_hex = hashlib.md5(data).hexdigest()
  151. logger.debug("[%s] Part %d/%d: uploading %s (offset=%d md5=%s)", self._log_tag, part_index, total_parts,
  152. format_size(length), offset, md5_hex)
  153. await self._put_to_presigned_url(part.presigned_url, data, part_index, total_parts)
  154. await self._part_finish_with_retry(job, part_index, length, md5_hex)
  155. job.completed += 1
  156. logger.debug("[%s] Part %d/%d done (%d/%d total)", self._log_tag, part_index, total_parts, job.completed,
  157. total_parts)
  158. async def _with_retries(self, attempt_fn: Callable[[], Awaitable[Any]], *, max_retries: int, base_delay: float,
  159. label: str, failure_label: str) -> Any:
  160. """Run *attempt_fn* up to ``max_retries + 1`` times with exponential backoff."""
  161. last_exc: Exception | None = None
  162. for attempt in range(max_retries + 1):
  163. try:
  164. return await attempt_fn()
  165. except Exception as exc:
  166. last_exc = exc
  167. if attempt < max_retries:
  168. delay = base_delay * (2 ** attempt)
  169. logger.warning("[%s] %s attempt %d failed, retry in %.1fs: %s", self._log_tag, label, attempt + 1,
  170. delay, exc)
  171. await asyncio.sleep(delay)
  172. raise RuntimeError(f"{failure_label} failed after {max_retries + 1} attempts: {last_exc}")
  173. async def _put_to_presigned_url(self, url: str, data: bytes, part_index: int, total_parts: int) -> None:
  174. """PUT part data to a pre-signed COS URL with retry."""
  175. async def _attempt() -> None:
  176. resp = await asyncio.wait_for(self._http_put(url, data=data, headers={"Content-Length": str(len(data))}),
  177. timeout=_PART_UPLOAD_TIMEOUT)
  178. status = getattr(resp, "status_code", 0)
  179. if 200 <= status < 300:
  180. logger.debug("[%s] PUT part %d/%d: %d OK", self._log_tag, part_index, total_parts, status)
  181. return
  182. try:
  183. body_preview = getattr(resp, "text", "")[:200]
  184. except Exception: # pragma: no cover — defensive
  185. body_preview = ""
  186. raise RuntimeError(f"COS PUT returned {status}: {body_preview}")
  187. await self._with_retries(_attempt, max_retries=_PART_UPLOAD_MAX_RETRIES, base_delay=1.0,
  188. label=f"PUT part {part_index}/{total_parts}",
  189. failure_label=f"Part {part_index}/{total_parts} upload")
  190. async def _part_finish_with_retry(self, job: _Job, part_index: int, block_size: int, md5: str) -> None:
  191. """Call ``upload_part_finish``, retrying on biz_code 40093001 until ``job.retry_timeout``."""
  192. body = {"upload_id": job.upload_id, "part_index": part_index, "block_size": block_size, "md5": md5}
  193. loop = asyncio.get_running_loop()
  194. start, attempt = loop.time(), 0
  195. while True:
  196. try:
  197. await self._post(job, "upload_part_finish", body)
  198. return
  199. except RuntimeError as exc:
  200. if f"{_BIZ_CODE_PART_RETRYABLE}" not in str(exc):
  201. raise
  202. elapsed = loop.time() - start
  203. if elapsed >= job.retry_timeout:
  204. raise RuntimeError(f"upload_part_finish persistent retry timed out after "
  205. f"{job.retry_timeout:.0f}s ({attempt} retries): {exc}") from exc
  206. attempt += 1
  207. logger.debug("[%s] part_finish retryable error, attempt %d, elapsed=%.1fs: %s", self._log_tag, attempt,
  208. elapsed, exc)
  209. await asyncio.sleep(_PART_FINISH_RETRY_INTERVAL)
  210. async def _complete(self, job: _Job) -> dict[str, Any]:
  211. """Call ``complete_upload`` with retry — the ``/files`` endpoint (same as the simple URL upload)
  212. selects the chunked-completion path when only ``upload_id`` is sent."""
  213. return await self._with_retries(
  214. lambda: self._post(job, "files", {"upload_id": job.upload_id}), max_retries=_COMPLETE_UPLOAD_MAX_RETRIES,
  215. base_delay=_COMPLETE_UPLOAD_BASE_DELAY, label="complete_upload", failure_label="complete_upload")
  216. def format_size(size_bytes: int) -> str:
  217. """Return a human-readable file size string (e.g. ``'12.3 MB'``)."""
  218. size = float(size_bytes)
  219. for unit in ("B", "KB", "MB", "GB"):
  220. if size < 1024.0:
  221. return f"{size:.1f} {unit}"
  222. size /= 1024.0
  223. return f"{size:.1f} TB"
  224. def _read_file_chunk(file_path: str, offset: int, length: int) -> bytes:
  225. """Read *length* bytes at *offset*; raises IOError on a short read (truncated file)."""
  226. with open(file_path, "rb") as fh:
  227. fh.seek(offset)
  228. data = fh.read(length)
  229. if len(data) != length:
  230. raise OSError(f"Short read from {file_path}: expected {length} bytes at offset {offset}, got {len(data)} "
  231. f"(file may be truncated)")
  232. return data
  233. def _compute_file_hashes(file_path: str, file_size: int) -> dict[str, str]:
  234. """Compute md5, sha1, and md5_10m in a single pass (for small files md5_10m is just the full md5)."""
  235. md5, sha1, md5_10m = hashlib.md5(), hashlib.sha1(), hashlib.md5()
  236. need_10m = file_size > _MD5_10M_SIZE
  237. bytes_read = 0
  238. with open(file_path, "rb") as fh:
  239. while chunk := fh.read(65536):
  240. md5.update(chunk)
  241. sha1.update(chunk)
  242. if need_10m and (remaining := _MD5_10M_SIZE - bytes_read) > 0:
  243. md5_10m.update(chunk[:remaining])
  244. bytes_read += len(chunk)
  245. full_md5 = md5.hexdigest()
  246. return {"md5": full_md5, "sha1": sha1.hexdigest(), "md5_10m": md5_10m.hexdigest() if need_10m else full_md5}