| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292 |
- """QQ Bot chunked upload flow (files 10 MB..~100 MB; inline uploads cap at ~10 MB):
- 1. POST .../upload_prepare → upload_id, block_size, pre-signed COS part URLs;
- 2. per part: PUT bytes to COS, then POST .../upload_part_finish;
- 3. POST .../files with {"upload_id"} → ``file_info`` for a RichMedia message.
- biz_code 40093001 = part_finish retryable until ``retry_timeout``; 40093002 = daily
- quota (UploadDailyLimitExceededError); other API/I/O failures raise RuntimeError.
- Ported from WideLee's qqbot-agent-sdk v1.2.2 (authorship via Co-authored-by).
- """
- from __future__ import annotations
- import asyncio
- import hashlib
- import logging
- from dataclasses import dataclass
- from pathlib import Path
- from typing import Any, Awaitable, Callable, Dict, List
- from .constants import FILE_UPLOAD_TIMEOUT
- logger = logging.getLogger(__name__)
- _BIZ_CODE_DAILY_LIMIT = 40093002 # upload_prepare: daily cumulative limit
- _BIZ_CODE_PART_RETRYABLE = 40093001 # upload_part_finish: transient
- _DEFAULT_CONCURRENT_PARTS = 1
- _MAX_CONCURRENT_PARTS = 10
- _PART_UPLOAD_TIMEOUT = 300.0 # 5 minutes per COS PUT
- _PART_UPLOAD_MAX_RETRIES = 2
- _PART_FINISH_RETRY_INTERVAL = 1.0
- _PART_FINISH_DEFAULT_TIMEOUT = 120.0
- _PART_FINISH_MAX_TIMEOUT = 600.0
- _COMPLETE_UPLOAD_MAX_RETRIES = 2
- _COMPLETE_UPLOAD_BASE_DELAY = 2.0
- _MD5_10M_SIZE = 10_002_432 # first N bytes used for the ``md5_10m`` hash (per QQ API spec)
- class _UploadError(Exception):
- def __init__(self, file_name: str, file_size: int, message: str) -> None:
- self.file_name, self.file_size = file_name, file_size
- super().__init__(message)
- @property
- def file_size_human(self) -> str:
- return format_size(self.file_size)
- class UploadDailyLimitExceededError(_UploadError):
- """Raised when ``upload_prepare`` returns biz_code 40093002 (daily quota hit)."""
- def __init__(self, file_name: str, file_size: int, message: str = "") -> None:
- super().__init__(file_name, file_size, message or f"Daily upload limit exceeded for {file_name!r}")
- class UploadFileTooLargeError(_UploadError):
- """Raised when a file exceeds the platform per-file size limit."""
- def __init__(self, file_name: str, file_size: int, limit_bytes: int = 0, message: str = "") -> None:
- self.limit_bytes = limit_bytes
- limit_str = f" ({format_size(limit_bytes)})" if limit_bytes else ""
- super().__init__(file_name, file_size,
- message or f"File {file_name!r} ({format_size(file_size)}) exceeds platform limit{limit_str}")
- @property
- def limit_human(self) -> str:
- return format_size(self.limit_bytes) if self.limit_bytes else "unknown"
- @dataclass
- class _PreparePart:
- index: int
- presigned_url: str
- block_size: int = 0
- @dataclass
- class _PrepareResult:
- upload_id: str
- block_size: int
- parts: list[_PreparePart]
- concurrency: int = _DEFAULT_CONCURRENT_PARTS
- retry_timeout: float = 0.0
- def _parse_prepare_response(raw: dict[str, Any]) -> _PrepareResult:
- """Parse upload_prepare response (either bare or wrapped in ``data``)."""
- src = raw.get("data") if isinstance(raw.get("data"), dict) else raw
- if not (upload_id := str(src.get("upload_id", ""))):
- raise ValueError(f"upload_prepare response missing upload_id: {str(raw)[:200]}")
- block_size = int(src.get("block_size", 0))
- raw_parts = src.get("parts") or src.get("part_list") or []
- if not isinstance(raw_parts, list) or not raw_parts:
- raise ValueError(f"upload_prepare response missing parts: {str(raw)[:200]}")
- parts = [_PreparePart(index=int(p.get("part_index") or p.get("index") or 0),
- presigned_url=str(p.get("presigned_url") or p.get("url") or ""),
- block_size=int(p.get("block_size", 0)))
- for p in raw_parts if isinstance(p, dict)]
- return _PrepareResult(
- upload_id=upload_id, block_size=block_size, parts=parts,
- concurrency=int(src.get("concurrency", _DEFAULT_CONCURRENT_PARTS)) or _DEFAULT_CONCURRENT_PARTS,
- retry_timeout=float(src.get("retry_timeout", 0.0) or 0.0))
- def _api_path(chat_type: str, target_id: str, endpoint: str) -> str:
- return f"{'/v2/users' if chat_type == 'c2c' else '/v2/groups'}/{target_id}/{endpoint}"
- @dataclass
- class _Job:
- """Per-upload state shared by the part workers."""
- chat_type: str
- target_id: str
- file_path: str
- file_size: int
- upload_id: str = ""
- block_size: int = 0
- total_parts: int = 0
- retry_timeout: float = 0.0
- completed: int = 0
- def path(self, endpoint: str) -> str:
- return _api_path(self.chat_type, self.target_id, endpoint)
- class ChunkedUploader:
- """Run the prepare → PUT parts → complete sequence. ``api_request`` is the adapter's
- bound ``_api_request(method, path, body=..., timeout=...)`` (injected to avoid a
- circular import; must raise RuntimeError with the biz_code in the message);
- ``http_put`` is ``(url, data, headers) -> httpx-like response`` for COS PUTs."""
- def __init__(self, api_request: Callable[..., Awaitable[dict[str, Any]]],
- http_put: Callable[..., Awaitable[Any]], log_tag: str = "QQBot") -> None:
- self._api_request = api_request
- self._http_put = http_put
- self._log_tag = log_tag
- async def _post(self, job: _Job, endpoint: str, body: dict[str, Any]) -> dict[str, Any]:
- return await self._api_request("POST", job.path(endpoint), body=body, timeout=FILE_UPLOAD_TIMEOUT)
- async def upload(self, chat_type: str, target_id: str, file_path: str, file_type: int, file_name: str) -> dict[str, Any]:
- """Run the full chunked upload (``chat_type`` 'c2c'|'group', ``file_type`` MEDIA_TYPE_*)
- and return the raw ``complete_upload`` response (contains ``file_info``).
- Raises UploadDailyLimitExceededError (40093002), UploadFileTooLargeError, RuntimeError."""
- if chat_type not in {"c2c", "group"}:
- raise ValueError(f"ChunkedUploader: unsupported chat_type {chat_type!r}")
- job = _Job(chat_type, target_id, file_path, Path(file_path).stat().st_size)
- logger.info("[%s] Chunked upload start: file=%s size=%s type=%d", self._log_tag, file_name,
- format_size(job.file_size), file_type)
- # Hashing is blocking I/O → executor.
- hashes = await asyncio.get_running_loop().run_in_executor(None, _compute_file_hashes, file_path, job.file_size)
- prepare = await self._prepare(job, file_type, file_name, hashes)
- max_concurrent = min(prepare.concurrency, _MAX_CONCURRENT_PARTS)
- job.upload_id, job.block_size, job.total_parts = prepare.upload_id, prepare.block_size, len(prepare.parts)
- job.retry_timeout = min(prepare.retry_timeout if prepare.retry_timeout > 0 else _PART_FINISH_DEFAULT_TIMEOUT,
- _PART_FINISH_MAX_TIMEOUT)
- logger.info("[%s] Prepared: upload_id=%s block_size=%s parts=%d concurrency=%d", self._log_tag,
- job.upload_id, format_size(job.block_size), job.total_parts, max_concurrent)
- sem = asyncio.Semaphore(max(max_concurrent, 1))
- async def _run(part: _PreparePart) -> None:
- async with sem:
- await self._upload_one_part(job, part)
- await asyncio.gather(*(_run(p) for p in prepare.parts))
- logger.info("[%s] All %d parts uploaded, completing…", self._log_tag, job.total_parts)
- return await self._complete(job)
- async def _prepare(self, job: _Job, file_type: int, file_name: str, hashes: dict[str, str]) -> _PrepareResult:
- body = {"file_type": file_type, "file_name": file_name, "file_size": job.file_size,
- "md5": hashes["md5"], "sha1": hashes["sha1"], "md5_10m": hashes["md5_10m"]}
- try:
- raw = await self._post(job, "upload_prepare", body)
- except RuntimeError as exc:
- if f"{_BIZ_CODE_DAILY_LIMIT}" in str(exc):
- raise UploadDailyLimitExceededError(file_name, job.file_size, str(exc)) from exc
- raise
- return _parse_prepare_response(raw)
- async def _upload_one_part(self, job: _Job, part: _PreparePart) -> None:
- """PUT one part to COS, then call ``upload_part_finish``."""
- part_index, total_parts = part.index, job.total_parts
- offset = (part_index - 1) * job.block_size
- # Per-part block_size wins; fall back to the response-level value.
- length = min(part.block_size if part.block_size > 0 else job.block_size, job.file_size - offset)
- data = await asyncio.get_running_loop().run_in_executor(None, _read_file_chunk, job.file_path, offset, length)
- md5_hex = hashlib.md5(data).hexdigest()
- logger.debug("[%s] Part %d/%d: uploading %s (offset=%d md5=%s)", self._log_tag, part_index, total_parts,
- format_size(length), offset, md5_hex)
- await self._put_to_presigned_url(part.presigned_url, data, part_index, total_parts)
- await self._part_finish_with_retry(job, part_index, length, md5_hex)
- job.completed += 1
- logger.debug("[%s] Part %d/%d done (%d/%d total)", self._log_tag, part_index, total_parts, job.completed,
- total_parts)
- async def _with_retries(self, attempt_fn: Callable[[], Awaitable[Any]], *, max_retries: int, base_delay: float,
- label: str, failure_label: str) -> Any:
- """Run *attempt_fn* up to ``max_retries + 1`` times with exponential backoff."""
- last_exc: Exception | None = None
- for attempt in range(max_retries + 1):
- try:
- return await attempt_fn()
- except Exception as exc:
- last_exc = exc
- if attempt < max_retries:
- delay = base_delay * (2 ** attempt)
- logger.warning("[%s] %s attempt %d failed, retry in %.1fs: %s", self._log_tag, label, attempt + 1,
- delay, exc)
- await asyncio.sleep(delay)
- raise RuntimeError(f"{failure_label} failed after {max_retries + 1} attempts: {last_exc}")
- async def _put_to_presigned_url(self, url: str, data: bytes, part_index: int, total_parts: int) -> None:
- """PUT part data to a pre-signed COS URL with retry."""
- async def _attempt() -> None:
- resp = await asyncio.wait_for(self._http_put(url, data=data, headers={"Content-Length": str(len(data))}),
- timeout=_PART_UPLOAD_TIMEOUT)
- status = getattr(resp, "status_code", 0)
- if 200 <= status < 300:
- logger.debug("[%s] PUT part %d/%d: %d OK", self._log_tag, part_index, total_parts, status)
- return
- try:
- body_preview = getattr(resp, "text", "")[:200]
- except Exception: # pragma: no cover — defensive
- body_preview = ""
- raise RuntimeError(f"COS PUT returned {status}: {body_preview}")
- await self._with_retries(_attempt, max_retries=_PART_UPLOAD_MAX_RETRIES, base_delay=1.0,
- label=f"PUT part {part_index}/{total_parts}",
- failure_label=f"Part {part_index}/{total_parts} upload")
- async def _part_finish_with_retry(self, job: _Job, part_index: int, block_size: int, md5: str) -> None:
- """Call ``upload_part_finish``, retrying on biz_code 40093001 until ``job.retry_timeout``."""
- body = {"upload_id": job.upload_id, "part_index": part_index, "block_size": block_size, "md5": md5}
- loop = asyncio.get_running_loop()
- start, attempt = loop.time(), 0
- while True:
- try:
- await self._post(job, "upload_part_finish", body)
- return
- except RuntimeError as exc:
- if f"{_BIZ_CODE_PART_RETRYABLE}" not in str(exc):
- raise
- elapsed = loop.time() - start
- if elapsed >= job.retry_timeout:
- raise RuntimeError(f"upload_part_finish persistent retry timed out after "
- f"{job.retry_timeout:.0f}s ({attempt} retries): {exc}") from exc
- attempt += 1
- logger.debug("[%s] part_finish retryable error, attempt %d, elapsed=%.1fs: %s", self._log_tag, attempt,
- elapsed, exc)
- await asyncio.sleep(_PART_FINISH_RETRY_INTERVAL)
- async def _complete(self, job: _Job) -> dict[str, Any]:
- """Call ``complete_upload`` with retry — the ``/files`` endpoint (same as the simple URL upload)
- selects the chunked-completion path when only ``upload_id`` is sent."""
- return await self._with_retries(
- lambda: self._post(job, "files", {"upload_id": job.upload_id}), max_retries=_COMPLETE_UPLOAD_MAX_RETRIES,
- base_delay=_COMPLETE_UPLOAD_BASE_DELAY, label="complete_upload", failure_label="complete_upload")
- def format_size(size_bytes: int) -> str:
- """Return a human-readable file size string (e.g. ``'12.3 MB'``)."""
- size = float(size_bytes)
- for unit in ("B", "KB", "MB", "GB"):
- if size < 1024.0:
- return f"{size:.1f} {unit}"
- size /= 1024.0
- return f"{size:.1f} TB"
- def _read_file_chunk(file_path: str, offset: int, length: int) -> bytes:
- """Read *length* bytes at *offset*; raises IOError on a short read (truncated file)."""
- with open(file_path, "rb") as fh:
- fh.seek(offset)
- data = fh.read(length)
- if len(data) != length:
- raise OSError(f"Short read from {file_path}: expected {length} bytes at offset {offset}, got {len(data)} "
- f"(file may be truncated)")
- return data
- def _compute_file_hashes(file_path: str, file_size: int) -> dict[str, str]:
- """Compute md5, sha1, and md5_10m in a single pass (for small files md5_10m is just the full md5)."""
- md5, sha1, md5_10m = hashlib.md5(), hashlib.sha1(), hashlib.md5()
- need_10m = file_size > _MD5_10M_SIZE
- bytes_read = 0
- with open(file_path, "rb") as fh:
- while chunk := fh.read(65536):
- md5.update(chunk)
- sha1.update(chunk)
- if need_10m and (remaining := _MD5_10M_SIZE - bytes_read) > 0:
- md5_10m.update(chunk[:remaining])
- bytes_read += len(chunk)
- full_md5 = md5.hexdigest()
- return {"md5": full_md5, "sha1": sha1.hexdigest(), "md5_10m": md5_10m.hexdigest() if need_10m else full_md5}
|