tts_tool_delivery.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425
  1. """Long-form chunking, ffmpeg encoding, container repair and delivery packing (``tools.tts_tool``).
  2. Provider-agnostic post-processing: split text under a per-request cap, wrap raw PCM as WAV,
  3. convert WAV/MP3 to the target container, sniff/repair mislabelled ``.ogg`` files, and combine
  4. final-encoded chunks under a destination platform's upload limit. Also home of the sibling
  5. helpers ``_origin`` / ``_section`` / ``_remove_quietly``.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import os
  10. import re
  11. import shlex
  12. import shutil
  13. import struct
  14. import subprocess
  15. import tempfile
  16. import uuid
  17. from dataclasses import dataclass
  18. from pathlib import Path
  19. from typing import Any, Dict, List, Optional, Tuple
  20. from hermes_cli._subprocess_compat import windows_hide_flags
  21. from tools.tts_command_provider import (
  22. BUILTIN_TTS_PROVIDERS, DEFAULT_COMMAND_TTS_MAX_TEXT_LENGTH, _get_named_provider_config,
  23. _is_command_provider_config)
  24. logger = logging.getLogger("tools.tts_tool")
  25. def _origin():
  26. """``tools.tts_tool``, resolved per call so seams monkeypatched there still apply."""
  27. from tools import tts_tool
  28. return tts_tool
  29. def _section(tts_config: Any, key: str) -> dict[str, Any]:
  30. """``tts.<key>`` as a dict (``null``/non-dict sections read as empty)."""
  31. section = tts_config.get(key) if isinstance(tts_config, dict) else None
  32. return section if isinstance(section, dict) else {}
  33. FALLBACK_MAX_TEXT_LENGTH = 4000 # provider not recognised at all
  34. # Per-provider input-character caps (official docs); override: ``tts.<provider>.max_text_length``.
  35. PROVIDER_MAX_TEXT_LENGTH: dict[str, int] = {
  36. "edge": 5000, # edge-tts practical sync limit
  37. "openai": 4096, # https://platform.openai.com/docs/guides/text-to-speech
  38. "xai": 15000, # https://docs.x.ai/developers/model-capabilities/audio/text-to-speech
  39. "minimax": 10000, # https://platform.minimax.io/docs/api-reference/speech-t2a-http (sync)
  40. "mistral": 4000, # conservative; no published per-request cap
  41. "gemini": 32000, # 32k-token context window; char cap is conservative
  42. "elevenlabs": 10000, # fallback when model-aware lookup can't resolve (multilingual_v2)
  43. "neutts": 2000, # local model, quality falls off on long text
  44. "kittentts": 2000, # local 25MB model
  45. "piper": 5000, # local VITS model, phoneme-based; practical cap
  46. }
  47. # ElevenLabs caps vary by model_id. https://elevenlabs.io/docs/overview/models
  48. ELEVENLABS_MODEL_MAX_TEXT_LENGTH: dict[str, int] = {
  49. "eleven_v3": 5000, "eleven_ttv_v3": 5000,
  50. "eleven_multilingual_v2": 10000, "eleven_multilingual_v1": 10000,
  51. "eleven_english_sts_v2": 10000, "eleven_english_sts_v1": 10000,
  52. "eleven_flash_v2": 30000, "eleven_flash_v2_5": 40000}
  53. def _positive_int(value: Any) -> Optional[int]:
  54. """*value* when it is a positive non-bool int, else None."""
  55. return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None
  56. def _resolve_max_text_length(provider: Optional[str], tts_config: Optional[dict[str, Any]] = None) -> int:
  57. """Input-character cap for *provider*: ``tts.<provider>.max_text_length`` > ElevenLabs model
  58. table > ``PROVIDER_MAX_TEXT_LENGTH`` > command provider's own ``max_text_length`` (else
  59. ``DEFAULT_COMMAND_TTS_MAX_TEXT_LENGTH``) > ``FALLBACK_MAX_TEXT_LENGTH``. Non-positive /
  60. non-int overrides fall through so a broken config can't disable truncation."""
  61. if not provider:
  62. return FALLBACK_MAX_TEXT_LENGTH
  63. key = provider.lower().strip()
  64. cfg = tts_config or {}
  65. prov_cfg = _section(cfg, key)
  66. override = _positive_int(prov_cfg.get("max_text_length"))
  67. if override:
  68. return override
  69. if key == "elevenlabs":
  70. from tools.tts_tool_providers import DEFAULT_ELEVENLABS_MODEL_ID # providers imports this module
  71. model_id = prov_cfg.get("model_id") or DEFAULT_ELEVENLABS_MODEL_ID
  72. mapped = ELEVENLABS_MODEL_MAX_TEXT_LENGTH.get(str(model_id).strip())
  73. if mapped:
  74. return mapped
  75. if key in PROVIDER_MAX_TEXT_LENGTH:
  76. return PROVIDER_MAX_TEXT_LENGTH[key]
  77. if key not in BUILTIN_TTS_PROVIDERS:
  78. named = _get_named_provider_config(cfg, key)
  79. if _is_command_provider_config(named):
  80. return _positive_int(named.get("max_text_length")) or DEFAULT_COMMAND_TTS_MAX_TEXT_LENGTH
  81. return FALLBACK_MAX_TEXT_LENGTH
  82. # PCM output specs for Gemini TTS (fixed by the API): 24kHz mono 16-bit (L16).
  83. GEMINI_TTS_SAMPLE_RATE, GEMINI_TTS_CHANNELS, GEMINI_TTS_SAMPLE_WIDTH = 24000, 1, 2
  84. # ffmpeg args producing the Ogg/Opus voice-bubble encoding Telegram & co expect.
  85. _OPUS_VOICE_ARGS = [
  86. "-acodec", "libopus", "-ac", "1", "-b:a", "48k", "-vbr", "on",
  87. "-application", "voip", "-compression_level", "10"]
  88. # --- Text chunking and delivery profiles ---
  89. @dataclass(frozen=True)
  90. class AudioDeliveryProfile:
  91. """Destination-platform constraints for generated TTS audio."""
  92. platform: str
  93. max_file_bytes: int
  94. safety_ratio: float = 0.85
  95. @property
  96. def target_file_bytes(self) -> int:
  97. """Conservative packing target below the platform hard limit."""
  98. return max(1, int(self.max_file_bytes * self.safety_ratio))
  99. _PLATFORM_AUDIO_DEFAULTS: dict[str, dict[str, Any]] = {
  100. "discord": {"max_file_bytes": 10 * 1024 * 1024, "safety_ratio": 0.85},
  101. "telegram": {"max_file_bytes": 50 * 1024 * 1024, "safety_ratio": 0.85},
  102. "default": {"max_file_bytes": 10 * 1024 * 1024, "safety_ratio": 0.85}}
  103. def _resolve_audio_delivery_profile(
  104. platform: Optional[str], tts_config: Optional[dict[str, Any]] = None) -> AudioDeliveryProfile:
  105. """Resolve upload constraints, including optional ``tts.delivery_profiles`` overrides."""
  106. key = (platform or "default").lower().strip() or "default"
  107. defaults = dict(_PLATFORM_AUDIO_DEFAULTS.get(key) or _PLATFORM_AUDIO_DEFAULTS["default"])
  108. overrides = _section(_section(tts_config, "delivery_profiles"), key)
  109. defaults.update({k: v for k, v in overrides.items() if v is not None})
  110. max_file_bytes = (_positive_int(defaults.get("max_file_bytes"))
  111. or _PLATFORM_AUDIO_DEFAULTS["default"]["max_file_bytes"])
  112. safety_ratio = defaults.get("safety_ratio", 0.85)
  113. if (isinstance(safety_ratio, bool) or not isinstance(safety_ratio, (int, float))
  114. or not 0 < safety_ratio <= 1):
  115. safety_ratio = 0.85
  116. return AudioDeliveryProfile(platform=key, max_file_bytes=max_file_bytes, safety_ratio=float(safety_ratio))
  117. def _pack_under_cap(pieces: list[str], max_chars: int, *, slice_oversized: bool = False) -> list[str]:
  118. """Greedily join *pieces* with single spaces, starting a new chunk past *max_chars*. With
  119. ``slice_oversized`` an over-long piece flushes the running chunk and emits its hard slices as
  120. their own chunks (the tail slice is not merged with following pieces)."""
  121. chunks: list[str] = []
  122. current = ""
  123. for piece in pieces:
  124. if slice_oversized and len(piece) > max_chars:
  125. if current:
  126. chunks.append(current)
  127. current = ""
  128. chunks.extend(piece[i:i + max_chars] for i in range(0, len(piece), max_chars))
  129. continue
  130. candidate = f"{current} {piece}".strip()
  131. if current and len(candidate) > max_chars:
  132. chunks.append(current)
  133. current = piece
  134. else:
  135. current = candidate
  136. if current:
  137. chunks.append(current)
  138. return chunks
  139. def _split_oversized_sentence(sentence: str, max_chars: int) -> list[str]:
  140. """Split one over-limit sentence on word boundaries, then hard boundaries."""
  141. return _pack_under_cap(sentence.split(), max_chars, slice_oversized=True)
  142. def _split_text_for_tts(text: str, max_chars: int) -> list[str]:
  143. """Split text under a provider cap without dropping normalized content."""
  144. if max_chars <= 0:
  145. max_chars = FALLBACK_MAX_TEXT_LENGTH
  146. normalized = " ".join((text or "").split())
  147. if not normalized:
  148. return []
  149. if len(normalized) <= max_chars:
  150. return [normalized]
  151. expanded: list[str] = []
  152. for sentence in filter(None, (s.strip() for s in re.split(r"(?<=[.!?;:,])\s+", normalized))):
  153. if len(sentence) <= max_chars:
  154. expanded.append(sentence)
  155. else:
  156. expanded.extend(_split_oversized_sentence(sentence, max_chars))
  157. return _pack_under_cap(expanded, max_chars)
  158. def _pack_audio_files_for_delivery(audio_paths: list[str], profile: AudioDeliveryProfile) -> list[list[str]]:
  159. """Group final-encoded chunks under the size target; never mixes suffixes (can't concat-copy)."""
  160. groups: list[list[str]] = []
  161. current: list[str] = []
  162. current_size, current_suffix = 0, ""
  163. for path in audio_paths:
  164. size, suffix = Path(path).stat().st_size, Path(path).suffix.lower()
  165. if current and (current_size + size > profile.target_file_bytes or suffix != current_suffix):
  166. groups.append(current)
  167. current, current_size = [], 0
  168. current.append(path)
  169. current_size, current_suffix = current_size + size, suffix
  170. return groups + [current] if current else groups
  171. # --- ffmpeg encoding helpers ---
  172. def _ffmpeg_run(
  173. ffmpeg: str, args: list[str], *, timeout: int = 30, check: bool = False, capture: bool = True,
  174. ) -> subprocess.CompletedProcess:
  175. """Run ``ffmpeg <args>`` headless (no stdin, hidden window on Windows)."""
  176. return subprocess.run([ffmpeg, *args], capture_output=capture, check=check, timeout=timeout,
  177. stdin=subprocess.DEVNULL, creationflags=windows_hide_flags())
  178. def _remove_quietly(path: Optional[str]) -> None:
  179. """Best-effort unlink (missing/locked files are ignored); None is a no-op."""
  180. if path:
  181. try:
  182. os.remove(path)
  183. except OSError:
  184. pass
  185. def _wav_sidecar_path(output_path: str) -> str:
  186. """Path a WAV-native engine writes to before conversion to *output_path*'s format."""
  187. return output_path if output_path.endswith(".wav") else output_path.rsplit(".", 1)[0] + ".wav"
  188. def _finalize_wav_output(wav_path: str, output_path: str) -> str:
  189. """Move a WAV-native engine's output into the requested container: ffmpeg-convert when
  190. available, else rename so the tool stays usable (misleading extension, audio plays)."""
  191. if wav_path == output_path:
  192. return output_path
  193. ffmpeg = shutil.which("ffmpeg")
  194. if not ffmpeg:
  195. os.rename(wav_path, output_path)
  196. return output_path
  197. _ffmpeg_run(ffmpeg, ["-i", wav_path, "-y", "-loglevel", "error", output_path],
  198. check=True, capture=False)
  199. _remove_quietly(wav_path)
  200. return output_path
  201. def _wrap_pcm_as_wav(
  202. pcm_bytes: bytes, sample_rate: int = GEMINI_TTS_SAMPLE_RATE,
  203. channels: int = GEMINI_TTS_CHANNELS, sample_width: int = GEMINI_TTS_SAMPLE_WIDTH) -> bytes:
  204. """Wrap raw signed-little-endian PCM (e.g. Gemini's L16) with a minimal WAV RIFF header."""
  205. block_align = channels * sample_width
  206. fmt_chunk = struct.pack("<4sIHHIIHH", b"fmt ", 16, 1, channels, sample_rate,
  207. sample_rate * block_align, block_align, sample_width * 8)
  208. data_chunk_header = struct.pack("<4sI", b"data", len(pcm_bytes))
  209. riff_size = 4 + len(fmt_chunk) + len(data_chunk_header) + len(pcm_bytes)
  210. riff_header = struct.pack("<4sI4s", b"RIFF", riff_size, b"WAVE")
  211. return riff_header + fmt_chunk + data_chunk_header + pcm_bytes
  212. def _write_wav_bytes_as(wav_bytes: bytes, output_path: str) -> str:
  213. """Write in-memory WAV to *output_path*, ffmpeg-converting to its container. ``.ogg`` is forced
  214. to Opus (ffmpeg's .ogg default is Vorbis, which voice bubbles reject). A failed conversion
  215. raises RuntimeError; without ffmpeg the raw WAV is written under the requested name
  216. (misleading extension, but the audio still plays)."""
  217. if output_path.lower().endswith(".wav"):
  218. with open(output_path, "wb") as f:
  219. f.write(wav_bytes)
  220. return output_path
  221. with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
  222. tmp.write(wav_bytes)
  223. wav_path = tmp.name
  224. try:
  225. ffmpeg = shutil.which("ffmpeg")
  226. if ffmpeg:
  227. opus = _OPUS_VOICE_ARGS if output_path.lower().endswith(".ogg") else []
  228. result = _ffmpeg_run(
  229. ffmpeg, ["-i", wav_path, *opus, "-y", "-loglevel", "error", output_path])
  230. if result.returncode != 0:
  231. stderr = result.stderr.decode("utf-8", errors="ignore")[:300]
  232. raise RuntimeError(f"ffmpeg conversion failed: {stderr}")
  233. else:
  234. logger.warning("ffmpeg not found; writing raw WAV to %s (extension may be misleading)", output_path)
  235. shutil.copyfile(wav_path, output_path)
  236. finally:
  237. _remove_quietly(wav_path)
  238. return output_path
  239. def _convert_to_opus(mp3_path: str) -> Optional[str]:
  240. """Convert any ffmpeg-readable audio file to OGG Opus next to it; None on failure."""
  241. from gateway.platforms.base import transcode_to_ogg_opus
  242. return transcode_to_ogg_opus(mp3_path, bitrate="48k", timeout=30, output_path=mp3_path.rsplit(".", 1)[0] + ".ogg")
  243. # --- Container sniffing / repair ---
  244. # Several backends ignore the requested opus format (Edge/xAI emit MP3, Piper WAV, some
  245. # OpenAI-compatible servers ignore response_format="opus"), which breaks native voice bubbles:
  246. # sniff the magic bytes once after synthesis and repair when they don't match the extension.
  247. def _sniff_audio_container(path: str) -> str:
  248. """Return a container id ('ogg', 'wav', 'mp3', 'flac', ...) or 'unknown'."""
  249. from tools.audio_container import sniff_container
  250. try:
  251. with open(path, "rb") as fh:
  252. return sniff_container(fh.read(12)) or "unknown"
  253. except OSError:
  254. return "unknown"
  255. def _repair_ogg_container(file_str: str) -> str:
  256. """Ensure a ``.ogg`` path really holds Ogg: transcode in place, else rename to the sniffed
  257. real extension so platforms get an honest file instead of a 0-second voice bubble."""
  258. container = _sniff_audio_container(file_str) if file_str.endswith(".ogg") else "ogg"
  259. if container in ("ogg", "unknown"):
  260. return file_str
  261. logger.info("TTS wrote %s bytes into a .ogg path (%s) — transcoding to real Ogg/Opus", container, file_str)
  262. from gateway.platforms.base import transcode_to_ogg_opus
  263. repaired = transcode_to_ogg_opus(file_str, bitrate="48k", timeout=30, output_path=file_str)
  264. if repaired:
  265. return repaired
  266. honest = f"{file_str[:-4]}.{container}"
  267. try:
  268. os.replace(file_str, honest)
  269. except OSError:
  270. return file_str
  271. logger.warning("Could not transcode %s to Ogg/Opus — renamed to %s so the "
  272. "file is delivered with its real format", file_str, honest)
  273. return honest
  274. # --- Long-form audio combination and delivery packing ---
  275. def _concat_audio_files(audio_paths: list[str], output_path: str, *, voice_compatible: bool = False) -> Optional[str]:
  276. """Combine independently encoded chunks with ffmpeg (never byte-joined). OGG/Opus is always
  277. re-encoded (even without voice opt-in); matching MP3 chunks keep their frames (``-c:a copy``).
  278. None when ffmpeg is missing/fails so callers keep the valid parts."""
  279. if not audio_paths:
  280. raise ValueError("No audio chunks to combine")
  281. if len(audio_paths) == 1:
  282. if os.path.abspath(audio_paths[0]) != os.path.abspath(output_path):
  283. shutil.copyfile(audio_paths[0], output_path)
  284. return output_path
  285. ffmpeg = shutil.which("ffmpeg")
  286. if not ffmpeg:
  287. return None
  288. destination = Path(output_path)
  289. destination.parent.mkdir(parents=True, exist_ok=True)
  290. concat_path = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.concat.txt")
  291. temp_output = destination.with_name(f".{destination.stem}.{uuid.uuid4().hex}.combining{destination.suffix}")
  292. try:
  293. entries = "".join(f"file {shlex.quote(os.path.abspath(p))}\n" for p in audio_paths)
  294. concat_path.write_text(entries, encoding="utf-8")
  295. args = ["-y", "-loglevel", "error", "-f", "concat", "-safe", "0", "-i", str(concat_path), "-vn"]
  296. suffix = destination.suffix.lower()
  297. if voice_compatible or suffix in {".ogg", ".opus"}:
  298. args += ["-c:a", "libopus", "-ac", "1", "-b:a", "64k", "-vbr", "off"]
  299. elif suffix == ".mp3" and all(Path(path).suffix.lower() == ".mp3" for path in audio_paths):
  300. args += ["-c:a", "copy"]
  301. result = _ffmpeg_run(ffmpeg, [*args, str(temp_output)], timeout=120)
  302. if result.returncode == 0 and temp_output.exists() and temp_output.stat().st_size > 0:
  303. os.replace(temp_output, destination)
  304. return str(destination)
  305. logger.warning("ffmpeg audio combine failed: %s", result.stderr.decode("utf-8", errors="ignore")[:500])
  306. except (OSError, subprocess.TimeoutExpired) as exc:
  307. logger.warning("ffmpeg audio combine failed: %s", exc)
  308. finally:
  309. for path in (concat_path, temp_output):
  310. _remove_quietly(path)
  311. return None
  312. def _build_audio_delivery_files(
  313. audio_paths: list[str], output_path: str, profile: AudioDeliveryProfile, *, voice_compatible: bool = False,
  314. ) -> tuple[list[str], bool]:
  315. """Pack final-encoded chunks under the hard upload limit -> ``(final_paths, combined_any)``.
  316. Groups are packed against the conservative target, then each combined artifact is checked
  317. at its real size; an over-limit group is split in half and retried. A failed combine
  318. returns the constituent files separately. A single chunk above the hard limit fails closed."""
  319. if not audio_paths:
  320. raise ValueError("No final-encoded TTS audio chunks")
  321. for path in audio_paths:
  322. size = Path(path).stat().st_size
  323. if size > profile.max_file_bytes:
  324. raise ValueError(
  325. f"Final-encoded TTS chunk exceeds {profile.platform} delivery "
  326. f"limit ({size} > {profile.max_file_bytes} bytes): {path}")
  327. base = Path(output_path)
  328. scratch_outputs: list[str] = []
  329. combined_any, combine_index = False, 0
  330. def emit(group: list[str]) -> list[str]:
  331. nonlocal combined_any, combine_index
  332. if len(group) == 1:
  333. return list(group)
  334. combine_index += 1
  335. scratch = base.with_name(f".{base.stem}.delivery{combine_index:03d}.{uuid.uuid4().hex}{base.suffix}")
  336. combined = _concat_audio_files(group, str(scratch), voice_compatible=voice_compatible)
  337. if not combined:
  338. return list(group)
  339. scratch_outputs.append(combined)
  340. if Path(combined).stat().st_size <= profile.max_file_bytes:
  341. combined_any = True
  342. return [combined]
  343. _remove_quietly(combined)
  344. midpoint = max(1, len(group) // 2)
  345. return emit(group[:midpoint]) + emit(group[midpoint:])
  346. groups = _pack_audio_files_for_delivery(audio_paths, profile)
  347. packed = [path for group in groups for path in emit(group)]
  348. final_paths: list[str] = []
  349. for index, source in enumerate(packed, start=1):
  350. destination = base
  351. if len(packed) > 1:
  352. destination = base.with_name(f"{base.stem}.part{index:02d}{Path(source).suffix or base.suffix}")
  353. if os.path.abspath(source) != os.path.abspath(destination):
  354. destination.parent.mkdir(parents=True, exist_ok=True)
  355. os.replace(source, destination)
  356. if destination.stat().st_size > profile.max_file_bytes:
  357. raise ValueError(f"Final TTS deliverable exceeds {profile.platform} delivery limit: {destination}")
  358. final_paths.append(str(destination))
  359. try:
  360. return final_paths, combined_any
  361. finally:
  362. for scratch in scratch_outputs:
  363. if scratch not in final_paths:
  364. _remove_quietly(scratch)