tts_tool_providers.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617
  1. """Cloud TTS backends for ``tools.tts_tool``: Edge, ElevenLabs, xAI, MiniMax, Mistral, Gemini.
  2. Each ``_generate_<provider>(text, output_path, tts_config) -> path`` writes one final-encoded
  3. file. Shared here: bounded upstream response reading (16 MiB cap so a hostile endpoint can't
  4. feed unbounded audio) and the auxiliary-model speech-tag rewrites. OpenAI/DeepInfra live in
  5. ``tts_tool_openai``. Origin seams (``_resolve_provider_key``, ``_import_*``)
  6. are resolved through :func:`_origin` at call time.
  7. """
  8. from __future__ import annotations
  9. import base64
  10. import contextlib
  11. import json
  12. import logging
  13. import os
  14. import re
  15. from dataclasses import dataclass, field
  16. from pathlib import Path
  17. from typing import Any, Dict, Optional
  18. from urllib.parse import urlparse
  19. from tools.tts_tool_delivery import _origin, _section, _wrap_pcm_as_wav, _write_wav_bytes_as
  20. from tools.xai_http import hermes_xai_user_agent
  21. logger = logging.getLogger("tools.tts_tool")
  22. DEFAULT_EDGE_VOICE = "en-US-AriaNeural"
  23. DEFAULT_ELEVENLABS_VOICE_ID = "pNInz6obpgDQGcFmaJgB" # Adam
  24. DEFAULT_ELEVENLABS_MODEL_ID = "eleven_multilingual_v2"
  25. DEFAULT_ELEVENLABS_STREAMING_MODEL_ID = "eleven_flash_v2_5"
  26. DEFAULT_MINIMAX_MODEL = "speech-02-hd"
  27. DEFAULT_MINIMAX_VOICE_ID = "English_expressive_narrator"
  28. DEFAULT_MINIMAX_BASE_URL = "https://api.minimax.io/v1/t2a_v2"
  29. DEFAULT_MINIMAX_CN_BASE_URL = "https://api.minimaxi.com/v1/t2a_v2"
  30. DEFAULT_MISTRAL_TTS_MODEL = "voxtral-mini-tts-2603"
  31. DEFAULT_MISTRAL_TTS_VOICE_ID = "c69964a6-ab8b-4f8a-9465-ec0925096ec8" # Paul - Neutral
  32. DEFAULT_XAI_VOICE_ID = "eve"
  33. DEFAULT_XAI_LANGUAGE = "en"
  34. DEFAULT_XAI_SAMPLE_RATE = 24000
  35. DEFAULT_XAI_BIT_RATE = 128000
  36. DEFAULT_XAI_AUTO_SPEECH_TAGS = False
  37. DEFAULT_XAI_BASE_URL = "https://api.x.ai/v1"
  38. # xAI `speed` accepts 0.7..1.5 (1.0 = API default, omitted from the payload);
  39. # `optimize_streaming_latency` is 0/1/2 (>0 trades quality for time-to-first-audio);
  40. # `text_normalization` speaks numbers/abbreviations/symbols in written form.
  41. DEFAULT_XAI_SPEED_MIN = 0.7
  42. DEFAULT_XAI_SPEED_MAX = 1.5
  43. DEFAULT_XAI_SPEED_DEFAULT = 1.0
  44. DEFAULT_XAI_OPTIMIZE_STREAMING_LATENCY_DEFAULT = 0
  45. DEFAULT_XAI_TEXT_NORMALIZATION_DEFAULT = False
  46. DEFAULT_GEMINI_TTS_MODEL = "gemini-2.5-flash-preview-tts"
  47. DEFAULT_GEMINI_TTS_VOICE = "Kore"
  48. DEFAULT_GEMINI_TTS_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
  49. DEFAULT_GEMINI_AUDIO_TAGS = False
  50. GEMINI_AUDIO_TAG_REWRITE_TASK = "tts_audio_tags"
  51. TTS_RESPONSE_BODY_LIMIT_BYTES = 16 * 1024 * 1024
  52. TTS_RESPONSE_BODY_CHUNK_BYTES = 64 * 1024
  53. _TRUE_WORDS = {"1", "true", "yes", "on", "enabled"}
  54. _FALSE_WORDS = {"0", "false", "no", "off", "disabled"}
  55. def _config_bool(value: Any, default: bool = False) -> bool:
  56. """Coerce common YAML/env bool spellings without treating random strings as true."""
  57. if isinstance(value, (bool, int, float)):
  58. return bool(value)
  59. normalized = value.strip().lower() if isinstance(value, str) else None
  60. return normalized in _TRUE_WORDS if normalized in _TRUE_WORDS | _FALSE_WORDS else default
  61. def _tts_response_format_from_path(output_path: str) -> str:
  62. """Pick an OpenAI-style response format (opus/wav/flac/mp3) from the output extension."""
  63. formats = ((".ogg", "opus"), (".wav", "wav"), (".flac", "flac"))
  64. return next((fmt for ext, fmt in formats if output_path.endswith(ext)), "mp3")
  65. def _require_key(env_var: str, provider_id: str, hint: str) -> str:
  66. """Resolve *env_var* via the origin key resolver; ValueError ``"<ENV> not set. <hint>"`` when absent."""
  67. api_key = _origin()._resolve_provider_key(env_var, provider_id) or ""
  68. if not api_key:
  69. raise ValueError(f"{env_var} not set. {hint}")
  70. return api_key
  71. # --- Bounded upstream response reading ---
  72. def _response_has_explicit_stream(response: Any) -> bool:
  73. """True for real ``requests`` responses (or doubles defining ``iter_content`` themselves)."""
  74. if not callable(getattr(response, "iter_content", None)):
  75. return False
  76. response_type = type(response)
  77. return response_type.__module__.startswith("requests.") or "iter_content" in vars(response_type)
  78. def _close_response(response: Any) -> None:
  79. close = getattr(response, "close", None)
  80. if callable(close):
  81. with contextlib.suppress(Exception):
  82. close()
  83. def _read_tts_response_bytes(response: Any, *, label: str, limit: Optional[int] = None) -> bytes:
  84. """Read an upstream TTS response with a hard byte cap."""
  85. limit = TTS_RESPONSE_BODY_LIMIT_BYTES if limit is None else limit
  86. chunks: list[bytes] = []
  87. total = 0
  88. try:
  89. if _response_has_explicit_stream(response):
  90. iterator = response.iter_content(chunk_size=TTS_RESPONSE_BODY_CHUNK_BYTES)
  91. else:
  92. content = vars(response).get("content", getattr(type(response), "content", b""))
  93. iterator = (content,) if isinstance(content, (str, bytes, bytearray)) else ()
  94. for chunk in iterator:
  95. if not chunk:
  96. continue
  97. if isinstance(chunk, str):
  98. chunk = chunk.encode("utf-8", errors="replace")
  99. chunk = bytes(chunk)
  100. total += len(chunk)
  101. if total > limit:
  102. _close_response(response)
  103. raise RuntimeError(f"{label} response exceeds {limit} bytes")
  104. chunks.append(chunk)
  105. return b"".join(chunks)
  106. finally:
  107. _close_response(response)
  108. def _parse_json_body(response: Any, raw: bytes) -> dict[str, Any]:
  109. """JSON from the already-read *raw* body. Unit-test doubles often only provide ``.json()``;
  110. real ``requests`` responses took the streaming path, so production never buffers eagerly."""
  111. if raw:
  112. return json.loads(raw.decode("utf-8"))
  113. if not _response_has_explicit_stream(response):
  114. json_reader = getattr(response, "json", None)
  115. if callable(json_reader):
  116. parsed = json_reader()
  117. return parsed if isinstance(parsed, dict) else {}
  118. return {}
  119. def _read_tts_response_json(response: Any, *, label: str, limit: Optional[int] = None) -> dict[str, Any]:
  120. return _parse_json_body(response, _read_tts_response_bytes(response, label=label, limit=limit))
  121. def _write_bytes(output_path: str, audio_bytes: bytes) -> str:
  122. with open(output_path, "wb") as f:
  123. f.write(audio_bytes)
  124. return output_path
  125. def _post_json(url: str, payload: dict[str, Any], headers: dict[str, str], **extra: Any):
  126. """Streaming ``requests.post`` with the shared 60s timeout (body read via the bounded readers)."""
  127. import requests
  128. return requests.post(url, headers=headers, json=payload, timeout=60, stream=True, **extra)
  129. # --- Auxiliary-model speech-tag rewrites ---
  130. def _auxiliary_reply_text(response: Any) -> str:
  131. """The first choice's message content with any ```fence``` unwrapped ("" when unreadable)."""
  132. try:
  133. message = getattr(response.choices[0], "message", None)
  134. content = message.get("content") if isinstance(message, dict) else getattr(message, "content", "")
  135. except Exception:
  136. return ""
  137. clean = str(content or "").strip()
  138. fence = re.fullmatch(r"```(?:[A-Za-z0-9_-]+)?\s*(.*?)\s*```", clean, flags=re.DOTALL)
  139. return fence.group(1).strip() if fence else clean
  140. _TAG_REWRITE_RULES = (
  141. "Rules:\n"
  142. "- Preserve the spoken words, order, and meaning.\n"
  143. "- Do not add new spoken sentences or remove existing spoken words.\n"
  144. )
  145. _TAG_REWRITE_TAIL = "- Do not explain or comment.\n- Return only the tagged TTS script."
  146. def _rewrite_with_auxiliary_model(
  147. system_prompt: str, user_prompt: str, fallback: str, *, label: str, fallback_label: str, level: int,
  148. ) -> str:
  149. """Ask the auxiliary model (task ``tts_audio_tags``) to rewrite a script; *fallback* on any failure/empty reply."""
  150. try:
  151. from agent.auxiliary_client import call_llm
  152. response = call_llm(
  153. task=GEMINI_AUDIO_TAG_REWRITE_TASK, temperature=0.7,
  154. messages=[{"role": "system", "content": system_prompt},
  155. {"role": "user", "content": user_prompt}])
  156. return _auxiliary_reply_text(response) or fallback
  157. except Exception as exc:
  158. logger.log(level, "%s audio tag rewrite failed; using %s: %s", label, fallback_label, exc)
  159. return fallback
  160. # --- Edge TTS (free default) ---
  161. async def _generate_edge_tts(text: str, output_path: str, tts_config: dict[str, Any]) -> str:
  162. edge_tts = _origin()._import_edge_tts()
  163. edge_config = tts_config.get("edge") or {}
  164. speed = float(edge_config.get("speed", tts_config.get("speed", 1.0)))
  165. kwargs = {"voice": edge_config.get("voice", DEFAULT_EDGE_VOICE)}
  166. if speed != 1.0:
  167. kwargs["rate"] = f"{round((speed - 1.0) * 100):+d}%"
  168. await edge_tts.Communicate(text, **kwargs).save(output_path)
  169. return output_path
  170. # --- ElevenLabs ---
  171. def _elevenlabs_environment_kwargs(el_config: dict[str, Any]) -> dict[str, Any]:
  172. """SDK client kwargs for ``tts.elevenlabs.base_url``/``wss_url``; empty (SDK default) without a
  173. base_url. ``wss_url`` defaults to the base_url host with a ``ws(s)://`` scheme."""
  174. base_url = (el_config.get("base_url") or "").rstrip("/")
  175. if not base_url:
  176. return {}
  177. from elevenlabs.environment import ElevenLabsEnvironment
  178. wss_url = (el_config.get("wss_url") or "").rstrip("/") or re.sub(r"^http", "ws", base_url)
  179. return {"environment": ElevenLabsEnvironment(base=base_url, wss=wss_url)}
  180. def _generate_elevenlabs(text: str, output_path: str, tts_config: dict[str, Any]) -> str:
  181. api_key = _require_key("ELEVENLABS_API_KEY", "elevenlabs", "Get one at https://elevenlabs.io/")
  182. el_config = tts_config.get("elevenlabs") or {}
  183. client = _origin()._import_elevenlabs()(api_key=api_key, **_elevenlabs_environment_kwargs(el_config))
  184. audio_generator = client.text_to_speech.convert(
  185. text=text, voice_id=el_config.get("voice_id", DEFAULT_ELEVENLABS_VOICE_ID),
  186. model_id=el_config.get("model_id", DEFAULT_ELEVENLABS_MODEL_ID),
  187. output_format="opus_48000_64" if output_path.endswith(".ogg") else "mp3_44100_128")
  188. with open(output_path, "wb") as f:
  189. f.writelines(audio_generator)
  190. return output_path
  191. # --- xAI TTS (dedicated /v1/tts endpoint, not the OpenAI audio shape) ---
  192. _XAI_INLINE_SPEECH_TAGS = (
  193. "pause", "long-pause", "hum-tune", "laugh", "chuckle", "giggle", "cry", "tsk",
  194. "tongue-click", "lip-smack", "breath", "inhale", "exhale", "sigh")
  195. _XAI_WRAPPING_SPEECH_TAGS = (
  196. "soft", "whisper", "loud", "build-intensity", "decrease-intensity", "higher-pitch",
  197. "lower-pitch", "slow", "fast", "sing-song", "singing", "laugh-speak", "emphasis")
  198. _XAI_SPEECH_TAG_RE = re.compile(
  199. rf"(\[(?:{'|'.join(_XAI_INLINE_SPEECH_TAGS)})\]|</?(?:{'|'.join(_XAI_WRAPPING_SPEECH_TAGS)})>)",
  200. flags=re.IGNORECASE)
  201. _XAI_FIRST_SENTENCE_RE = re.compile(r"^(.{12,120}?[.!?…])\s+(?=\S)", flags=re.DOTALL)
  202. def _apply_xai_auto_speech_tags(text: str) -> str:
  203. """Add xAI speech tags: a conservative local pass ([pause] between paragraphs / after the first
  204. sentence), then — only when the text carried no explicit tags — an auxiliary-model rewrite
  205. with the richer xAI tag set, falling back to the locally tagged text on any failure."""
  206. clean = text.strip()
  207. if not clean:
  208. return text
  209. local = re.sub(r"\s*\n\s*", " ", re.sub(r"\n\s*\n+", " [pause] ", clean))
  210. if not _XAI_SPEECH_TAG_RE.search(local):
  211. local = _XAI_FIRST_SENTENCE_RE.sub(r"\1 [pause] ", local, count=1)
  212. local = re.sub(r"\s{2,}", " ", local).strip()
  213. if _XAI_SPEECH_TAG_RE.search(clean): # explicit user/model tags are trusted as-is
  214. return local
  215. system_prompt = (
  216. "You rewrite transcripts for the xAI /v1/tts endpoint by inserting "
  217. "expressive speech tags.\n\n"
  218. "Valid inline tags (use as `[tag]`): " + ", ".join(_XAI_INLINE_SPEECH_TAGS) + ".\n"
  219. "Valid wrapping tags (use as `[tag]...[/tag]`): " + ", ".join(_XAI_WRAPPING_SPEECH_TAGS) + ".\n\n"
  220. + _TAG_REWRITE_RULES +
  221. "- Use inline `[tag]` for short modifiers (laughs, sighs, pause, etc.).\n"
  222. "- Use wrapping `[tag]...[/tag]` for sustained effects (whisper, soft, slow, fast, loud, etc.).\n"
  223. "- Do not use angle-bracket tags like `<tag>...</tag>` — xAI uses BBCode-style closing tags with `[/tag]`.\n"
  224. "- Do not use SSML.\n"
  225. + _TAG_REWRITE_TAIL)
  226. return _rewrite_with_auxiliary_model(
  227. system_prompt, f"TRANSCRIPT TO TAG:\n{local}", local, label="xAI TTS", fallback_label="locally-tagged text", level=logging.DEBUG,
  228. )
  229. def _clamped_number(raw: Any, cast, lo, hi):
  230. """Parse an optional numeric knob and clamp into [lo, hi]; ``None``/unparseable -> None. An empty
  231. string is deliberately clamped unconverted (its TypeError surfaces as a generic TTS failure)."""
  232. if raw is None:
  233. return None
  234. if raw != "":
  235. try:
  236. raw = cast(raw)
  237. except (TypeError, ValueError):
  238. return None
  239. return max(lo, min(hi, raw))
  240. def _generate_xai_tts(text: str, output_path: str, tts_config: dict[str, Any]) -> str:
  241. from tools.xai_http import resolve_xai_http_credentials
  242. # TTS is API-billed: a subscription OAuth bearer can authorize chat while
  243. # returning 403 for /v1/tts, so prefer an explicit XAI_API_KEY over OAuth.
  244. # See #87045, #88040.
  245. creds = resolve_xai_http_credentials(prefer_api_key=True)
  246. api_key = str(creds.get("api_key") or "").strip()
  247. if not api_key:
  248. raise ValueError("No xAI credentials found. Configure xAI OAuth in `hermes model` or set XAI_API_KEY.")
  249. xai_config = tts_config.get("xai") or {}
  250. voice_id = str(xai_config.get("voice_id", DEFAULT_XAI_VOICE_ID)).strip() or DEFAULT_XAI_VOICE_ID
  251. language = str(xai_config.get("language", DEFAULT_XAI_LANGUAGE)).strip() or DEFAULT_XAI_LANGUAGE
  252. sample_rate, bit_rate = (int(xai_config.get("sample_rate", DEFAULT_XAI_SAMPLE_RATE)),
  253. int(xai_config.get("bit_rate", DEFAULT_XAI_BIT_RATE)))
  254. auto_speech_tags = xai_config.get("auto_speech_tags", xai_config.get("speech_tags"))
  255. if _config_bool(auto_speech_tags, DEFAULT_XAI_AUTO_SPEECH_TAGS):
  256. text = _apply_xai_auto_speech_tags(text)
  257. # ``tts.xai.<knob>`` overrides global ``tts.<knob>``; out-of-range values are clamped into the
  258. # API's band rather than 400ing the request.
  259. speed = _clamped_number(xai_config.get("speed", tts_config.get("speed")), float,
  260. DEFAULT_XAI_SPEED_MIN, DEFAULT_XAI_SPEED_MAX)
  261. optimize_streaming_latency = _clamped_number(
  262. xai_config.get("optimize_streaming_latency", tts_config.get("optimize_streaming_latency")),
  263. int, 0, 2)
  264. text_normalization = _config_bool(
  265. xai_config.get("text_normalization"), DEFAULT_XAI_TEXT_NORMALIZATION_DEFAULT)
  266. if creds.get("provider") == "xai-oauth":
  267. base_url = creds.get("base_url")
  268. else:
  269. from hermes_cli.config import get_env_value
  270. base_url = xai_config.get("base_url") or creds.get("base_url") or get_env_value("XAI_BASE_URL")
  271. base_url = str(base_url or DEFAULT_XAI_BASE_URL).strip().rstrip("/")
  272. # Documented minimal POST /v1/tts shape; optional fields only when they differ from defaults.
  273. codec = "wav" if output_path.endswith(".wav") else "mp3"
  274. payload: dict[str, Any] = {"text": text, "voice_id": voice_id, "language": language}
  275. if codec != "mp3" or sample_rate != DEFAULT_XAI_SAMPLE_RATE or bit_rate != DEFAULT_XAI_BIT_RATE:
  276. output_format: dict[str, Any] = {"codec": codec}
  277. if sample_rate:
  278. output_format["sample_rate"] = sample_rate
  279. if codec == "mp3" and bit_rate:
  280. output_format["bit_rate"] = bit_rate
  281. payload["output_format"] = output_format
  282. if speed is not None and speed != DEFAULT_XAI_SPEED_DEFAULT:
  283. payload["speed"] = speed
  284. if optimize_streaming_latency not in (None, DEFAULT_XAI_OPTIMIZE_STREAMING_LATENCY_DEFAULT):
  285. payload["optimize_streaming_latency"] = optimize_streaming_latency
  286. if text_normalization:
  287. payload["text_normalization"] = True
  288. response = _post_json(f"{base_url}/tts", payload, {
  289. "Authorization": f"Bearer {api_key}", "Content-Type": "application/json",
  290. "User-Agent": hermes_xai_user_agent()})
  291. response.raise_for_status()
  292. return _write_bytes(output_path, _read_tts_response_bytes(response, label="xAI TTS"))
  293. # --- MiniMax TTS ---
  294. @dataclass(frozen=True)
  295. class _MiniMaxTTSRuntime:
  296. """A region-bound MiniMax endpoint and credential (key excluded from ``repr``)."""
  297. region: str
  298. endpoint: str
  299. credential_source: str
  300. api_key: str = field(repr=False)
  301. _MINIMAX_ENDPOINTS = {"global": DEFAULT_MINIMAX_BASE_URL, "cn": DEFAULT_MINIMAX_CN_BASE_URL}
  302. _MINIMAX_OFFICIAL_HOSTS = {
  303. "global": frozenset({"api.minimax.io", "api.minimax.chat"}),
  304. "cn": frozenset({"api.minimaxi.com"})}
  305. def _resolve_minimax_tts_runtime(tts_config: dict[str, Any]) -> _MiniMaxTTSRuntime:
  306. """Select MiniMax region, endpoint and credential atomically: explicit ``tts.minimax.region`` wins,
  307. else the legacy global credential; ``cn`` only when it is the sole configured credential."""
  308. mm_config = _section(tts_config, "minimax")
  309. resolve_key = _origin()._resolve_provider_key
  310. credentials = {
  311. region: (env_var, str(resolve_key(env_var, "minimax") or "").strip())
  312. for region, env_var in (("global", "MINIMAX_API_KEY"), ("cn", "MINIMAX_CN_API_KEY"))}
  313. region = str(mm_config.get("region") or "").strip().lower()
  314. if region and region not in _MINIMAX_ENDPOINTS:
  315. raise ValueError("tts.minimax.region must be 'global' or 'cn'")
  316. if not region:
  317. region = "cn" if credentials["cn"][1] and not credentials["global"][1] else "global"
  318. credential_source, api_key = credentials[region]
  319. if not api_key:
  320. raise ValueError(f"{credential_source} not set for MiniMax TTS region {region!r}")
  321. endpoint = str(mm_config.get("base_url") or _MINIMAX_ENDPOINTS[region]).strip()
  322. other_region = "cn" if region == "global" else "global"
  323. if (urlparse(endpoint).hostname or "").lower() in _MINIMAX_OFFICIAL_HOSTS[other_region]:
  324. raise ValueError(
  325. f"tts.minimax.base_url points to the {other_region!r} MiniMax endpoint but region is {region!r}")
  326. return _MiniMaxTTSRuntime(region=region, endpoint=endpoint, credential_source=credential_source, api_key=api_key)
  327. def _raise_minimax_api_error(result: dict[str, Any]) -> None:
  328. base_resp = result.get("base_resp", {})
  329. status_code = base_resp.get("status_code", -1)
  330. if status_code != 0:
  331. raise RuntimeError(
  332. f"MiniMax TTS API error (code {status_code}): {base_resp.get('status_msg', 'unknown error')}")
  333. def _generate_minimax_tts(text: str, output_path: str, tts_config: dict[str, Any]) -> str:
  334. """Generate audio via MiniMax: ``t2a_v2`` (nested payload, JSON reply with hex audio) or the legacy
  335. ``text_to_speech`` endpoint (flat payload, raw ``audio/*`` body), detected from the URL."""
  336. runtime = _resolve_minimax_tts_runtime(tts_config)
  337. mm_config = _section(tts_config, "minimax")
  338. model = mm_config.get("model", DEFAULT_MINIMAX_MODEL)
  339. voice_id = mm_config.get("voice_id", DEFAULT_MINIMAX_VOICE_ID)
  340. base_url = runtime.endpoint
  341. # MiniMax scopes TTS requests by GroupId (``?GroupId=<id>`` on the t2a_v2 URL): config or
  342. # MINIMAX_GROUP_ID, attached only when absent from the URL.
  343. from hermes_cli.config import get_env_value
  344. group_id = (str(mm_config.get("group_id") or "").strip()
  345. or (get_env_value("MINIMAX_GROUP_ID") or "").strip())
  346. if group_id and "GroupId=" not in base_url:
  347. base_url = f"{base_url}{'&' if '?' in base_url else '?'}GroupId={group_id}"
  348. is_t2a_v2 = "t2a_v2" in base_url
  349. if is_t2a_v2:
  350. payload = {
  351. "model": model, "text": text,
  352. "voice_setting": {
  353. "voice_id": voice_id, "speed": mm_config.get("speed", 1.0), "vol": mm_config.get("vol", 1.0),
  354. "pitch": mm_config.get("pitch", 0), "emotion": mm_config.get("emotion", "neutral"),
  355. },
  356. "audio_setting": {
  357. "sample_rate": mm_config.get("sample_rate", 32000), "bitrate": mm_config.get("bitrate", 128000),
  358. "format": "mp3", "channel": 1,
  359. },
  360. }
  361. else:
  362. payload = {"model": model, "text": text, "voice_id": voice_id}
  363. response = _post_json(base_url, payload, {
  364. "Content-Type": "application/json", "Authorization": f"Bearer {runtime.api_key}"})
  365. if is_t2a_v2:
  366. response.raise_for_status()
  367. result = _read_tts_response_json(response, label="MiniMax TTS")
  368. _raise_minimax_api_error(result)
  369. hex_audio = result.get("data", {}).get("audio", "")
  370. if not hex_audio:
  371. raise RuntimeError("MiniMax TTS returned empty audio data")
  372. return _write_bytes(output_path, bytes.fromhex(hex_audio))
  373. content_type = response.headers.get("Content-Type", "")
  374. if "audio/" in content_type:
  375. return _write_bytes(output_path, _read_tts_response_bytes(response, label="MiniMax TTS"))
  376. # Non-audio reply: surface the API error if the body is JSON.
  377. raw_body = b""
  378. try:
  379. raw_body = _read_tts_response_bytes(response, label="MiniMax TTS")
  380. _raise_minimax_api_error(json.loads(raw_body.decode("utf-8")) if raw_body else {})
  381. except (json.JSONDecodeError, UnicodeDecodeError, TypeError):
  382. response.raise_for_status()
  383. raise RuntimeError(
  384. f"MiniMax TTS returned unexpected Content-Type '{content_type}' ({len(raw_body)} bytes)")
  385. raise RuntimeError("MiniMax TTS returned no audio data")
  386. # --- Mistral (Voxtral TTS) — base64 audio, native Opus for voice bubbles ---
  387. def _generate_mistral_tts(text: str, output_path: str, tts_config: dict[str, Any]) -> str:
  388. api_key = _require_key("MISTRAL_API_KEY", "mistral", "Get one at https://console.mistral.ai/")
  389. mi_config = tts_config.get("mistral") or {}
  390. client_kwargs: dict[str, Any] = {"api_key": api_key}
  391. if mi_config.get("base_url"):
  392. client_kwargs["server_url"] = mi_config["base_url"] # the Mistral SDK calls it server_url
  393. Mistral = _origin()._import_mistral_client() # ImportError must escape the RuntimeError wrap
  394. try:
  395. with Mistral(**client_kwargs) as client:
  396. response = client.audio.speech.complete(
  397. model=mi_config.get("model", DEFAULT_MISTRAL_TTS_MODEL), input=text,
  398. voice_id=mi_config.get("voice_id") or DEFAULT_MISTRAL_TTS_VOICE_ID,
  399. response_format=_tts_response_format_from_path(output_path))
  400. audio_bytes = base64.b64decode(response.audio_data)
  401. except ValueError:
  402. raise
  403. except Exception as e:
  404. logger.error("Mistral TTS failed: %s", e, exc_info=True)
  405. raise RuntimeError(f"Mistral TTS failed: {type(e).__name__}") from e
  406. return _write_bytes(output_path, audio_bytes)
  407. # --- Google Gemini TTS ---
  408. def _read_gemini_persona_prompt(gemini_config: dict[str, Any]) -> str:
  409. """Read ``tts.gemini.persona_prompt_file`` (relative -> under HERMES_HOME), failing soft."""
  410. raw = gemini_config.get("persona_prompt_file")
  411. if not isinstance(raw, str) or not raw.strip():
  412. return ""
  413. path = Path(os.path.expandvars(raw.strip())).expanduser()
  414. if not path.is_absolute():
  415. try:
  416. from hermes_constants import get_hermes_home
  417. path = get_hermes_home() / path
  418. except Exception:
  419. path = Path.cwd() / path
  420. try:
  421. return path.read_text(encoding="utf-8-sig").strip()
  422. except (OSError, UnicodeDecodeError) as exc:
  423. logger.warning("Gemini TTS persona prompt file unavailable at %s: %s", path, exc)
  424. return ""
  425. def _gemini_audio_tags_enabled(gemini_config: dict[str, Any], model: str) -> bool:
  426. """Audio tags are opt-in and only Gemini 3.1 TTS models are known to honor them."""
  427. raw = gemini_config.get("audio_tags")
  428. if isinstance(raw, dict):
  429. raw = raw.get("enabled")
  430. if not _config_bool(raw, default=DEFAULT_GEMINI_AUDIO_TAGS):
  431. return False
  432. normalized = (model or "").strip().lower().rsplit("/", 1)[-1]
  433. if "gemini-3.1" in normalized and "tts" in normalized:
  434. return True
  435. logger.warning("Gemini TTS audio_tags enabled, but model %s is not known to support "
  436. "Gemini audio tags; skipping hidden tag rewrite", model)
  437. return False
  438. def _rewrite_gemini_tts_audio_tags(text: str, persona_prompt: str = "") -> str:
  439. """Use the configured auxiliary model to insert Gemini audio tags (falls back to *text*)."""
  440. transcript = text.strip()
  441. if not transcript:
  442. return text
  443. system_prompt = (
  444. "You rewrite transcripts for Gemini 3.1 Flash TTS by inserting expressive "
  445. "audio tags.\n\n"
  446. "Audio tags are inline square-bracket modifiers such as [whispers], "
  447. "[excitedly], [very slow], [sarcastically], [laughs], [sighs], or [gasp]. "
  448. "There is no fixed allowlist. Use creative freeform tags generously but "
  449. "naturally to control tone, pace, emotional vibe, emphasis, section-level "
  450. "delivery, and non-verbal sounds. Use English audio tags even when the "
  451. "spoken transcript is not English.\n\n"
  452. + _TAG_REWRITE_RULES +
  453. "- Use square brackets for every audio tag.\n"
  454. "- Do not use SSML or XML tags.\n"
  455. + _TAG_REWRITE_TAIL)
  456. user_prompt = (f"PERSONA AND DIRECTOR CONTEXT:\n{persona_prompt.strip() or '(none)'}\n\n"
  457. f"TRANSCRIPT TO TAG:\n{transcript}")
  458. return _rewrite_with_auxiliary_model(system_prompt, user_prompt, text, label="Gemini TTS",
  459. fallback_label="untagged text", level=logging.WARNING)
  460. def _compose_gemini_tts_prompt(text: str, gemini_config: dict[str, Any], persona_prompt: Optional[str] = None) -> str:
  461. """Gemini prompt = persona direction + transcript; a ``{transcript}`` / ``{{transcript}}``
  462. placeholder is substituted in place, otherwise the transcript is appended under a heading."""
  463. transcript = text.strip()
  464. if persona_prompt is None:
  465. persona_prompt = _read_gemini_persona_prompt(gemini_config)
  466. if not persona_prompt:
  467. return transcript
  468. preamble = (
  469. "Synthesize speech from the TRANSCRIPT only. Treat AUDIO PROFILE, "
  470. "SCENE, DIRECTOR'S NOTES, and SAMPLE CONTEXT as performance direction; "
  471. "do not speak those sections aloud.")
  472. for pattern in (r"\{\{\s*transcript\s*\}\}", r"\{\s*transcript\s*\}"):
  473. compiled = re.compile(pattern, flags=re.IGNORECASE)
  474. if compiled.search(persona_prompt):
  475. return f"{preamble}\n\n{compiled.sub(transcript, persona_prompt)}".strip()
  476. return f"{preamble}\n\n{persona_prompt}\n\n#### TRANSCRIPT\n{transcript}".strip()
  477. def _gemini_error_detail(response: Any) -> str:
  478. """Best-effort ``error.message`` from a non-200 Gemini reply, else the first 300 body chars."""
  479. raw_body = _read_tts_response_bytes(response, label="Gemini TTS")
  480. try:
  481. message = _parse_json_body(response, raw_body).get("error", {}).get("message")
  482. except Exception:
  483. message = None
  484. return message or raw_body.decode("utf-8", errors="replace")[:300]
  485. def _generate_gemini_tts(text: str, output_path: str, tts_config: dict[str, Any]) -> str:
  486. """Generate audio via Gemini ``generateContent`` (``responseModalities=["AUDIO"]``). The reply is
  487. base64 24kHz mono 16-bit PCM, wrapped as WAV and ffmpeg-converted to the requested container."""
  488. origin = _origin()
  489. api_key = origin._resolve_provider_key("GEMINI_API_KEY", "gemini") or origin._resolve_provider_key(
  490. "GOOGLE_API_KEY", "gemini")
  491. if not api_key:
  492. raise ValueError("GEMINI_API_KEY not set. Get one at https://aistudio.google.com/app/apikey")
  493. gemini_config = _section(tts_config, "gemini")
  494. model = str(gemini_config.get("model", DEFAULT_GEMINI_TTS_MODEL)).strip() or DEFAULT_GEMINI_TTS_MODEL
  495. voice = str(gemini_config.get("voice", DEFAULT_GEMINI_TTS_VOICE)).strip() or DEFAULT_GEMINI_TTS_VOICE
  496. from hermes_cli.config import get_env_value
  497. from agent.gemini_native_adapter import normalize_gemini_base_url
  498. base_url = normalize_gemini_base_url(
  499. gemini_config.get("base_url") or get_env_value("GEMINI_BASE_URL") or DEFAULT_GEMINI_TTS_BASE_URL,
  500. )
  501. persona_prompt = _read_gemini_persona_prompt(gemini_config)
  502. tts_script = text
  503. if _gemini_audio_tags_enabled(gemini_config, model):
  504. tts_script = _rewrite_gemini_tts_audio_tags(text, persona_prompt=persona_prompt)
  505. prompt_text = _compose_gemini_tts_prompt(
  506. tts_script, gemini_config, persona_prompt=persona_prompt)
  507. max_len = origin._resolve_max_text_length("gemini", tts_config)
  508. if len(prompt_text) > max_len:
  509. raise ValueError(
  510. "Gemini TTS composed prompt exceeds the provider request limit "
  511. f"({len(prompt_text)} > {max_len} chars). Reduce the persona/audio-tag "
  512. "prompt or lower tts.gemini.max_text_length so long-form text is "
  513. "split with enough prompt headroom.")
  514. payload: dict[str, Any] = {
  515. "contents": [{"parts": [{"text": prompt_text}]}],
  516. "generationConfig": {
  517. "responseModalities": ["AUDIO"],
  518. "speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": voice}}},
  519. },
  520. }
  521. # HTTP errors include request URLs, so credentials must stay in headers.
  522. headers = {"Content-Type": "application/json", "x-goog-api-key": api_key}
  523. if urlparse(base_url).hostname == "generativelanguage.googleapis.com":
  524. try:
  525. from hermes_cli.version_info import get_version_info
  526. headers["X-Goog-Api-Client"] = f"hermes-agent/{get_version_info().base_version}"
  527. except Exception:
  528. headers["X-Goog-Api-Client"] = "hermes-agent/0.0.0"
  529. response = _post_json(f"{base_url}/models/{model}:generateContent", payload, headers)
  530. if response.status_code != 200:
  531. raise RuntimeError(f"Gemini TTS API error (HTTP {response.status_code}): {_gemini_error_detail(response)}")
  532. try:
  533. data = _read_tts_response_json(response, label="Gemini TTS")
  534. parts = data["candidates"][0]["content"]["parts"]
  535. audio_part = next((p for p in parts if "inlineData" in p or "inline_data" in p), None)
  536. if audio_part is None:
  537. raise RuntimeError("Gemini TTS response contained no audio data")
  538. audio_b64 = (audio_part.get("inlineData") or audio_part.get("inline_data") or {}).get("data", "")
  539. except (KeyError, IndexError, TypeError) as e:
  540. raise RuntimeError(f"Gemini TTS response was malformed: {e}") from e
  541. if not audio_b64:
  542. raise RuntimeError("Gemini TTS returned empty audio data")
  543. return _write_wav_bytes_as(_wrap_pcm_as_wav(base64.b64decode(audio_b64)), output_path)