tts_tool.py 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747
  1. #!/usr/bin/env python3
  2. """Text-to-speech tool: config resolution, built-in provider dispatch, output policy, registration.
  3. Built-ins: Edge (free default), ElevenLabs, OpenAI, DeepInfra, MiniMax, Mistral, Gemini, xAI,
  4. local NeuTTS / KittenTTS / Piper; plus ``type: command`` providers under ``tts.providers.<name>``
  5. and plugin-registered ones. Output is Opus (.ogg) on voice-bubble platforms, MP3 elsewhere.
  6. Sibling ``tts_tool_*`` modules hold backends/delivery/lifecycle; they read the seams defined
  7. here (config, provider resolution, lazy SDK importers) through ``_origin()`` at call time.
  8. """
  9. from pm import install_hint
  10. import asyncio
  11. import contextlib
  12. import datetime
  13. import importlib.util
  14. import json
  15. import logging
  16. import os
  17. import re
  18. import tempfile
  19. import uuid
  20. from pathlib import Path
  21. from typing import Callable, Dict, Any, List, Optional
  22. import copy
  23. from hermes_constants import display_hermes_home
  24. logger = logging.getLogger(__name__)
  25. def _resolve_provider_key(env_var: str, provider_id: str) -> str:
  26. """Resolve a TTS provider API key via the shared voice-key resolver (config > env/.env > pool)."""
  27. from tools.tool_backend_helpers import resolve_provider_secret
  28. return resolve_provider_secret(env_var, provider_id)
  29. from tools.tts_command_provider import (
  30. BUILTIN_TTS_PROVIDERS, _configured_command_tts_output_path, _generate_command_tts,
  31. _get_command_tts_output_format, _is_command_tts_voice_compatible, _resolve_command_provider_config)
  32. from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER
  33. from tools.tts_tool_delivery import (
  34. _resolve_max_text_length, _build_audio_delivery_files, _convert_to_opus, _remove_quietly,
  35. _repair_ogg_container, _resolve_audio_delivery_profile, _split_text_for_tts)
  36. from tools.tts_tool_providers import (
  37. _generate_edge_tts, _generate_elevenlabs, _generate_gemini_tts, _generate_minimax_tts,
  38. _generate_mistral_tts, _generate_xai_tts, _resolve_minimax_tts_runtime)
  39. from tools.tts_tool_local import _generate_kittentts, _generate_neutts, _generate_piper_tts
  40. from tools.tts_tool_plugins import (
  41. _dispatch_to_plugin_provider, _plugin_provider_is_available,
  42. _plugin_provider_is_voice_compatible)
  43. from tools.tts_tool_openai import _generate_deepinfra_tts, _generate_openai_tts, _has_openai_audio_backend
  44. _PM_FEATURE_ALIASES = {"tts.edge": "edge-tts", "tts.elevenlabs": "tts-premium", "tts.mistral": "mistral"}
  45. # --- Lazy SDK importers -- providers import only when used (headless boxes lack PortAudio etc.) ---
  46. def _sdk_importer(module: str, attr: Optional[str] = None, feature: Optional[str] = None) -> Callable[[], Any]:
  47. """Lazy SDK importer: returns ``module`` (or ``module.attr``), raising ImportError when absent.
  48. ``feature`` names a ``pm.ensure_import`` extra to best-effort install first (users who enabled
  49. a provider in config.yaml never ran the post-setup hook); any failure there falls through so
  50. the raw import still raises cleanly. sounddevice also raises OSError without PortAudio."""
  51. def _import():
  52. if feature:
  53. with contextlib.suppress(Exception):
  54. from pm import ensure_import as _pm_ensure
  55. _pm_ensure(_PM_FEATURE_ALIASES.get(feature, feature))
  56. mod = importlib.import_module(module)
  57. return getattr(mod, attr) if attr else mod
  58. _import.__name__ = f"_import_{module.split('.')[0]}"
  59. return _import
  60. _import_edge_tts = _sdk_importer("edge_tts", feature="tts.edge")
  61. _import_elevenlabs = _sdk_importer("elevenlabs.client", "ElevenLabs", feature="tts.elevenlabs")
  62. _import_openai_client = _sdk_importer("openai", "OpenAI")
  63. _import_mistral_client = _sdk_importer("mistralai.client", "Mistral", feature="tts.mistral")
  64. _import_sounddevice = _sdk_importer("sounddevice")
  65. _import_kittentts = _sdk_importer("kittentts", "KittenTTS")
  66. _import_piper = _sdk_importer("piper", "PiperVoice") # piper-tts wheels embed espeak-ng
  67. def _importable(importer: Callable[[], Any]) -> bool:
  68. try:
  69. importer()
  70. return True
  71. except ImportError:
  72. return False
  73. def _package_installed(name: str) -> bool:
  74. try:
  75. return importlib.util.find_spec(name) is not None
  76. except Exception:
  77. return False
  78. def _check_neutts_available() -> bool: return _package_installed("neutts")
  79. def _check_kittentts_available() -> bool: return _package_installed("kittentts")
  80. def _check_piper_available() -> bool: return _package_installed("piper")
  81. # --- Defaults / config ---
  82. DEFAULT_PROVIDER = "edge"
  83. def _get_default_output_dir() -> str:
  84. from hermes_constants import get_hermes_dir
  85. return str(get_hermes_dir("cache/audio", "audio_cache"))
  86. DEFAULT_OUTPUT_DIR = _DEFAULT_OUTPUT_DIR_AT_IMPORT = _get_default_output_dir()
  87. def _default_output_dir() -> str:
  88. """The active profile's audio output dir at call time (long-lived runtimes switch profiles
  89. after import); a monkeypatched ``DEFAULT_OUTPUT_DIR`` wins.
  90. Same bug class as skills_tool (f8723c478) and skills_sync (#65828): long-lived multi-profile runtimes
  91. (dashboard console, TUI/Desktop backend, cron, kanban workers) import this module once under the launch
  92. HERMES_HOME and later scope requests to a different profile via
  93. ``hermes_constants.set_hermes_home_override()`` — a frozen module constant keeps writing synthesized
  94. audio into the launch profile's cache instead of the active profile's (#98749). Keep the legacy
  95. ``DEFAULT_OUTPUT_DIR`` module attribute for tests and external patchers; when it has not been patched,
  96. re-resolve from the live profile-scoped HERMES_HOME on every call.
  97. """
  98. if DEFAULT_OUTPUT_DIR != _DEFAULT_OUTPUT_DIR_AT_IMPORT:
  99. return DEFAULT_OUTPUT_DIR
  100. return _get_default_output_dir()
  101. def _load_tts_config() -> dict[str, Any]:
  102. """Return the ``tts`` config section ({} when unavailable)."""
  103. try:
  104. from hermes_cli.config import load_config
  105. return load_config().get("tts") or {}
  106. except ImportError:
  107. logger.debug("hermes_cli.config not available, using default TTS config")
  108. except Exception as e:
  109. logger.warning("Failed to load TTS config: %s", e, exc_info=True)
  110. return {}
  111. def _get_provider(tts_config: dict[str, Any]) -> str:
  112. """Configured provider or the free default (inference credentials never imply consent to paid
  113. speech); ``nous`` is serviced by the OpenAI path through the managed openai-audio gateway."""
  114. provider = (tts_config.get("provider") or DEFAULT_PROVIDER).lower().strip()
  115. return "openai" if provider == NOUS_MANAGED_PROVIDER else provider
  116. # Platforms whose native voice-bubble delivery requires Ogg/Opus (MP3 renders broken there).
  117. OPUS_VOICE_PLATFORMS = frozenset({"telegram", "matrix", "feishu", "whatsapp", "signal"})
  118. # MEDIA:<path> is a line-level gateway protocol. A filename containing an anchored media
  119. # directive forges a second attachment whenever the path is echoed into the tool result
  120. # (media_tag / file_path fields, error text): the collector scans producer output with a
  121. # bare MEDIA: matcher and cannot tell a filename from a directive. Mirrors the collector's
  122. # grammar (gateway.platforms.base.MEDIA_TAG_CLEANUP_RE): an anchored path OR a quoted payload,
  123. # which the collector accepts with no anchor and no extension.
  124. _MEDIA_DIRECTIVE_RE = re.compile(r"media:\s*[`'\"*_]*(?:[`'\"]|[a-z]:[/\\]|~?/)", re.IGNORECASE)
  125. # Built-ins that emit Opus natively when asked for .ogg; the rest need ffmpeg for voice bubbles.
  126. _NATIVE_OPUS_PROVIDERS = frozenset({"openai", "elevenlabs", "mistral", "gemini"})
  127. _FFMPEG_OPUS_PROVIDERS = frozenset({"edge", "neutts", "minimax", "xai", "kittentts", "piper"})
  128. # --- Built-in provider dispatch ---
  129. # provider -> (availability predicate or None, log label, generator name, "package missing" error).
  130. # Predicates/generator names resolve module globals at call time so test monkeypatches apply.
  131. _BUILTIN_DISPATCH: dict[str, tuple] = {
  132. "elevenlabs": (lambda: _importable(_import_elevenlabs), "ElevenLabs", "_generate_elevenlabs",
  133. "ElevenLabs provider selected but 'elevenlabs' package not installed. Run: "
  134. f"{install_hint('tts-premium')}"),
  135. "openai": (lambda: _importable(_import_openai_client), "OpenAI TTS", "_generate_openai_tts",
  136. "OpenAI provider selected but 'openai' package not installed."),
  137. "deepinfra": (lambda: _importable(_import_openai_client), "DeepInfra TTS", "_generate_deepinfra_tts",
  138. "DeepInfra TTS uses the 'openai' SDK but it isn't installed."),
  139. "minimax": (None, "MiniMax TTS", "_generate_minimax_tts", None),
  140. "xai": (None, "xAI TTS", "_generate_xai_tts", None),
  141. "mistral": (lambda: _importable(_import_mistral_client), "Mistral Voxtral TTS", "_generate_mistral_tts",
  142. "Mistral provider selected but 'mistralai' package not installed. "
  143. "Run `hermes setup` to install Mistral support."),
  144. "gemini": (None, "Google Gemini TTS", "_generate_gemini_tts", None),
  145. "neutts": (lambda: _check_neutts_available(), "NeuTTS (local)", "_generate_neutts",
  146. "NeuTTS provider selected but neutts is not installed. "
  147. "Run hermes setup tts and choose NeuTTS; espeak-ng is also required."),
  148. "kittentts": (lambda: _importable(_import_kittentts), "KittenTTS (local, ~25MB)", "_generate_kittentts",
  149. "KittenTTS provider selected but 'kittentts' package not installed. "
  150. "Run 'hermes setup tts' and choose KittenTTS."),
  151. "piper": (lambda: _importable(_import_piper), "Piper (local)", "_generate_piper_tts",
  152. "Piper provider selected but 'piper-tts' package not installed. "
  153. "Run 'hermes tools' and select Piper under TTS.")}
  154. def _error_json(message: str) -> str:
  155. return json.dumps({"success": False, "error": message}, ensure_ascii=False)
  156. def _run_edge_tts(text: str, file_str: str, tts_config: dict[str, Any]) -> None:
  157. """Run the async Edge generator from sync code (worker thread; direct run if that fails)."""
  158. run = lambda: asyncio.run(_generate_edge_tts(text, file_str, tts_config))
  159. try:
  160. from concurrent.futures import ThreadPoolExecutor
  161. with ThreadPoolExecutor(max_workers=1) as pool:
  162. pool.submit(run).result(timeout=60)
  163. except RuntimeError:
  164. run()
  165. def _select_builtin_engine(provider: str) -> tuple:
  166. """SDK check -> ``(engine, None)`` or ``(provider, error_json)``. Unknown names take the Edge
  167. default; without edge-tts NeuTTS is the fallback (engine != provider)."""
  168. entry = _BUILTIN_DISPATCH.get(provider)
  169. if entry is not None:
  170. available, _label, _generator, missing_error = entry
  171. return provider, (_error_json(missing_error) if available is not None and not available() else None)
  172. if _importable(_import_edge_tts):
  173. return provider, None # Edge default; the reported provider stays as configured
  174. if _check_neutts_available():
  175. logger.info("Edge TTS not available, falling back to NeuTTS (local)...")
  176. return "neutts", None
  177. return provider, _error_json(
  178. "No TTS provider available. Enable Edge TTS with: "
  179. f"{install_hint('edge-tts')} "
  180. "or run 'hermes setup tts' and choose NeuTTS for local synthesis.")
  181. def _synthesize_builtin(engine: str, text: str, file_str: str, tts_config: dict[str, Any], instructions: Optional[str]) -> None:
  182. """Run the already-selected built-in *engine*."""
  183. entry = _BUILTIN_DISPATCH.get(engine)
  184. logger.info("Generating speech with %s...", entry[1] if entry else "Edge TTS")
  185. if entry is None:
  186. _run_edge_tts(text, file_str, tts_config)
  187. elif engine == "openai":
  188. _generate_openai_tts(text, file_str, tts_config, instructions=instructions)
  189. else:
  190. globals()[entry[2]](text, file_str, tts_config)
  191. def _finalize_voice_delivery(
  192. file_str: str, provider: str, command_provider_config: Optional[dict[str, Any]], want_opus: bool,
  193. ) -> tuple:
  194. """Voice-bubble eligibility (Opus-converting when needed) -> ``(path, voice_compatible)``.
  195. Command/plugin providers are documents unless they opt in via ``voice_compatible``; native-Opus
  196. built-ins qualify when the platform wants Opus and they wrote .ogg; MP3/WAV built-ins are
  197. ffmpeg-converted only when the platform needs Opus."""
  198. if command_provider_config is not None:
  199. opted_in = _is_command_tts_voice_compatible(command_provider_config)
  200. elif provider not in BUILTIN_TTS_PROVIDERS:
  201. opted_in = _plugin_provider_is_voice_compatible(provider)
  202. elif want_opus and provider in _FFMPEG_OPUS_PROVIDERS and not file_str.endswith(".ogg"):
  203. opus_path = _convert_to_opus(file_str)
  204. return (opus_path, True) if opus_path else (file_str, False)
  205. else:
  206. native = provider in _NATIVE_OPUS_PROVIDERS
  207. return file_str, native and want_opus and file_str.endswith(".ogg")
  208. if not opted_in:
  209. return file_str, False
  210. # Plugin-registered provider (issue #30398). Voice-bubble delivery opts in via
  211. # ``TTSProvider.voice_compatible`` (mirrors the command-provider opt-in). Plugins that already write
  212. # Opus skip the ffmpeg conversion.
  213. if not file_str.endswith(".ogg"):
  214. file_str = _convert_to_opus(file_str) or file_str
  215. return file_str, file_str.endswith(".ogg")
  216. # --- Main tool function ---
  217. _once_warnings: set = set()
  218. def _warn_ignored_tts_provider_override(requested: str, configured: str) -> None:
  219. """Once-per-process (per distinct requested value) warning for ignored per-call provider
  220. overrides — a stale session whose cached tool schema still advertises the param would
  221. otherwise repeat it every call; and callers with no tts config at all are silent (nothing
  222. to disagree WITH — default resolution just runs). See #90109."""
  223. if not configured:
  224. return
  225. key = ("tts_provider_override", requested)
  226. if key in _once_warnings:
  227. return
  228. _once_warnings.add(key)
  229. logger.warning(
  230. "Ignoring per-call TTS provider override %r; tts.provider is %r",
  231. requested,
  232. configured,
  233. )
  234. def _apply_call_overrides(tts_config: dict[str, Any], speed: Optional[float], provider: Optional[str]):
  235. """Apply per-call ``speed`` (clamped, on a shallow copy so the cached config isn't mutated) and
  236. resolve the provider name. ``tts.provider`` in config.yaml is the authoritative backend
  237. selector (#90109): the per-call argument stays for internal/test callers, but a value that
  238. disagrees with the configured provider is ignored — the model-facing schema no longer
  239. advertises the override, and a leaked platform hint passing one anyway must not reroute
  240. speech to another vendor behind the operator's back."""
  241. if speed is not None:
  242. tts_config = {**tts_config, "speed": max(0.25, min(4.0, float(speed)))}
  243. configured_provider = _get_provider(tts_config)
  244. if provider:
  245. requested = provider.lower().strip()
  246. if requested and requested != configured_provider:
  247. _warn_ignored_tts_provider_override(requested, configured_provider)
  248. return tts_config, configured_provider
  249. # ElevenLabs voice ids are short alphanumeric tokens; anything else never reaches the URL path.
  250. _ELEVENLABS_VOICE_ID_RE = re.compile(r"^[A-Za-z0-9]{8,64}$")
  251. def elevenlabs_voice_selection_active(tts_config: Optional[dict[str, Any]] = None) -> bool:
  252. """Whether the model may pick voices: built-in ElevenLabs is the configured provider (gates
  253. the ``voice_id`` and ``find_voices`` schema fields, so other providers see neither)."""
  254. tts_config = _load_tts_config() if tts_config is None else tts_config
  255. return (_get_provider(tts_config) == "elevenlabs"
  256. and _resolve_command_provider_config("elevenlabs", tts_config) is None)
  257. def _apply_voice_override(tts_config: dict[str, Any], provider: str, voice_id: Optional[str]):
  258. """Per-call ElevenLabs ``voice_id`` on a copy of the config -> ``(tts_config, error_json)``.
  259. Ignored (logged) on any other provider: a schema cached before a provider switch may still
  260. send it, and it must not fail the speech."""
  261. voice_id = (voice_id or "").strip()
  262. if not voice_id:
  263. return tts_config, None
  264. if provider != "elevenlabs" or not elevenlabs_voice_selection_active(tts_config):
  265. logger.warning("Ignoring per-call TTS voice_id; tts.provider is %r", provider)
  266. return tts_config, None
  267. if not _ELEVENLABS_VOICE_ID_RE.match(voice_id):
  268. return tts_config, tool_error(
  269. f"Invalid ElevenLabs voice_id {voice_id!r}: use a voice_id returned by find_voices.",
  270. success=False)
  271. section = dict(tts_config.get("elevenlabs") or {})
  272. section["voice_id"] = voice_id
  273. return {**tts_config, "elevenlabs": section}, None
  274. def _find_voices(find: Any) -> str:
  275. """``text_to_speech(find_voices=...)``: one page of ElevenLabs voices to pick a voice_id from."""
  276. if not isinstance(find, dict):
  277. return tool_error("find_voices must be an object, e.g. {\"query\": \"orc\"}", success=False)
  278. tts_config = _load_tts_config()
  279. if not elevenlabs_voice_selection_active(tts_config):
  280. return tool_error("find_voices needs tts.provider: elevenlabs", success=False)
  281. api_key = _resolve_provider_key("ELEVENLABS_API_KEY", "elevenlabs")
  282. if not api_key:
  283. return tool_error("ELEVENLABS_API_KEY not set. Get one at https://elevenlabs.io/", success=False)
  284. from tools.tts_elevenlabs_voices import search_voices
  285. return search_voices(find, tts_config, api_key)
  286. def _session_platform() -> tuple:
  287. """``(platform, wants_opus)`` — platforms delivering voice bubbles only as Ogg/Opus want Opus."""
  288. from gateway.session_context import get_session_env
  289. platform = get_session_env("HERMES_SESSION_PLATFORM", "").lower()
  290. return platform, platform in OPUS_VOICE_PLATFORMS
  291. def _resolve_output_base(
  292. output_path: Optional[str], provider: str, command_provider_config: Optional[dict[str, Any]], want_opus: bool,
  293. ) -> tuple:
  294. """Pick the output file -> ``(Path, None)`` or ``(None, error_json)``.
  295. A caller path is rejected on ``..`` traversal (bug or prompt-injection; absolute is fine) and
  296. on protected credential/system locations. Default ``<audio cache>/tts_<timestamp>.<ext>``: the
  297. command format, ``.ogg`` for native-Opus providers on Opus platforms, else ``.mp3``."""
  298. if output_path:
  299. from tools.path_security import has_traversal_component, has_unsafe_path_chars
  300. if has_unsafe_path_chars(output_path):
  301. return None, _error_json(
  302. "output_path contains control characters or line separators; "
  303. "use a plain filesystem path")
  304. # Must precede the traversal/protected checks: their error text echoes the path,
  305. # and a MEDIA: substring in it would forge a delivery tag downstream.
  306. if _MEDIA_DIRECTIVE_RE.search(output_path):
  307. return None, _error_json(
  308. "output_path must not contain a media directive (MEDIA:<path>)")
  309. if has_traversal_component(output_path):
  310. return None, _error_json(
  311. f"output_path contains '..' traversal component: {output_path}. "
  312. "Use an absolute path or one relative to the current directory without '..'.")
  313. file_path = Path(output_path).expanduser()
  314. if command_provider_config is not None:
  315. file_path = _configured_command_tts_output_path(file_path, command_provider_config)
  316. from agent.file_safety import is_write_approval_required, is_write_denied
  317. if is_write_denied(str(file_path)) or is_write_approval_required(str(file_path)):
  318. return None, _error_json(
  319. f"output_path targets a protected credential or system path: "
  320. f"{file_path}. Choose a normal audio output location.")
  321. else:
  322. if command_provider_config is not None:
  323. ext = _get_command_tts_output_format(command_provider_config)
  324. else:
  325. ext = "ogg" if want_opus and provider in _NATIVE_OPUS_PROVIDERS else "mp3"
  326. timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
  327. file_path = Path(_default_output_dir()) / f"tts_{timestamp}.{ext}"
  328. file_path.parent.mkdir(parents=True, exist_ok=True)
  329. return file_path, None
  330. def _media_tag(paths: list[str], voice_compatible: bool) -> str:
  331. """``MEDIA:<path>`` lines; the ``[[audio_as_voice]]`` marker asks the platform for a voice bubble."""
  332. media_tag = "\n".join(f"MEDIA:{path}" for path in paths)
  333. return f"[[audio_as_voice]]\n{media_tag}" if voice_compatible else media_tag
  334. def _tool_failure(prefix: str, provider: str, exc: BaseException) -> str:
  335. """Log and wrap a synthesis failure as the standard error envelope (traceback except for config errors)."""
  336. error_msg = f"{prefix} ({provider}): {exc}"
  337. logger.error("%s", error_msg, exc_info=not isinstance(exc, ValueError))
  338. return tool_error(error_msg, success=False)
  339. def _text_to_speech_single(
  340. text: str, file_str: str, *, provider: str, tts_config: dict[str, Any],
  341. command_provider_config: Optional[dict[str, Any]], want_opus: bool, instructions: Optional[str],
  342. ) -> str:
  343. """Synthesize one provider-safe chunk into *file_str*; returns the result envelope.
  344. Command providers resolve BEFORE built-in dispatch, but built-in names short-circuit so
  345. ``tts.providers.openai.command`` can't shadow OpenAI. Plugins fire only for names that are
  346. neither; a None return falls through to built-in dispatch (unknown -> Edge default)."""
  347. try:
  348. if command_provider_config is not None:
  349. logger.info("Generating speech with command TTS provider '%s'...", provider)
  350. file_str = _generate_command_tts(
  351. text, file_str, provider, command_provider_config, tts_config)
  352. # Plugin-registered TTS backend (issue #30398). Fires when the configured provider is neither a
  353. # built-in nor a command-type entry, AND a plugin is registered under that name. The walrus binds
  354. # `_plugin_path` only when the dispatcher returns a path (i.e. a plugin was actually found); a None
  355. # return falls through to the built-in elif chain so unknown names hit the Edge TTS default at the
  356. # bottom. The dispatcher itself enforces built-ins-always-win + command-wins-over-plugin
  357. # defensively.
  358. elif provider not in BUILTIN_TTS_PROVIDERS and (
  359. _plugin_path := _dispatch_to_plugin_provider(text, file_str, provider, tts_config)
  360. ) is not None:
  361. file_str = _plugin_path
  362. else:
  363. provider, error = _select_builtin_engine(provider)
  364. if error:
  365. return error
  366. _synthesize_builtin(provider, text, file_str, tts_config, instructions)
  367. if not os.path.exists(file_str) or os.path.getsize(file_str) == 0:
  368. return _error_json(f"TTS generation produced no output (provider: {provider})")
  369. # Sniff once for every provider: MP3/WAV bytes in a .ogg path render as 0-second bubbles.
  370. file_str = _repair_ogg_container(file_str)
  371. file_str, voice_compatible = _finalize_voice_delivery(
  372. file_str, provider, command_provider_config, want_opus)
  373. logger.info("TTS audio saved: %s (%s bytes, provider: %s)", file_str, f"{os.path.getsize(file_str):,}", provider)
  374. return json.dumps({
  375. "success": True, "file_path": file_str, "media_tag": _media_tag([file_str], voice_compatible),
  376. "provider": provider, "voice_compatible": voice_compatible,
  377. }, ensure_ascii=False)
  378. except ValueError as e:
  379. return _tool_failure("TTS configuration error", provider, e)
  380. except FileNotFoundError as e:
  381. return _tool_failure("TTS dependency missing", provider, e)
  382. except Exception as e:
  383. return _tool_failure("TTS generation failed", provider, e)
  384. class _ChunkFailed(Exception):
  385. """One chunk's synthesis returned an error envelope; message is the final tool error text."""
  386. def _synthesize_chunks(chunks: list[str], base_path: Path, generated_artifacts: set, **single_kwargs) -> tuple:
  387. """Synthesize chunks into ``<base>.chunkNNN<ext>`` (or ``base`` alone) -> ``(encoded_paths, results)``.
  388. Every touched path lands in *generated_artifacts* for the caller's sweep. Raises
  389. :class:`_ChunkFailed` on a reported failure, ``RuntimeError`` on garbage or missing audio."""
  390. provider = single_kwargs["provider"]
  391. encoded_paths: list[str] = []
  392. chunk_results: list[dict[str, Any]] = []
  393. for index, chunk in enumerate(chunks, start=1):
  394. chunk_path = base_path
  395. if len(chunks) > 1:
  396. chunk_path = base_path.with_name(f"{base_path.stem}.chunk{index:03d}{base_path.suffix}")
  397. if os.path.lexists(chunk_path):
  398. # A file we didn't create: synthesize beside it so a failed attempt can neither truncate
  399. # nor sweep it; on success the delivery step os.replace()s the audio over it (overwrite).
  400. chunk_path = chunk_path.with_name(f".{chunk_path.stem}.{uuid.uuid4().hex}{chunk_path.suffix}")
  401. generated_artifacts.add(str(chunk_path))
  402. raw_result = _text_to_speech_single(chunk, str(chunk_path), **single_kwargs)
  403. try:
  404. chunk_result = json.loads(raw_result)
  405. except (json.JSONDecodeError, TypeError):
  406. raise RuntimeError(f"TTS chunk {index} returned invalid JSON: {str(raw_result)[:200]}")
  407. if not chunk_result.get("success"):
  408. error_msg = chunk_result.get("error", "unknown error")
  409. raise _ChunkFailed(f"TTS chunk {index} failed ({provider}): {error_msg}")
  410. actual_path = str(chunk_result.get("file_path") or chunk_path)
  411. if not os.path.isfile(actual_path) or os.path.getsize(actual_path) <= 0:
  412. raise RuntimeError(f"TTS chunk {index} produced no final audio: {actual_path}")
  413. generated_artifacts.add(actual_path)
  414. encoded_paths.append(actual_path)
  415. chunk_results.append(chunk_result)
  416. return encoded_paths, chunk_results
  417. def text_to_speech_tool(
  418. text: str, output_path: Optional[str] = None, speed: Optional[float] = None,
  419. instructions: Optional[str] = None, provider: Optional[str] = None,
  420. voice_id: Optional[str] = None, find_voices: Optional[dict[str, Any]] = None) -> str:
  421. """Convert text to speech with long-form chunking; returns the JSON result envelope.
  422. Text is normalized, split into provider-safe chunks (never silently truncated), synthesized
  423. sequentially, then packed against the platform's upload limit: a failed combine keeps the
  424. separate valid files and no over-limit artifact is ever returned. ``find_voices`` (ElevenLabs
  425. only) searches voices instead of speaking."""
  426. if find_voices is not None: # {} = browse with the defaults
  427. return _find_voices(find_voices)
  428. if not text or not text.strip():
  429. return tool_error("Text is required", success=False)
  430. try: # shared cleaner: markdown, emoji, think blocks, verifier footer, units, newlines
  431. from tools.tts_text_normalize import prepare_spoken_text
  432. text = prepare_spoken_text(text, max_chars=None)
  433. except Exception:
  434. text = text.strip()
  435. if not text:
  436. return tool_error("Text is empty after TTS cleanup", success=False)
  437. tts_config, provider = _apply_call_overrides(_load_tts_config(), speed, provider)
  438. tts_config, error = _apply_voice_override(tts_config, provider, voice_id)
  439. if error:
  440. return error
  441. command_provider_config = _resolve_command_provider_config(provider, tts_config)
  442. max_len = _resolve_max_text_length(provider, tts_config)
  443. chunks = _split_text_for_tts(text, max_len)
  444. if not chunks:
  445. return tool_error("Text is required", success=False)
  446. if len(chunks) > 1:
  447. logger.info("TTS text for provider %s split into %d chunks (input=%d chars, cap=%d)",
  448. provider, len(chunks), len(text), max_len)
  449. platform, want_opus = _session_platform()
  450. delivery_profile = _resolve_audio_delivery_profile(platform, tts_config)
  451. base_path, error = _resolve_output_base(
  452. output_path, provider, command_provider_config, want_opus)
  453. if error:
  454. return error
  455. generated_artifacts: set[str] = set()
  456. final_paths: list[str] = []
  457. try:
  458. encoded_paths, chunk_results = _synthesize_chunks(
  459. chunks, base_path, generated_artifacts, provider=provider, tts_config=tts_config,
  460. command_provider_config=command_provider_config, want_opus=want_opus,
  461. instructions=instructions)
  462. voice_compatible = bool(chunk_results) and all(bool(r.get("voice_compatible")) for r in chunk_results)
  463. delivery_base = base_path.with_suffix(Path(encoded_paths[0]).suffix)
  464. final_paths, combined_chunks = _build_audio_delivery_files(
  465. encoded_paths, str(delivery_base), delivery_profile, voice_compatible=voice_compatible)
  466. for path in final_paths:
  467. logger.info("TTS audio saved: %s (%s bytes, provider: %s)", path, f"{os.path.getsize(path):,}", provider)
  468. return json.dumps({
  469. "success": True, "file_path": final_paths[0], "file_paths": final_paths,
  470. "media_tag": _media_tag(final_paths, voice_compatible),
  471. "provider": chunk_results[0].get("provider", provider), "voice_compatible": voice_compatible,
  472. "chunk_count": len(chunks), "delivery_file_count": len(final_paths),
  473. "combined_chunks": bool(combined_chunks),
  474. "delivery_profile": {
  475. "platform": delivery_profile.platform, "max_file_bytes": delivery_profile.max_file_bytes,
  476. "target_file_bytes": delivery_profile.target_file_bytes},
  477. }, ensure_ascii=False)
  478. except _ChunkFailed as exc:
  479. return tool_error(str(exc), success=False)
  480. except ValueError as exc:
  481. return _tool_failure("TTS delivery error", provider, exc)
  482. except Exception as exc:
  483. return _tool_failure("TTS long-form generation failed", provider, exc)
  484. finally:
  485. final_absolute = {os.path.abspath(path) for path in final_paths}
  486. for artifact in generated_artifacts:
  487. if os.path.abspath(artifact) not in final_absolute:
  488. _remove_quietly(artifact)
  489. # --- check_fn ---
  490. def _minimax_requirements() -> bool:
  491. try:
  492. _resolve_minimax_tts_runtime(_load_tts_config())
  493. return True
  494. except ValueError:
  495. return False
  496. def _xai_requirements() -> bool:
  497. try:
  498. from tools.xai_http import resolve_xai_http_credentials
  499. # Same ordering as _generate_xai_tts / XAIStreamer: an explicit key wins over the
  500. # subscription OAuth bearer (which 403s on metered /v1/tts) — never touch the OAuth
  501. # pool for an availability probe when a key is configured. See #87045, #113727.
  502. return bool(resolve_xai_http_credentials(prefer_api_key=True).get("api_key"))
  503. except Exception:
  504. return False
  505. # Must mirror text_to_speech_tool dispatch: unrelated cloud credentials never make the Edge
  506. # default usable, and an explicit provider is checked on its own.
  507. #
  508. # PASSIVE ONLY: every entry answers from availability/credentials and never installs. The SDK
  509. # importers (`_import_edge_tts`/`_import_elevenlabs`/`_import_mistral_client`) call
  510. # ``pm.ensure_import`` on import, so reaching them from here turned ``check_tts_requirements``
  511. # — the ``text_to_speech`` tool's ``check_fn`` — into an installer that ran during every tool
  512. # listing.
  513. _BUILTIN_REQUIREMENTS: dict[str, Callable[[], bool]] = {
  514. "edge": lambda: _pm_extra_available("edge-tts") or _check_neutts_available(),
  515. "elevenlabs": lambda: _pm_extra_available("tts-premium") and bool(_resolve_provider_key("ELEVENLABS_API_KEY", "elevenlabs")),
  516. "openai": lambda: _package_installed("openai") and _has_openai_audio_backend(),
  517. "deepinfra": lambda: _package_installed("openai") and bool(_resolve_provider_key("DEEPINFRA_API_KEY", "deepinfra")),
  518. "minimax": _minimax_requirements,
  519. "xai": _xai_requirements,
  520. "gemini": lambda: bool(_resolve_provider_key("GEMINI_API_KEY", "gemini") or _resolve_provider_key("GOOGLE_API_KEY", "gemini")),
  521. "mistral": lambda: _pm_extra_available("mistral") and bool(_resolve_provider_key("MISTRAL_API_KEY", "mistral")),
  522. "neutts": lambda: _check_neutts_available(),
  523. "kittentts": lambda: _check_kittentts_available(),
  524. "piper": lambda: _check_piper_available()}
  525. def _pm_extra_available(extra: str) -> bool:
  526. """Whether the extra's anchor is importable, by pm's own answer for it.
  527. Not ``find_spec`` on a hand-written module name: ``pm.extras`` owns the extra→anchor table and
  528. counts a module already in ``sys.modules`` as installed, which is how every other feature in
  529. the tree reports availability.
  530. """
  531. try:
  532. from pm.extras import available
  533. except Exception:
  534. return False
  535. return bool(available(extra))
  536. # Providers whose SDK pm installs on first use: provider -> the credential it needs REGARDLESS of
  537. # the install (an install cannot conjure a key; None = none). Extra names come from
  538. # ``_PM_FEATURE_ALIASES`` (upstream's ``tts.<provider>`` ids), so that table stays their one
  539. # source. The install belongs to synthesis (``_select_builtin_engine`` and the command/streaming
  540. # paths), never to a requirement check — so a missing-but-installable SDK counts as READY here.
  541. _SDK_ON_DEMAND: dict[str, Optional[str]] = {
  542. "edge": None,
  543. "elevenlabs": "ELEVENLABS_API_KEY",
  544. "mistral": "MISTRAL_API_KEY"}
  545. def _ready_after_first_use_install(provider: str) -> bool:
  546. """True when the SDK is absent but pm may install it at first synthesis AND this machine
  547. could actually get it (platform gate open) AND the provider's own credential is present.
  548. Installs nothing itself."""
  549. if provider not in _SDK_ON_DEMAND:
  550. return False
  551. feature = _PM_FEATURE_ALIASES.get(f"tts.{provider}")
  552. if feature is None:
  553. return False
  554. key_env = _SDK_ON_DEMAND[provider]
  555. if key_env and not _resolve_provider_key(key_env, provider):
  556. return False
  557. try:
  558. from pm.install import lazy_installs_allowed
  559. from pm.extras import extra_supported
  560. except Exception:
  561. return False
  562. return extra_supported(feature) and bool(lazy_installs_allowed())
  563. def check_tts_requirements() -> bool:
  564. """Return whether the explicitly resolved TTS provider can run — now, or after the
  565. first-use SDK install that synthesis performs. This is the ``text_to_speech`` tool's
  566. ``check_fn``, so it runs on every tool listing and must never install."""
  567. tts_config = _load_tts_config()
  568. provider = _get_provider(tts_config)
  569. if _resolve_command_provider_config(provider, tts_config) is not None:
  570. return True
  571. check = _BUILTIN_REQUIREMENTS.get(provider)
  572. if check is not None:
  573. return bool(check()) or _ready_after_first_use_install(provider)
  574. return _plugin_provider_is_available(provider)
  575. # --- Registry ---
  576. from tools.registry import registry, tool_error
  577. def _output_path_description(home: str) -> str:
  578. return f"Optional custom file path to save the audio. Defaults to {home}/audio_cache/<timestamp>.mp3"
  579. def _tts_schema_overrides() -> dict:
  580. """Rebuild the ``output_path`` default hint from the ACTIVE profile at every get_definitions():
  581. the multiplexed gateway serves every profile from one process, so a path baked in at import
  582. would name the launch profile's home for everyone else (#95685)."""
  583. params = copy.deepcopy(TTS_SCHEMA["parameters"])
  584. params["properties"]["output_path"]["description"] = _output_path_description(display_hermes_home())
  585. if not elevenlabs_voice_selection_active():
  586. return {"parameters": params}
  587. from tools.tts_elevenlabs_voices import FIND_VOICES_SCHEMA
  588. params["properties"]["voice_id"] = {
  589. "type": "string",
  590. "description": ("ElevenLabs voice for this audio (a voice_id from find_voices). "
  591. "Omit to use the user's configured voice.")}
  592. params["properties"]["find_voices"] = copy.deepcopy(FIND_VOICES_SCHEMA)
  593. params["required"] = [] # a find_voices call carries no text
  594. return {"parameters": params, "description": TTS_SCHEMA["description"].replace(
  595. _CONFIGURED_VOICE_SENTENCE, _ELEVENLABS_VOICE_SENTENCE)}
  596. _CONFIGURED_VOICE_SENTENCE = "Voice and provider are user-configured (built-in providers like edge/openai or custom command providers under tts.providers.<name>), not model-selected."
  597. _ELEVENLABS_VOICE_SENTENCE = "The provider is user-configured (ElevenLabs); pick a voice per call with voice_id, found with find_voices (one character per voice, e.g. for game NPCs)."
  598. TTS_SCHEMA = {
  599. "name": "text_to_speech",
  600. "description": "Convert text to speech audio. Returns a MEDIA: path that the platform delivers as native audio. Compatible providers render as a voice bubble on Telegram; otherwise audio is sent as a regular attachment. In CLI mode, saves to ~/voice-memos/. " + _CONFIGURED_VOICE_SENTENCE,
  601. "parameters": {
  602. "type": "object",
  603. "properties": {
  604. "text": {
  605. "type": "string",
  606. "description": "The text to convert to speech. Provider-specific per-request character caps apply automatically (OpenAI 4096, xAI 15000, MiniMax 10000, ElevenLabs 5k-40k depending on model); longer input is split into ordered chunks without silent truncation."
  607. },
  608. "output_path": {
  609. "type": "string",
  610. "description": _output_path_description("the profile HERMES_HOME")
  611. },
  612. "speed": {
  613. "type": "number",
  614. "description": "Playback speed multiplier. 1.0 = normal, 0.5 = very slow (language learning), 2.0 = fast. Range: 0.25-4.0. Overrides the speed configured in config.yaml."
  615. },
  616. "instructions": {
  617. "type": "string",
  618. "description": (
  619. "Optional voice-design guidance: tone, emotion, pacing, accent, "
  620. "whispering, impressions (e.g. 'Speak in a cheerful, excited whisper'). "
  621. "Forwarded to the OpenAI backend (gpt-4o-mini-tts and OpenAI-compatible "
  622. "voice-design servers). Silently ignored by backends that don't support it."
  623. )
  624. }
  625. },
  626. "required": ["text"]
  627. }
  628. }
  629. registry.register(
  630. name="text_to_speech",
  631. toolset="tts",
  632. schema=TTS_SCHEMA,
  633. handler=lambda args, **kw: text_to_speech_tool(
  634. text=args.get("text", ""),
  635. **{k: args.get(k) for k in ("output_path", "speed", "instructions", "voice_id", "find_voices")}),
  636. check_fn=check_tts_requirements,
  637. emoji="🔊",
  638. dynamic_schema_overrides=_tts_schema_overrides)