tts_tool_local.py 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195
  1. """Local on-device TTS engines for ``tools.tts_tool``: NeuTTS, Piper, KittenTTS.
  2. All three synthesize WAV natively; :func:`_finalize_wav_output` converts/renames to the requested
  3. container. Piper and KittenTTS keep loaded models in small LRU caches registered in
  4. ``_LOCAL_TTS_MODEL_CACHES`` so warm/release can pre-load or drop them. ``_import_piper`` /
  5. ``_import_kittentts`` are resolved through the origin module at call time (test monkeypatches).
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import subprocess
  10. import sys
  11. from pathlib import Path
  12. from typing import Any, Callable, Dict, Tuple
  13. from tools.tts_tool_delivery import _finalize_wav_output, _origin, _section, _wav_sidecar_path
  14. logger = logging.getLogger("tools.tts_tool")
  15. DEFAULT_KITTENTTS_MODEL = "KittenML/kitten-tts-nano-0.8-int8" # 25MB
  16. DEFAULT_KITTENTTS_VOICE = "Jasper"
  17. DEFAULT_PIPER_VOICE = "en_US-lessac-medium" # balanced size/quality
  18. _NEUTTS_SAMPLES = Path(__file__).parent / "neutts_samples"
  19. # --- Bounded model caches ---
  20. # Each entry is a whole loaded model (tens of MB); unbounded, one would be pinned per distinct
  21. # voice for the process lifetime. Most sessions use one or two voices; a cold reload is cheap.
  22. _TTS_MODEL_CACHE_MAX = 3
  23. # Provider name -> the cache it populates (warm/release in tts_tool_lifecycle; a new local engine
  24. # adds a row here plus a loader in _local_tts_warmers()). Piper keyed on absolute .onnx path
  25. # (+cuda flag); KittenTTS on model name.
  26. _piper_voice_cache: dict[str, Any] = {}
  27. _kittentts_model_cache: dict[str, Any] = {}
  28. _LOCAL_TTS_MODEL_CACHES: dict[str, dict[str, Any]] = {
  29. "piper": _piper_voice_cache, "kittentts": _kittentts_model_cache}
  30. def _tts_cache_get_or_load(cache: dict[str, Any], key: str, load: Callable[[], Any]) -> Any:
  31. """Get ``key`` from ``cache`` or load it, LRU-bounded at ``_TTS_MODEL_CACHE_MAX`` (a hit refreshes
  32. recency via pop + reinsert; eviction only releases the slot, not live references)."""
  33. if key in cache:
  34. cache[key] = cache.pop(key)
  35. return cache[key]
  36. value = load()
  37. cache[key] = value
  38. while len(cache) > _TTS_MODEL_CACHE_MAX:
  39. cache.pop(next(iter(cache)), None)
  40. return value
  41. def _run_helper(cmd: list, timeout: int) -> subprocess.CompletedProcess:
  42. return subprocess.run(
  43. cmd, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout, stdin=subprocess.DEVNULL,
  44. check=False,
  45. )
  46. # --- NeuTTS (subprocess via tools/neutts_synth.py so the ~500MB model exits after use) ---
  47. def _generate_neutts(text: str, output_path: str, tts_config: dict[str, Any]) -> str:
  48. neutts_config = tts_config.get("neutts") or {}
  49. wav_path = _wav_sidecar_path(output_path)
  50. cmd = [
  51. sys.executable, str(Path(__file__).parent / "neutts_synth.py"),
  52. "--text", text,
  53. "--out", wav_path,
  54. "--ref-audio", neutts_config.get("ref_audio", "") or str(_NEUTTS_SAMPLES / "jo.wav"),
  55. "--ref-text", neutts_config.get("ref_text", "") or str(_NEUTTS_SAMPLES / "jo.txt"),
  56. "--model", neutts_config.get("model", "neuphonic/neutts-air-q4-gguf"),
  57. "--device", neutts_config.get("device", "cpu")]
  58. result = _run_helper(cmd, 120)
  59. if result.returncode != 0: # the synth script reports success lines as "OK:" on stderr too
  60. error_lines = [l for l in result.stderr.strip().splitlines() if not l.startswith("OK:")]
  61. raise RuntimeError(f"NeuTTS synthesis failed: {chr(10).join(error_lines) or 'unknown error'}")
  62. return _finalize_wav_output(wav_path, output_path)
  63. # --- Piper (local neural VITS, 44 languages) ---
  64. def _get_piper_voices_dir() -> Path:
  65. """``<HERMES_HOME>/cache/piper-voices/`` so voice downloads follow profile boundaries."""
  66. from hermes_constants import get_hermes_dir
  67. root = Path(get_hermes_dir("cache/piper-voices", "piper_voices_cache"))
  68. root.mkdir(parents=True, exist_ok=True)
  69. return root
  70. def _resolve_piper_voice_path(voice: str, download_dir: Path) -> str:
  71. """Resolve *voice* (an .onnx path or a name like ``en_US-lessac-medium``, downloaded into
  72. *download_dir* on first use) to a concrete .onnx file; RuntimeError when it can't be."""
  73. voice = voice or DEFAULT_PIPER_VOICE
  74. candidate = Path(voice).expanduser()
  75. if candidate.suffix.lower() == ".onnx" and candidate.exists():
  76. return str(candidate)
  77. cached = download_dir / f"{voice}.onnx"
  78. if cached.exists() and (download_dir / f"{voice}.onnx.json").exists():
  79. return str(cached)
  80. logger.info("[Piper] Downloading voice '%s' to %s (first use)", voice, download_dir)
  81. try:
  82. result = _run_helper(
  83. [sys.executable, "-m", "piper.download_voices", voice, "--download-dir", str(download_dir)], 300,
  84. )
  85. except subprocess.TimeoutExpired as exc:
  86. raise RuntimeError(f"Piper voice download timed out after 300s for '{voice}'") from exc
  87. if result.returncode != 0:
  88. stderr = (result.stderr or "").strip() or "no stderr output"
  89. raise RuntimeError(f"Piper voice download failed for '{voice}': {stderr[:400]}")
  90. if not cached.exists():
  91. raise RuntimeError(
  92. f"Piper voice download completed but {cached} is missing — "
  93. f"check voice name (see: https://github.com/OHF-Voice/piper1-gpl/"
  94. f"blob/main/docs/VOICES.md)")
  95. return str(cached)
  96. def _load_piper_voice_for_config(tts_config: dict[str, Any]) -> tuple[Any, dict[str, Any]]:
  97. """Resolve + load (or fetch from cache) the selected Piper voice -> ``(voice, piper_config)``.
  98. Shared by synthesis and ``warm_tts_provider`` so a warm-up fills exactly the slot synthesis hits."""
  99. PiperVoice = _origin()._import_piper()
  100. piper_config = _section(tts_config, "piper")
  101. voice_name = piper_config.get("voice") or DEFAULT_PIPER_VOICE
  102. download_dir = Path(piper_config.get("voices_dir") or _get_piper_voices_dir()).expanduser()
  103. download_dir.mkdir(parents=True, exist_ok=True)
  104. use_cuda = bool(piper_config.get("use_cuda", False))
  105. model_path = _resolve_piper_voice_path(voice_name, download_dir)
  106. def _load_piper_voice():
  107. logger.info("[Piper] Loading voice: %s", model_path)
  108. v = PiperVoice.load(model_path, use_cuda=use_cuda)
  109. logger.info("[Piper] Voice loaded")
  110. return v
  111. # speaker_id is applied per call via syn_config, so one instance serves every speaker.
  112. cache_key = f"{model_path}::cuda={use_cuda}"
  113. return _tts_cache_get_or_load(_piper_voice_cache, cache_key, _load_piper_voice), piper_config
  114. _PIPER_ADVANCED_KNOBS = ("length_scale", "noise_scale", "noise_w_scale", "volume", "normalize_audio", "speaker_id")
  115. def _generate_piper_tts(text: str, output_path: str, tts_config: dict[str, Any]) -> str:
  116. import wave
  117. voice, piper_config = _load_piper_voice_for_config(tts_config)
  118. # Bad speaker_id drops to 0 (Piper's default); bools are rejected (they'd coerce to 1/0).
  119. _raw_speaker = piper_config.get("speaker_id", 0)
  120. speaker_id = _raw_speaker if type(_raw_speaker) is int else 0
  121. # Only build a SynthesisConfig when an advanced knob is configured, so we don't depend on a
  122. # newer piper-tts than the user's unless we must.
  123. syn_config = None
  124. if any(k in piper_config for k in _PIPER_ADVANCED_KNOBS):
  125. try:
  126. from piper import SynthesisConfig # type: ignore
  127. syn_config = SynthesisConfig(
  128. length_scale=float(piper_config.get("length_scale", 1.0)),
  129. noise_scale=float(piper_config.get("noise_scale", 0.667)),
  130. noise_w_scale=float(piper_config.get("noise_w_scale", 0.8)),
  131. volume=float(piper_config.get("volume", 1.0)),
  132. normalize_audio=bool(piper_config.get("normalize_audio", True)),
  133. speaker_id=speaker_id)
  134. except ImportError:
  135. logger.warning("[Piper] SynthesisConfig not available in this piper-tts version — advanced knobs ignored")
  136. wav_path = _wav_sidecar_path(output_path)
  137. with wave.open(wav_path, "wb") as wav_file:
  138. if syn_config is not None:
  139. voice.synthesize_wav(text, wav_file, syn_config=syn_config)
  140. else:
  141. voice.synthesize_wav(text, wav_file)
  142. return _finalize_wav_output(wav_path, output_path)
  143. # --- KittenTTS (local ONNX, 25-80MB models, CPU only) ---
  144. def _load_kittentts_model_for_config(tts_config: dict[str, Any]) -> tuple[Any, dict[str, Any]]:
  145. """Load (or fetch from cache) the KittenTTS model; returns ``(model, kittentts_config)``."""
  146. KittenTTS = _origin()._import_kittentts()
  147. kt_config = _section(tts_config, "kittentts")
  148. model_name = kt_config.get("model", DEFAULT_KITTENTTS_MODEL)
  149. def _load_kittentts_model():
  150. logger.info("[KittenTTS] Loading model: %s", model_name)
  151. m = KittenTTS(model_name)
  152. logger.info("[KittenTTS] Model loaded successfully")
  153. return m
  154. return _tts_cache_get_or_load(_kittentts_model_cache, model_name, _load_kittentts_model), kt_config
  155. def _generate_kittentts(text: str, output_path: str, tts_config: dict[str, Any]) -> str:
  156. model, kt_config = _load_kittentts_model_for_config(tts_config)
  157. audio = model.generate( # numpy array at 24kHz
  158. text, voice=kt_config.get("voice", DEFAULT_KITTENTTS_VOICE),
  159. speed=kt_config.get("speed", 1.0), clean_text=kt_config.get("clean_text", True))
  160. import soundfile as sf
  161. wav_path = _wav_sidecar_path(output_path)
  162. sf.write(wav_path, audio, 24000)
  163. return _finalize_wav_output(wav_path, output_path)