transcription_local.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344
  1. """Local STT backends.
  2. faster-whisper loading (CUDA->CPU fallback, Apple Silicon pinning), the
  3. anti-hallucination transcribe kwargs and segment gate, and the local whisper CLI
  4. (``local_command``) provider. The cached-model singleton and idle-unload watcher
  5. stay in ``transcription_tools`` (module state) and are read from it lazily.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import os
  10. import platform
  11. import shlex
  12. import subprocess
  13. import tempfile
  14. import importlib.util as _ilu
  15. from pathlib import Path
  16. from typing import Any, Dict, Optional
  17. from tools.transcription_audio import _find_whisper_binary, _prepare_local_audio, _run_quiet
  18. from tools.transcription_common import (
  19. DEFAULT_LOCAL_MODEL, DEFAULT_LOCAL_STT_LANGUAGE, GROQ_MODELS, LOCAL_STT_COMMAND_ENV, RETIRED_GROQ_MODELS,
  20. OPENAI_MODELS, STT_MODEL_CATALOG, _config_number, _error_result, _log_prompt_unsupported, _ok_result,
  21. _process_error_detail)
  22. # Log-record parity with the origin module.
  23. logger = logging.getLogger("tools.transcription_tools")
  24. _LOCAL_LANGUAGE_ALIASES = {
  25. "繁體中文": "zh",
  26. "繁体中文": "zh",
  27. "简体中文": "zh",
  28. "簡體中文": "zh",
  29. }
  30. _THREE_LETTER_WHISPER_CODES = frozenset({"haw", "yue"})
  31. def _normalize_local_stt_language(
  32. language: Optional[str], supported_languages: object = None
  33. ) -> Optional[str]:
  34. """Return a Whisper language code, or None so the caller can fall back safely."""
  35. if not isinstance(language, str) or not language.strip():
  36. return None
  37. raw = language.strip()
  38. folded = raw.casefold().replace("_", "-")
  39. candidate = _LOCAL_LANGUAGE_ALIASES.get(raw, folded.split("-", 1)[0])
  40. code_shape_is_valid = len(candidate) == 2 or candidate in _THREE_LETTER_WHISPER_CODES
  41. if not (candidate.isascii() and candidate.isalpha() and code_shape_is_valid):
  42. logger.warning("Local STT language %r is not a language code; using fallback", raw)
  43. return None
  44. if isinstance(supported_languages, (list, tuple, set, frozenset)):
  45. supported_codes = {str(code).casefold() for code in supported_languages}
  46. if candidate not in supported_codes:
  47. logger.warning("Local STT language %r is unsupported; using fallback", raw)
  48. return None
  49. return candidate
  50. def _get_local_command_template() -> Optional[str]:
  51. configured = os.getenv(LOCAL_STT_COMMAND_ENV, "").strip()
  52. if configured:
  53. return configured
  54. from tools.transcription_whisper_cpp import whisper_cpp_command
  55. managed_command = whisper_cpp_command()
  56. if managed_command:
  57. return managed_command
  58. whisper_binary = _find_whisper_binary()
  59. return (f"{shlex.quote(whisper_binary)} {{input_path}} --model {{model}} --output_format txt "
  60. "--output_dir {output_dir} --language {language}") if whisper_binary else None
  61. def _has_local_command() -> bool:
  62. return _get_local_command_template() is not None
  63. def _normalize_local_model(model_name: Optional[str]) -> str:
  64. """Return a valid faster-whisper size; cloud-only names (``whisper-1`` …) fall back to the default with a warning."""
  65. if not model_name:
  66. return DEFAULT_LOCAL_MODEL
  67. if model_name in OPENAI_MODELS | GROQ_MODELS | RETIRED_GROQ_MODELS:
  68. logger.warning(
  69. "STT model '%s' is a cloud-only name and cannot be used with the local "
  70. "provider. Falling back to '%s'. Set stt.local.model to a valid "
  71. "faster-whisper size (%s).",
  72. model_name, DEFAULT_LOCAL_MODEL, ", ".join(STT_MODEL_CATALOG["local"]))
  73. return DEFAULT_LOCAL_MODEL
  74. return model_name
  75. def _try_lazy_install_stt() -> bool:
  76. """Install faster-whisper and re-check dynamically so it's usable without a restart.
  77. ACTION paths only (``_transcribe_local``). Nothing that merely *resolves* or *reports* a
  78. provider may call this: the install takes the per-install lock for as long as a full extra-set
  79. rebuild, and a status probe must never start one."""
  80. try:
  81. # pm installs are gated by security.allow_lazy_installs; never a blocking
  82. # prompt mid-session. See #40490.
  83. import pm
  84. pm.ensure_import("stt-whisper")
  85. if _ilu.find_spec("faster_whisper"):
  86. return True
  87. logger.warning("faster-whisper was installed but importlib still cannot find it (may require Python restart)")
  88. except Exception as exc:
  89. logger.warning(
  90. "Lazy install of faster-whisper failed: %s. "
  91. "When the message names a restart, this process selected its dependency generation at "
  92. "boot and a new one cannot take effect in-flight; otherwise the Hermes process user "
  93. "may not be able to write to the dependency environment. Run `hermes tools` as the "
  94. "Hermes installation owner and select Local Whisper under Speech-to-Text.",
  95. exc)
  96. return False
  97. # Substrings identifying a missing/unloadable CUDA runtime library: the "auto" device
  98. # picker has already committed to CUDA, so we fall back to CPU and reload. Deliberately
  99. # narrow (library names + dlopen phrasing) so legitimate runtime failures like "CUDA
  100. # out of memory" surface to the user instead of silently running on CPU.
  101. _CUDA_LIB_ERROR_MARKERS = (
  102. "libcublas", "libcudnn", "libcudart", "cannot be loaded", "cannot open shared object",
  103. "no kernel image is available", "CUBLAS_STATUS_NOT_SUPPORTED", "no CUDA-capable device",
  104. "CUDA driver version is insufficient")
  105. def _looks_like_cuda_lib_error(exc: BaseException) -> bool:
  106. """Heuristic: is this a missing/broken CUDA runtime library (not a legitimate runtime failure)?"""
  107. return any(marker in str(exc) for marker in _CUDA_LIB_ERROR_MARKERS)
  108. def _sysctl_value(name: str) -> str:
  109. """Return a sysctl value, or an empty string when unavailable."""
  110. try:
  111. return subprocess.check_output(["/usr/sbin/sysctl", "-n", name], stderr=subprocess.DEVNULL,
  112. stdin=subprocess.DEVNULL, text=True, encoding="utf-8", errors="replace",
  113. timeout=2).strip()
  114. except Exception:
  115. return ""
  116. def _should_force_faster_whisper_cpu() -> bool:
  117. """Force CPU on Apple Silicon (incl. x86_64 under Rosetta), where ctranslate2's
  118. ``device="auto"`` can abort inside native code before Python can catch it."""
  119. if platform.system() != "Darwin":
  120. return False
  121. if platform.machine().lower() in {"arm64", "aarch64"}:
  122. return True
  123. # Under Rosetta platform.machine() reports x86_64; sysctl.proc_translated
  124. # flags translation and hw.optional.arm64 distinguishes Apple Silicon hosts.
  125. return _sysctl_value("sysctl.proc_translated") == "1" or _sysctl_value("hw.optional.arm64") == "1"
  126. def _get_idle_unload_seconds(local_cfg: dict[str, Any]) -> int:
  127. """Resolve the idle unload timeout from config; 0 = never (default), negatives clamp to 0."""
  128. return max(_config_number(local_cfg, "unload_after_idle_seconds", 0, int), 0)
  129. def _hub_cache_miss_error() -> type:
  130. """Exception faster-whisper raises for a model missing from the local Hub cache.
  131. ``huggingface_hub`` is an optional dependency (it arrives with faster-whisper); when it is
  132. absent, its ``LocalEntryNotFoundError`` base class ``OSError`` is the closest match.
  133. """
  134. try:
  135. from huggingface_hub.errors import LocalEntryNotFoundError
  136. except ImportError:
  137. return OSError
  138. return LocalEntryNotFoundError
  139. def _create_whisper_model(model_name: str, *, device: str, compute_type: str):
  140. """Use a cached model without contacting the Hub, downloading only on a cache miss."""
  141. from faster_whisper import WhisperModel
  142. kwargs = {"device": device, "compute_type": compute_type}
  143. try:
  144. return WhisperModel(model_name, local_files_only=True, **kwargs)
  145. except (_hub_cache_miss_error(), RuntimeError) as exc:
  146. # An interrupted first download leaves a snapshot folder without the weights;
  147. # snapshot_download still returns it and ctranslate2 raises "Unable to open file".
  148. if isinstance(exc, RuntimeError) and "Unable to open file" not in str(exc):
  149. raise
  150. logger.info("faster-whisper model '%s' is not cached; downloading it from the Hugging Face Hub", model_name)
  151. # huggingface_hub surfaces every Hub/network failure as an OSError subclass
  152. # (LocalEntryNotFoundError wrapping the ConnectTimeout, HfHubHTTPError). Anything else
  153. # (CUDA runtime, invalid model size) is not a download problem and propagates untouched.
  154. try:
  155. return WhisperModel(model_name, local_files_only=False, **kwargs)
  156. except OSError as exc:
  157. raise RuntimeError(
  158. f"Unable to download faster-whisper model '{model_name}': {exc}. "
  159. "If huggingface.co is unreachable, set HF_ENDPOINT to an accessible mirror; "
  160. "when using a mirror with hf-xet installed, also set HF_HUB_DISABLE_XET=1."
  161. ) from exc
  162. def _load_local_whisper_model(model_name: str, device: str = "auto", compute_type: str = "auto"):
  163. """Load faster-whisper with graceful CUDA → CPU fallback. ``device="auto"`` picks CUDA
  164. whenever the ctranslate2 wheel ships CUDA libs, even on hosts without the NVIDIA runtime (WSL2,
  165. headless servers): try the requested config first; on a CUDA library load failure fall back to
  166. CPU + int8. Pass ``stt.local.device`` / ``compute_type`` to pin.
  167. ``device`` / ``compute_type`` default to ``"auto"`` so the historical behaviour is unchanged; pass
  168. explicit values from ``stt.local.device`` / ``stt.local.compute_type`` to pin a configuration (#9088).
  169. """
  170. force_cpu = _should_force_faster_whisper_cpu()
  171. if force_cpu:
  172. # Importing ctranslate2 can itself abort on Apple Silicon/Rosetta when
  173. # multiple Intel OpenMP runtimes are loaded — set before the import.
  174. os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE")
  175. if force_cpu:
  176. logger.info("Apple Silicon/Rosetta detected — loading faster-whisper on CPU "
  177. "(int8) to avoid native device autodetection crashes")
  178. return _create_whisper_model(model_name, device="cpu", compute_type="int8")
  179. try:
  180. return _create_whisper_model(model_name, device=device, compute_type=compute_type)
  181. except Exception as exc:
  182. if not _looks_like_cuda_lib_error(exc):
  183. raise
  184. logger.warning("faster-whisper CUDA load failed (%s) — falling back to CPU (int8). "
  185. "Install the NVIDIA CUDA runtime (libcublas/libcudnn) to use GPU.", exc)
  186. return _create_whisper_model(model_name, device="cpu", compute_type="int8")
  187. # Silence-hallucination hardening for local faster-whisper (whisper decodes junk like
  188. # "You"/"Thank you." from pure silence). Three layers, all tunable under ``stt.local``:
  189. # Silero VAD so silence never reaches the model (``vad: false`` restores raw behaviour
  190. # for music/ambient audio); condition_on_previous_text=False so one hallucinated token
  191. # can't seed a run; and the segment confidence gate in _is_hallucinated_segment.
  192. _VAD_MIN_SILENCE_MS_DEFAULT = 500
  193. _NO_SPEECH_PROB_THRESHOLD_DEFAULT = 0.6
  194. _LOGPROB_THRESHOLD_DEFAULT = -1.0
  195. def build_local_transcribe_kwargs(stt_config: Optional[dict[str, Any]] = None) -> dict[str, Any]:
  196. """Kwargs for EVERY local faster-whisper ``model.transcribe`` call — single owner of the anti-hallucination hardening."""
  197. from tools.transcription_tools import _load_stt_config, _resolve_stt_language
  198. stt_config = stt_config if isinstance(stt_config, dict) else _load_stt_config()
  199. local_cfg = stt_config.get("local") or {}
  200. # ``vad: null`` in YAML means "default on".
  201. vad_enabled = local_cfg.get("vad", True)
  202. kwargs: dict[str, Any] = {
  203. "beam_size": 5,
  204. "condition_on_previous_text": False,
  205. "vad_filter": vad_enabled is None or bool(vad_enabled)}
  206. if kwargs["vad_filter"]:
  207. kwargs["vad_parameters"] = {
  208. "min_silence_duration_ms": _config_number(local_cfg, "vad_min_silence_ms", _VAD_MIN_SILENCE_MS_DEFAULT, int)
  209. }
  210. # Push the confidence gate into faster-whisper itself: its internal defaults drop
  211. # low-confidence segments BEFORE our post-filter sees them, so the ``stt.local``
  212. # threshold knobs were dead for that first gate (non-English speech decodes at
  213. # lower avg_logprob and was silently discarded). Same values feed both gates.
  214. kwargs["no_speech_threshold"], kwargs["log_prob_threshold"] = _confidence_thresholds(local_cfg)
  215. forced_lang = _resolve_stt_language("local", stt_config)
  216. if forced_lang:
  217. kwargs["language"] = forced_lang
  218. initial_prompt = local_cfg.get("initial_prompt")
  219. if isinstance(initial_prompt, str) and initial_prompt.strip():
  220. kwargs["initial_prompt"] = initial_prompt
  221. return kwargs
  222. def _confidence_thresholds(local_cfg: dict[str, Any]) -> tuple[float, float]:
  223. """Resolve (no_speech_prob, avg_logprob) gate thresholds from config."""
  224. return (_config_number(local_cfg, "no_speech_prob_threshold", _NO_SPEECH_PROB_THRESHOLD_DEFAULT),
  225. _config_number(local_cfg, "logprob_threshold", _LOGPROB_THRESHOLD_DEFAULT))
  226. def _is_hallucinated_segment(segment: Any, no_speech_threshold: float, logprob_threshold: float) -> bool:
  227. """True when a segment is very likely a silence hallucination. Conservative AND gate
  228. (openai-whisper's own heuristic): non-speech AND low decode confidence, so quiet-but-real speech
  229. survives. Unknown segment shapes are never dropped."""
  230. try:
  231. return (float(segment.no_speech_prob) > no_speech_threshold
  232. and float(segment.avg_logprob) < logprob_threshold)
  233. except (AttributeError, TypeError, ValueError):
  234. return False
  235. def _join_confident_segments(segments: Any, local_cfg: dict[str, Any]) -> str:
  236. """Join segment texts, dropping probable silence hallucinations."""
  237. no_speech_threshold, logprob_threshold = _confidence_thresholds(local_cfg)
  238. kept: list[str] = []
  239. for segment in segments:
  240. if _is_hallucinated_segment(segment, no_speech_threshold, logprob_threshold):
  241. logger.debug("Dropping probable hallucinated segment %r (no_speech_prob=%.3f, avg_logprob=%.3f)",
  242. getattr(segment, "text", ""), getattr(segment, "no_speech_prob", float("nan")),
  243. getattr(segment, "avg_logprob", float("nan")))
  244. continue
  245. kept.append(segment.text.strip())
  246. return " ".join(kept).strip()
  247. def _transcribe_local_command(
  248. file_path: str, model_name: str, *, language: Optional[str] = None, prompt: Optional[str] = None
  249. ) -> dict[str, Any]:
  250. """Run the configured local STT command template and read back a .txt transcript."""
  251. from tools.transcription_tools import _resolve_stt_language
  252. if prompt:
  253. _log_prompt_unsupported("STT provider 'local_command'")
  254. command_template = _get_local_command_template()
  255. if not command_template:
  256. return _error_result(f"{LOCAL_STT_COMMAND_ENV} not configured and no local whisper binary was found")
  257. # Language: hook override > stt.local.language > stt.language > env > "en".
  258. configured_language = language or _resolve_stt_language("local")
  259. language = _normalize_local_stt_language(configured_language) or DEFAULT_LOCAL_STT_LANGUAGE
  260. normalized_model = _normalize_local_model(model_name)
  261. try:
  262. if not os.getenv(LOCAL_STT_COMMAND_ENV, "").strip():
  263. from tools.transcription_whisper_cpp import ensure_whisper_cpp_models, whisper_cpp_command
  264. if command_template == whisper_cpp_command():
  265. ensure_whisper_cpp_models(normalized_model)
  266. with tempfile.TemporaryDirectory(prefix="hermes-local-stt-") as output_dir:
  267. prepared_input, prep_error = _prepare_local_audio(file_path, output_dir)
  268. if prep_error:
  269. return _error_result(prep_error)
  270. command = command_template.format(
  271. input_path=shlex.quote(prepared_input), output_dir=shlex.quote(output_dir),
  272. language=shlex.quote(language), model=shlex.quote(normalized_model))
  273. # Scrub Hermes secrets from the child env (same policy as _run_command_stt).
  274. # Scrub Hermes secrets from the child env (sibling path to #56332 / _run_command_stt — this
  275. # local-whisper path previously inherited the full process environment).
  276. from tools.environments.local import hermes_subprocess_env
  277. _run_quiet(shlex.split(command), timeout=300, env=hermes_subprocess_env(inherit_credentials=False))
  278. txt_files = sorted(Path(output_dir).glob("*.txt"))
  279. if not txt_files:
  280. return _error_result("Local STT command completed but did not produce a .txt transcript")
  281. transcript_text = txt_files[0].read_text(encoding="utf-8-sig").strip()
  282. logger.info("Transcribed %s via local STT command (%s, %d chars)",
  283. Path(file_path).name, normalized_model, len(transcript_text))
  284. return _ok_result(transcript_text, "local_command")
  285. except KeyError as e:
  286. return _error_result(f"Invalid {LOCAL_STT_COMMAND_ENV} template, missing placeholder: {e}")
  287. except subprocess.CalledProcessError as e:
  288. details = _process_error_detail(e)
  289. logger.error("Local STT command failed for %s: %s", file_path, details)
  290. return _error_result(f"Local STT failed: {details}")
  291. except Exception as e:
  292. logger.error("Unexpected error during local command transcription: %s", e, exc_info=True)
  293. return _error_result(f"Local transcription failed: {e}")