transcription_tools.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602
  1. #!/usr/bin/env python3
  2. """Speech-to-text transcription used by the gateway for voice messages.
  3. Built-in providers: local (faster-whisper, default/free), local_command, groq, openai
  4. (also serves the managed ``nous`` selection), mistral, xai, elevenlabs, deepinfra; plus
  5. user-declared command providers and plugin providers. ``transcribe_audio(path)`` returns
  6. ``{"success", "transcript", "error"?, "provider"?}``. This module owns provider resolution,
  7. the dispatcher and the cached local model + idle-unload state; backends live in
  8. ``transcription_{common,audio,local,cloud,command}``.
  9. """
  10. import contextvars
  11. import logging
  12. import os
  13. import shutil
  14. import threading
  15. import time
  16. import importlib.util as _ilu
  17. from contextlib import ExitStack
  18. from pathlib import Path
  19. from tempfile import TemporaryDirectory
  20. from typing import Optional, Dict, Any
  21. from utils import is_truthy_value
  22. from tools.transcription_common import (
  23. BUILTIN_STT_PROVIDERS, CLOUD_STT_PROVIDERS, DEFAULT_ELEVENLABS_STT_MODEL,
  24. DEFAULT_GROQ_STT_MODEL, DEFAULT_LOCAL_MODEL, DEFAULT_MISTRAL_STT_MODEL, DEFAULT_PROVIDER,
  25. DEFAULT_STT_MODEL, LOCAL_STT_COMMAND_ENV, LOCAL_STT_LANGUAGE_ENV,
  26. normalize_xai_stt_model, _error_result, _get_stt_section, _ok_result)
  27. from tools.transcription_audio import (
  28. _convert_caf_to_wav, _prepare_audio_for_transcription, _trim_silence_for_cloud_stt,
  29. _validate_audio_file, _validate_audio_source_file)
  30. from tools.transcription_chunking import exceeds_upload_limit, transcribe_oversized, upload_limit
  31. from tools.transcription_local import (
  32. _get_idle_unload_seconds, _has_local_command, _join_confident_segments,
  33. _load_local_whisper_model, _looks_like_cuda_lib_error, _normalize_local_model,
  34. _normalize_local_stt_language, _transcribe_local_command, _try_lazy_install_stt,
  35. build_local_transcribe_kwargs)
  36. # The ``_transcribe_<provider>`` handlers are looked up in this module's globals by _dispatch_stt_provider.
  37. from tools.transcription_cloud import (
  38. _has_xai_stt_credentials, _resolve_openai_audio_client_config, _transcribe_deepinfra,
  39. _transcribe_elevenlabs, _transcribe_groq, _transcribe_mistral, _transcribe_openai,
  40. _transcribe_xai)
  41. from tools.transcription_command import (
  42. _apply_pre_transcription_hook, _dispatch_to_plugin_provider, _enforce_prompt_length_limit,
  43. _resolve_command_stt_provider_config, _transcribe_command_stt, _unregistered_stt_provider_error)
  44. logger = logging.getLogger(__name__)
  45. def _resolve_provider_key(env_var: str, provider_id: str) -> str:
  46. """STT API key via the shared voice-key resolver (config > env/.env > credential pool); resolved per call."""
  47. from tools.tool_backend_helpers import resolve_provider_secret
  48. return resolve_provider_secret(env_var, provider_id)
  49. def _safe_find_spec(module_name: str) -> bool:
  50. try:
  51. return _ilu.find_spec(module_name) is not None
  52. except (ImportError, ValueError):
  53. return module_name in globals() or module_name in os.sys.modules
  54. _HAS_FASTER_WHISPER, _HAS_OPENAI, _HAS_MISTRAL, _HAS_PILK = map(
  55. _safe_find_spec, ("faster_whisper", "openai", "mistralai", "pilk"))
  56. # Local model singleton; the lock guards check-then-load against concurrent voice messages.
  57. _local_model: Optional[object] = None
  58. _local_model_name: Optional[str] = None
  59. # See #24767.
  60. _local_model_lock = threading.Lock()
  61. # Idle unload: one daemon thread releases the model (hundreds of MB of RAM/VRAM) after a
  62. # configurable idle period, then exits; the next voice message reloads and restarts it.
  63. # _idle_unload_mgmt_lock serializes the start check so no duplicate watchers spawn.
  64. _last_transcription_time: float = 0.0
  65. _idle_unload_thread: Optional[threading.Thread] = None
  66. _idle_unload_stop = threading.Event()
  67. _idle_unload_mgmt_lock = threading.Lock()
  68. # (context, fallback timeout) of the last caller that loaded or used the model: whose profile's
  69. # stt.local.unload_after_idle_seconds the watcher applies. Guarded by _idle_unload_mgmt_lock.
  70. _idle_unload_policy: tuple[contextvars.Context, int] = (contextvars.copy_context(), 0)
  71. _IDLE_UNLOAD_CHECK_INTERVAL = 30 # seconds between idle checks
  72. # ---- Config helpers -----------------------------------------------------
  73. def _load_stt_config() -> dict:
  74. """Load the ``stt`` section from user config, falling back to defaults."""
  75. try:
  76. from hermes_cli.config import load_config
  77. return load_config().get("stt") or {}
  78. except Exception:
  79. return {}
  80. def is_stt_enabled(stt_config: Optional[dict] = None) -> bool:
  81. cfg = _load_stt_config() if stt_config is None else stt_config
  82. return is_truthy_value(cfg.get("enabled", True), default=True)
  83. def _resolve_stt_language(
  84. provider_key: str, stt_config: Optional[dict[str, Any]] = None, *, extra_keys: tuple = ()
  85. ) -> Optional[str]:
  86. """Language hint for an STT provider, first non-empty wins (never ""): ``stt.<provider>.language``
  87. (plus *extra_keys* aliases, e.g. ``language_code``) > ``stt.language`` > ``HERMES_LOCAL_STT_LANGUAGE``
  88. env > None (provider auto-detects)."""
  89. if stt_config is None:
  90. stt_config = _load_stt_config()
  91. provider_cfg = _get_stt_section(stt_config, provider_key)
  92. candidates = [provider_cfg.get(key) for key in ("language", *extra_keys)]
  93. if isinstance(stt_config, dict):
  94. candidates.append(stt_config.get("language"))
  95. candidates.append(os.getenv(LOCAL_STT_LANGUAGE_ENV))
  96. return next((c.strip() for c in candidates if isinstance(c, str) and c.strip()), None)
  97. def _openai_audio_unavailable_reason() -> Optional[str]:
  98. """None when OpenAI audio has usable credentials (config, env, or managed gateway); else the reason."""
  99. try:
  100. # Resolve directly instead of via the boolean probe: the probe flattens
  101. # _resolve_openai_audio_client_config's selection-specific ValueError into False, so a managed
  102. # openai-audio gateway outage would be logged as a generic "no API key" hint (#93045).
  103. _resolve_openai_audio_client_config()
  104. return None
  105. except ValueError as exc:
  106. return str(exc)
  107. def _has_openai_audio_backend() -> bool:
  108. return _openai_audio_unavailable_reason() is None
  109. def _is_local_stt_provider(provider: str, stt_config: dict[str, Any]) -> bool:
  110. """Whether *provider* is exempt from Hermes's remote upload cap."""
  111. return (provider or "").lower().strip() in {"local", "local_command"}
  112. # ---- Provider resolution ------------------------------------------------
  113. def _has_key(env_var: str, provider: str, *, needs_openai: bool = False, needs_mistral: bool = False):
  114. """Availability probe factory: optional SDK flag AND a resolvable API key."""
  115. def probe() -> bool:
  116. sdk_ok = (not needs_openai or _HAS_OPENAI) and (not needs_mistral or _HAS_MISTRAL)
  117. return sdk_ok and bool(_resolve_provider_key(env_var, provider))
  118. return probe
  119. def _has_xai_stt_credentials_quietly() -> bool:
  120. try:
  121. return _has_xai_stt_credentials()
  122. except Exception:
  123. return False
  124. def _resolve_explicit_openai() -> str:
  125. if not _HAS_OPENAI:
  126. logger.warning("STT provider 'openai' configured but no API key available")
  127. return "none"
  128. # Resolved directly so a managed openai-audio gateway outage is logged with its real reason.
  129. reason = _openai_audio_unavailable_reason()
  130. if reason is None:
  131. return "openai"
  132. logger.warning("STT provider 'openai' configured but unavailable: %s", reason)
  133. return "none"
  134. def _detect_local_backend() -> Optional[str]:
  135. """faster-whisper > local whisper CLI; None when no local backend is installed.
  136. Resolution only — it must never install. Asking a status probe to resolve the provider used
  137. to run a full dependency sync here (``_try_lazy_install_stt`` → ``pm.ensure_import``) under
  138. the per-install lock, so ``wake.status`` and ``/voice status`` could hold a sibling profile's
  139. backend off its port for the length of a venv rebuild. A missing faster-whisper now reports
  140. unavailable; the install happens on first transcription, in ``_transcribe_local``."""
  141. if _HAS_FASTER_WHISPER:
  142. return "local"
  143. return "local_command" if _has_local_command() else None
  144. def _resolve_explicit_local() -> str:
  145. backend = _detect_local_backend()
  146. if not backend:
  147. logger.warning("STT provider 'local' configured but unavailable "
  148. "(install faster-whisper or set HERMES_LOCAL_STT_COMMAND)")
  149. return backend or "none"
  150. def _resolve_explicit_local_command() -> str:
  151. if _has_local_command():
  152. return "local_command"
  153. if _HAS_FASTER_WHISPER:
  154. logger.info("Local STT command unavailable, using local faster-whisper")
  155. return "local"
  156. logger.warning("STT provider 'local_command' configured but unavailable")
  157. return "none"
  158. _has_groq_key = _has_key("GROQ_API_KEY", "groq", needs_openai=True)
  159. _has_mistral_key = _has_key("MISTRAL_API_KEY", "mistral", needs_mistral=True)
  160. _has_elevenlabs_key = _has_key("ELEVENLABS_API_KEY", "elevenlabs")
  161. _has_deepinfra_key = _has_key("DEEPINFRA_API_KEY", "deepinfra", needs_openai=True)
  162. # Cloud providers in AUTO-DETECT priority order:
  163. # name -> (explicit-selection probe, auto-detect probe, explicit warning, auto-detect log)
  164. # The probes differ only for openai (explicit has its own resolver in _EXPLICIT_RESOLVERS;
  165. # auto-detect also requires the SDK) and xai (auto-detect must never raise). DeepInfra is
  166. # LAST so a DEEPINFRA_API_KEY set for chat never displaces an xAI/ElevenLabs auto-selection.
  167. # Mistral only auto-selects when the SDK is present — no lazy-install during passive
  168. # auto-detection (explicit ``provider: mistral`` installs on first use).
  169. _CLOUD_PROVIDER_SPECS = {
  170. "groq": (_has_groq_key, _has_groq_key,
  171. "STT provider 'groq' configured but GROQ_API_KEY not set",
  172. "No local STT available, using Groq Whisper API"),
  173. "openai": (None, lambda: _HAS_OPENAI and _has_openai_audio_backend(),
  174. None,
  175. "No local STT available, using OpenAI Whisper API"),
  176. "mistral": (_has_mistral_key, _has_mistral_key,
  177. "STT provider 'mistral' configured but mistralai package not installed or MISTRAL_API_KEY not set",
  178. "No local STT available, using Mistral Voxtral Transcribe API"),
  179. "xai": (_has_xai_stt_credentials, _has_xai_stt_credentials_quietly,
  180. "STT provider 'xai' configured but no xAI credentials are available",
  181. "No local STT available, using xAI Grok STT API"),
  182. "elevenlabs": (_has_elevenlabs_key, _has_elevenlabs_key,
  183. "STT provider 'elevenlabs' configured but ELEVENLABS_API_KEY not set",
  184. "No local STT available, using ElevenLabs Scribe STT API"),
  185. "deepinfra": (_has_deepinfra_key, _has_deepinfra_key,
  186. "STT provider 'deepinfra' configured but DEEPINFRA_API_KEY not set (or openai package missing)",
  187. "No local STT available, using DeepInfra Whisper API")}
  188. # Explicit selections whose resolution is more than a probe + warning.
  189. _EXPLICIT_RESOLVERS = {
  190. "local": _resolve_explicit_local,
  191. "local_command": _resolve_explicit_local_command,
  192. "openai": _resolve_explicit_openai}
  193. def _resolve_explicit_provider(provider: str) -> str:
  194. """Explicit ``stt.provider`` -> usable name or ``"none"``; unknown names pass through untouched
  195. so the dispatcher fails with the provider-not-registered message."""
  196. resolver = _EXPLICIT_RESOLVERS.get(provider)
  197. if resolver is not None:
  198. return resolver()
  199. spec = _CLOUD_PROVIDER_SPECS.get(provider)
  200. if spec is None or spec[0]():
  201. return provider
  202. logger.warning(spec[2])
  203. return "none"
  204. def _get_provider(stt_config: dict) -> str:
  205. """Which STT provider to use: an explicit ``stt.provider`` is honoured (no silent cloud
  206. fallback); otherwise auto-detect local > groq > openai > mistral > xai > elevenlabs > deepinfra."""
  207. if not is_stt_enabled(stt_config):
  208. return "none"
  209. explicit = "provider" in stt_config
  210. provider = stt_config.get("provider", DEFAULT_PROVIDER)
  211. # The managed "Nous Subscription" selection is the OpenAI backend routed via the managed gateway.
  212. if isinstance(provider, str) and provider.strip().lower() == "nous":
  213. provider = "openai"
  214. if explicit and provider == "local":
  215. # Legacy DEFAULT_CONFIG seeded ``stt.provider: local`` on every install, so only a
  216. # raw config.yaml selection counts as explicit; otherwise autodetect (local-first anyway).
  217. try:
  218. from tools.tool_backend_helpers import read_selection
  219. if read_selection("stt") is None:
  220. explicit = False
  221. except Exception: # pragma: no cover — helpers are in-repo
  222. pass
  223. if explicit:
  224. return _resolve_explicit_provider(provider)
  225. backend = _detect_local_backend()
  226. if backend:
  227. return backend
  228. for name, (_probe, available, _warning, message) in _CLOUD_PROVIDER_SPECS.items():
  229. if available():
  230. logger.info(message)
  231. return name
  232. return "none"
  233. # ---- Provider: local (faster-whisper) -----------------------------------
  234. def _unload_local_model() -> None:
  235. """Release the cached local whisper model. Thread-safe via the model lock."""
  236. global _local_model, _local_model_name
  237. with _local_model_lock:
  238. if _local_model is not None:
  239. logger.info("Unloading local whisper model '%s' after idle timeout", _local_model_name or "unknown")
  240. _local_model = None
  241. _local_model_name = None
  242. def _configured_idle_unload_seconds() -> int:
  243. return _get_idle_unload_seconds(_load_stt_config().get("local") or {})
  244. def _start_idle_unload_watcher(timeout_seconds: int) -> None:
  245. """Ensure the single idle-unload watcher thread is running, governed by the caller's profile.
  246. One process can serve several profiles, each with its own
  247. ``stt.local.unload_after_idle_seconds``, and they share the one cached model. The policy that
  248. governs it is the one of whoever last loaded or used it: every call records the caller's
  249. context (profile scope included) and the loop re-reads the setting inside that context each
  250. cycle, so config edits still apply within one interval and a watcher started under profile A
  251. stops applying A's policy once profile B warms or transcribes. ``timeout_seconds`` is the
  252. fallback when that read fails. Exits after unloading, when the timeout becomes 0, or when the
  253. model is gone."""
  254. global _idle_unload_thread, _idle_unload_policy
  255. with _idle_unload_mgmt_lock:
  256. _idle_unload_policy = (contextvars.copy_context(), timeout_seconds)
  257. if _idle_unload_thread is not None and _idle_unload_thread.is_alive():
  258. return
  259. def _watch():
  260. while not _idle_unload_stop.wait(_IDLE_UNLOAD_CHECK_INTERVAL) and _local_model is not None:
  261. with _idle_unload_mgmt_lock:
  262. owner_context, fallback = _idle_unload_policy
  263. try:
  264. # A copy: a Context can't be entered twice at once, and the owner's own
  265. # request may still be running in the original.
  266. timeout = owner_context.copy().run(_configured_idle_unload_seconds)
  267. except Exception:
  268. timeout = fallback
  269. if timeout <= 0:
  270. break # unload disabled mid-flight — stand down
  271. if time.monotonic() - _last_transcription_time >= timeout:
  272. _unload_local_model()
  273. break
  274. _idle_unload_stop.clear()
  275. _idle_unload_thread = threading.Thread(target=_watch, name="hermes-stt-idle-unload", daemon=True)
  276. _idle_unload_thread.start()
  277. def _touch_transcription_time() -> None:
  278. """Record transcription activity (resets the idle timer)."""
  279. global _last_transcription_time
  280. _last_transcription_time = time.monotonic()
  281. def _get_or_load_local_model(model_name: str, local_cfg: dict[str, Any]):
  282. """Cached faster-whisper model, (re)loaded under a double-checked lock when needed. The returned
  283. strong reference stays valid even if the idle watcher nulls the global mid-transcription."""
  284. global _local_model, _local_model_name
  285. model = _local_model
  286. # Lazy-load the model (downloads on first use, ~150 MB for 'base'). Double-checked lock: concurrent
  287. # voice messages must not both download/load the model (#24767). ``model`` is a strong local reference
  288. # bound under the lock: the idle watcher may null the module global at any time, but this transcription
  289. # keeps using the instance it grabbed.
  290. if model is None or _local_model_name != model_name:
  291. with _local_model_lock:
  292. if _local_model is None or _local_model_name != model_name:
  293. logger.info("Loading faster-whisper model '%s' (first load downloads the model)...", model_name)
  294. # stt.local.device / compute_type pin a configuration where ``auto`` mis-detects.
  295. _local_model = _load_local_whisper_model(model_name, device=local_cfg.get("device", "auto"),
  296. compute_type=local_cfg.get("compute_type", "auto"))
  297. _local_model_name = model_name
  298. model = _local_model
  299. return model
  300. def _replace_cached_model_on_cpu(model_name: str):
  301. """Load *model_name* on CPU/int8 and make it the cached singleton."""
  302. global _local_model, _local_model_name
  303. model = _load_local_whisper_model(model_name, device="cpu", compute_type="int8")
  304. with _local_model_lock:
  305. _local_model, _local_model_name = model, model_name
  306. return model
  307. def _transcribe_local(
  308. file_path: str, model_name: str, *, language: Optional[str] = None, prompt: Optional[str] = None
  309. ) -> dict[str, Any]:
  310. """Transcribe using faster-whisper (local, free)."""
  311. if not _HAS_FASTER_WHISPER and not _try_lazy_install_stt():
  312. return _error_result("faster-whisper not installed")
  313. try:
  314. stt_config = _load_stt_config()
  315. local_cfg = stt_config.get("local") or {}
  316. # Reset the idle timer BEFORE loading so a long in-flight transcription isn't counted as idle.
  317. _touch_transcription_time()
  318. model = _get_or_load_local_model(model_name, local_cfg)
  319. if model is None: # defensive: load failed without raising
  320. return _error_result("Local whisper model failed to load")
  321. # pre_transcription hook overrides win over config-resolved values.
  322. transcribe_kwargs = build_local_transcribe_kwargs(stt_config)
  323. effective_language = language if language is not None else transcribe_kwargs.get("language")
  324. normalized_language = _normalize_local_stt_language(
  325. effective_language, getattr(model, "supported_languages", None))
  326. if normalized_language:
  327. transcribe_kwargs["language"] = normalized_language
  328. else:
  329. transcribe_kwargs.pop("language", None)
  330. if prompt:
  331. transcribe_kwargs["initial_prompt"] = prompt
  332. try:
  333. segments, info = model.transcribe(file_path, **transcribe_kwargs)
  334. # faster-whisper's transcribe() is lazy: the decode (and with it the
  335. # dlopen-on-first-use of the CUDA runtime on Windows) happens while
  336. # ITERATING segments, after this call has already returned (#103793).
  337. # Consume inside the guard so a first-use cuBLAS/cuDNN load failure
  338. # retries on CPU exactly like a load-time failure does.
  339. segments = list(segments)
  340. except Exception as exc:
  341. # CUDA libs can fail at dlopen-on-first-use, AFTER loading: evict the poisoned
  342. # cached model, reload on CPU and retry once, else every later message fails.
  343. if not _looks_like_cuda_lib_error(exc):
  344. raise
  345. logger.warning("faster-whisper CUDA runtime failed mid-transcribe (%s) — "
  346. "evicting cached model and retrying on CPU (int8).", exc)
  347. model = _replace_cached_model_on_cpu(model_name)
  348. segments, info = model.transcribe(file_path, **transcribe_kwargs)
  349. segments = list(segments)
  350. transcript = _join_confident_segments(segments, local_cfg)
  351. logger.info("Transcribed %s via local whisper (%s, lang=%s, %.1fs audio)",
  352. Path(file_path).name, model_name, info.language, info.duration)
  353. _touch_transcription_time()
  354. idle_timeout = _get_idle_unload_seconds(local_cfg)
  355. if idle_timeout > 0:
  356. _start_idle_unload_watcher(idle_timeout)
  357. return _ok_result(transcript, "local")
  358. except Exception as e:
  359. logger.error("Local transcription failed: %s", e, exc_info=True)
  360. return _error_result(f"Local transcription failed: {e}")
  361. # ---- Public API ---------------------------------------------------------
  362. def _read_block_error(file_path: str) -> Optional[dict[str, Any]]:
  363. """Refuse to ship a credential store (auth.json, .env, OAuth tokens) to an STT provider.
  364. Mirrors the image-gen / video-gen read guards."""
  365. from agent.file_safety import get_read_block_error
  366. blocked = get_read_block_error(file_path)
  367. return _error_result(blocked) if blocked else None
  368. def _transcribe_prepared_audio(
  369. file_path: str, model: Optional[str] = None, source: Optional[str] = None) -> dict[str, Any]:
  370. """Transcribe a validated audio file with the configured STT provider. ``model`` overrides the
  371. config default; ``source`` is a caller-surface label (``"gateway"``, ``"voice_mode"``) forwarded
  372. to the ``pre_transcription`` hook only."""
  373. # Validate before provider resolution so invalid files can't trigger provider setup
  374. # or lazy installation; the remote-upload size cap applies to non-local only.
  375. error = _read_block_error(file_path) or _validate_audio_file(file_path, enforce_size_limit=False)
  376. if error:
  377. return error
  378. stt_config = _load_stt_config()
  379. if not is_stt_enabled(stt_config):
  380. return _error_result("STT is disabled in config.yaml (stt.enabled: false).")
  381. provider = _get_provider(stt_config)
  382. remote = not _is_local_stt_provider(provider, stt_config)
  383. with ExitStack() as cleanup:
  384. # Never overwrite a neighboring WAV or leave converted voice notes behind.
  385. if remote and Path(file_path).suffix.lower() == ".caf":
  386. work_dir = cleanup.enter_context(TemporaryDirectory(prefix="hermes-caf-", ignore_cleanup_errors=True))
  387. file_path = _convert_caf_to_wav(file_path, work_dir)
  388. if not file_path:
  389. return _error_result("CAF audio could not be converted to WAV.")
  390. # Best-effort pre-upload silence trim for built-in cloud providers.
  391. if provider in CLOUD_STT_PROVIDERS:
  392. trimmed = _trim_silence_for_cloud_stt(file_path, stt_config)
  393. if trimmed:
  394. file_path = trimmed
  395. cleanup.callback(shutil.rmtree, os.path.dirname(trimmed), ignore_errors=True)
  396. if remote:
  397. limit = upload_limit(provider, _limit_model_name(provider, stt_config, model))
  398. if exceeds_upload_limit(file_path, limit):
  399. # Hints are resolved once for the whole recording, not once per segment.
  400. hints = _resolve_dispatch_hints(file_path, provider, stt_config, model, source)
  401. return transcribe_oversized(
  402. file_path, limit, lambda part: _route_stt_provider(part, provider, stt_config, *hints))
  403. return _dispatch_stt_provider(file_path, provider, stt_config, model, source)
  404. def _limit_model_name(provider: str, stt_config: dict[str, Any], model: Optional[str]) -> Optional[str]:
  405. """Model whose duration cap applies to an upload (built-ins resolve config/defaults)."""
  406. return _builtin_model_name(provider, stt_config, model) if provider in _BUILTIN_MODEL_KEYS else model
  407. # Built-in provider -> (stt section, config key, default, treat-empty-as-missing). "local_command"
  408. # shares ``stt.local``; deepinfra uses the live catalog when empty.
  409. _BUILTIN_MODEL_KEYS = {
  410. "local": ("local", "model", DEFAULT_LOCAL_MODEL, False),
  411. "local_command": ("local", "model", DEFAULT_LOCAL_MODEL, False),
  412. "groq": ("groq", "model", DEFAULT_GROQ_STT_MODEL, True),
  413. "openai": ("openai", "model", DEFAULT_STT_MODEL, False),
  414. "mistral": ("mistral", "model", DEFAULT_MISTRAL_STT_MODEL, False),
  415. "xai": ("xai", "model", "", True), # blank -> normalize_xai_stt_model resolves env/default
  416. "elevenlabs": ("elevenlabs", "model_id", DEFAULT_ELEVENLABS_STT_MODEL, False),
  417. "deepinfra": ("deepinfra", "model", "", True)}
  418. def _builtin_model_name(provider: str, stt_config: dict[str, Any], model: Optional[str]) -> str:
  419. """Resolve the model for a built-in provider: caller override > ``stt.<provider>`` config > default."""
  420. if model:
  421. return model
  422. section, key, default, empty_is_missing = _BUILTIN_MODEL_KEYS[provider]
  423. cfg = _get_stt_section(stt_config, section)
  424. resolved = (cfg.get(key) or default) if empty_is_missing else cfg.get(key, default)
  425. return normalize_xai_stt_model(resolved) if provider == "xai" else resolved
  426. def _resolve_dispatch_hints(
  427. file_path: str, provider: str, stt_config: dict[str, Any], model: Optional[str], source: Optional[str],
  428. ) -> tuple[Optional[str], Optional[str], Optional[str]]:
  429. """``(model, language, prompt)`` for one recording: static ``stt.prompt`` plus ``pre_transcription`` hooks."""
  430. # Static ``stt.prompt`` is the base; hook results mutate on top (last hook to set a field wins).
  431. prompt = stt_config.get("prompt")
  432. prompt = prompt if isinstance(prompt, str) and prompt.strip() else None
  433. # Fires after provider resolution and BEFORE any backend; ``language`` stays None unless a hook sets it.
  434. model, language, prompt = _apply_pre_transcription_hook(
  435. file_path=file_path, provider=provider, model=model,
  436. language=_get_stt_section(stt_config, provider).get("language"), prompt=prompt, source=source,
  437. )
  438. return model, language, _enforce_prompt_length_limit(prompt, provider)
  439. def _dispatch_stt_provider(
  440. file_path: str, provider: str, stt_config: dict[str, Any], model: Optional[str] = None,
  441. source: Optional[str] = None) -> dict[str, Any]:
  442. """Route *file_path* to the handler for *provider* (built-in > command > plugin)."""
  443. return _route_stt_provider(
  444. file_path, provider, stt_config, *_resolve_dispatch_hints(file_path, provider, stt_config, model, source))
  445. def _route_stt_provider(
  446. file_path: str, provider: str, stt_config: dict[str, Any], model: Optional[str],
  447. language: Optional[str], prompt: Optional[str]) -> dict[str, Any]:
  448. if provider in BUILTIN_STT_PROVIDERS:
  449. # Looked up in this module at call time so tests may patch ``_transcribe_*``.
  450. handler = globals()[f"_transcribe_{provider}"]
  451. model_name = _builtin_model_name(provider, stt_config, model)
  452. if provider in ("local", "local_command"):
  453. model_name = _normalize_local_model(model_name)
  454. return handler(file_path, model_name, language=language, prompt=prompt)
  455. # Command providers: after built-ins (``stt.providers.openai.command`` can't override the
  456. # real handler) and BEFORE plugins, since config is more local than a plugin install.
  457. # User-declared command-type provider (``stt.providers.<name>: type: command``). See #17843.
  458. command_provider_config = _resolve_command_stt_provider_config(provider, stt_config)
  459. if command_provider_config is not None:
  460. return _transcribe_command_stt(file_path, provider, command_provider_config, stt_config,
  461. model_override=model, language_override=language, prompt=prompt)
  462. # Plugin backend: reads ``stt.<provider>`` like built-ins; the ``model`` argument overrides it.
  463. plugin_result = _dispatch_to_plugin_provider(
  464. file_path, provider, stt_config, model=model or _get_stt_section(stt_config, provider).get("model"),
  465. language=language or _resolve_stt_language(provider, stt_config), prompt=prompt)
  466. return plugin_result if plugin_result is not None else _no_provider_error(provider, stt_config)
  467. def _no_provider_error(provider: str, stt_config: dict[str, Any]) -> dict[str, Any]:
  468. """Error envelope when nothing claimed *provider*: unregistered name > openai selection reason > generic hint."""
  469. provider_key = str(provider or "").strip().lower()
  470. if "provider" in stt_config and provider_key and provider_key not in BUILTIN_STT_PROVIDERS and provider_key != "none":
  471. return _unregistered_stt_provider_error(provider_key)
  472. # An explicit openai selection flattened to "none" has a specific reason (e.g. managed gateway down).
  473. # Surface it — with its `hermes tools` remediation — instead of the all-provider setup hint (#93045).
  474. if provider_key == "none" and str(stt_config.get("provider") or "") == "openai" and _HAS_OPENAI:
  475. reason = _openai_audio_unavailable_reason()
  476. if reason is not None:
  477. return _error_result(reason)
  478. return _error_result(
  479. "No STT provider available. Install faster-whisper for free local "
  480. f"transcription, configure {LOCAL_STT_COMMAND_ENV} or install a local whisper CLI, "
  481. "set GROQ_API_KEY for free Groq Whisper, set MISTRAL_API_KEY for Mistral "
  482. "Voxtral Transcribe, configure xAI OAuth or set XAI_API_KEY for xAI Grok STT, "
  483. "set ELEVENLABS_API_KEY for ElevenLabs Scribe, or set VOICE_TOOLS_OPENAI_KEY "
  484. "or OPENAI_API_KEY for the OpenAI Whisper API.")
  485. def transcribe_audio(
  486. file_path: str, model: Optional[str] = None, source: Optional[str] = None) -> dict[str, Any]:
  487. """Validate, preprocess supported inputs, and dispatch transcription. ``source`` is a caller-surface
  488. label (``"gateway"``, ``"voice_mode"``) forwarded to the ``pre_transcription`` hook only."""
  489. # Secret-store refusal runs before ANY validation so the error names the real reason.
  490. blocked = _read_block_error(file_path)
  491. if blocked:
  492. return blocked
  493. # Cap .silk sources before the decoder runs; for other inputs the upload cap is
  494. # provider-scoped in _transcribe_prepared_audio so local whisper can take big files.
  495. is_silk = Path(file_path).suffix.lower() == ".silk"
  496. source_error = _validate_audio_source_file(file_path, enforce_size_limit=is_silk)
  497. if source_error:
  498. return source_error
  499. prepared_path, cleanup_dir, prep_error = _prepare_audio_for_transcription(file_path)
  500. if prep_error or prepared_path is None:
  501. return prep_error or _error_result("Audio preprocessing did not produce a file for transcription.")
  502. try:
  503. return (_validate_audio_file(prepared_path, enforce_size_limit=False)
  504. or _transcribe_prepared_audio(prepared_path, model, source))
  505. finally:
  506. if cleanup_dir:
  507. shutil.rmtree(cleanup_dir, ignore_errors=True)
  508. def transcribe_audio_local_fallback(file_path: str, model: Optional[str] = None) -> dict[str, Any]:
  509. """Try an already-installed local STT backend without changing config: passive inbound-media
  510. recovery after the configured provider failed — never lazy-installs or falls through to cloud."""
  511. error = _validate_audio_file(file_path, enforce_size_limit=False)
  512. if error:
  513. return error
  514. local_model = model or (_load_stt_config().get("local") or {}).get("model", DEFAULT_LOCAL_MODEL)
  515. if _HAS_FASTER_WHISPER:
  516. return _transcribe_local(file_path, _normalize_local_model(local_model))
  517. if _has_local_command():
  518. return _transcribe_local_command(file_path, _normalize_local_model(local_model))
  519. return _error_result("No installed local STT backend is available.", provider="local")