transcription_command.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278
  1. """User-declared and plugin STT providers.
  2. ``stt.providers.<name>: type: command`` registry, plugin-registered
  3. ``TranscriptionProvider`` dispatch, and the ``pre_transcription`` hook that
  4. threads prompt/language/model overrides into every backend. ``_resolve_stt_language``
  5. is read lazily from ``tools.transcription_tools``.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import subprocess
  10. import tempfile
  11. from functools import partial
  12. from pathlib import Path
  13. from typing import Any, Dict, Optional
  14. from agent.model_metadata import CHARS_PER_TOKEN
  15. from tools.tts_command_provider import (
  16. _command_output_format, _command_timeout, _is_command_provider_config as _is_command_stt_provider_config,
  17. _named_provider_config, _resolve_command_config, command_env_passthrough as _command_stt_env_passthrough,
  18. command_failure_detail, render_command_template as _render_command_stt_template,
  19. run_command_provider as _run_command_stt)
  20. from tools.transcription_audio import _transcode_audio_for_stt
  21. from tools.transcription_common import (
  22. BUILTIN_STT_PROVIDERS, _error_result, _log_prompt_unsupported, _ok_result)
  23. from utils import is_truthy_value
  24. # Log-record parity with the origin module.
  25. logger = logging.getLogger("tools.transcription_tools")
  26. # ---- Command-provider registry (``stt.providers.<name>: type: command``) ---
  27. #
  28. # Mirrors the TTS command-provider registry (same placeholder grammar, quote-aware
  29. # rendering, process-tree termination on timeout). Resolution order: built-in name
  30. # (always wins) > stt.providers.<name> command > plugin TranscriptionProvider >
  31. # "No STT provider available". The single-env-var HERMES_LOCAL_STT_COMMAND escape
  32. # hatch stays untouched via the built-in ``local_command`` path.
  33. # Lets any whisper CLI / ASR CLI / curl pipeline become an STT backend with zero Python. 1. Built-in
  34. # (``local``, ``local_command``, ``groq``, ``openai``, ``mistral``, ``xai``) → native handler.
  35. # **Always wins.** 2. 3. 4. Use the command-provider registry when you want MULTIPLE shell-driven STT
  36. # engines, or you want a named provider you can pick via ``stt.provider`` in config.yaml. See #17843.
  37. DEFAULT_COMMAND_STT_TIMEOUT_SECONDS = 300
  38. DEFAULT_COMMAND_STT_LANGUAGE = "en"
  39. DEFAULT_COMMAND_STT_OUTPUT_FORMAT = "txt"
  40. COMMAND_STT_OUTPUT_FORMATS = frozenset({"txt", "json", "srt", "vtt"})
  41. _NON_COMMAND_STT_NAMES = frozenset(BUILTIN_STT_PROVIDERS | {"none"})
  42. # ``stt.providers.<name>`` (canonical), else ``stt.<name>`` for non-built-in names only.
  43. _get_named_stt_provider_config = partial(_named_provider_config, builtins=BUILTIN_STT_PROVIDERS)
  44. # The provider config if it is a command type; None for built-ins, ``none``, unknown.
  45. _resolve_command_stt_provider_config = partial(_resolve_command_config,
  46. reserved=_NON_COMMAND_STT_NAMES)
  47. _get_command_stt_timeout = partial(_command_timeout, default=DEFAULT_COMMAND_STT_TIMEOUT_SECONDS)
  48. _get_command_stt_output_format = partial(_command_output_format, formats=COMMAND_STT_OUTPUT_FORMATS,
  49. default=DEFAULT_COMMAND_STT_OUTPUT_FORMAT)
  50. def _read_command_stt_output(output_path: Path, stdout: str, fmt: str) -> str:
  51. """Transcript: non-empty output file > non-empty stdout (curl one-liners) > RuntimeError. JSON is returned raw."""
  52. content = (output_path.read_text(encoding="utf-8-sig", errors="replace").strip()
  53. if output_path.exists() else "")
  54. if content or (stdout or "").strip():
  55. return content or stdout.strip()
  56. raise RuntimeError(f"Command STT provider wrote no output file at {output_path} and produced no stdout")
  57. def _normalize_command_stt_input(audio: Path, tmpdir: str, provider_name: str,
  58. config: dict[str, Any]) -> tuple[Optional[str], Optional[str]]:
  59. """Resolve the file handed to a command STT provider. ``stt.providers.<name>.normalize``
  60. (default off) transcodes to 16 kHz mono m4a first — desktop voice notes arrive as
  61. WebM/Opus 48 kHz and providers with container/sample-rate contracts (Tencent 16k_zh)
  62. reject the raw container. Failure is an error, not a silent pass-through: the user
  63. opted into normalization, so feeding the raw file would reproduce the bug. Returns
  64. ``(input_path, None)`` or ``(None, error)``."""
  65. if not is_truthy_value(config.get("normalize"), default=False):
  66. return str(audio.resolve()), None
  67. converted_path, transcode_error = _transcode_audio_for_stt(str(audio.resolve()), tmpdir)
  68. if transcode_error or not converted_path:
  69. return None, (f"stt.providers.{provider_name}.normalize: true but the audio could not be "
  70. f"normalized for the command provider: {transcode_error or 'unknown ffmpeg failure'}")
  71. logger.info("Normalized %s to 16 kHz mono for command STT provider '%s'",
  72. audio.name, provider_name)
  73. return converted_path, None
  74. def _transcribe_command_stt(
  75. file_path: str, provider_name: str, config: dict[str, Any], stt_config: dict[str, Any],
  76. model_override: Optional[str] = None, language_override: Optional[str] = None,
  77. prompt: Optional[str] = None) -> dict[str, Any]:
  78. """Transcribe via a user-declared ``stt.providers.<name>: type: command``. Placeholders
  79. (shell-quote-aware; ``{{``/``}}`` stay literal): ``{input_path}`` (the original file, or a
  80. 16 kHz mono m4a when ``normalize: true``), ``{output_path}`` (transcript file),
  81. ``{output_dir}``, ``{format}`` txt/json/srt/vtt, ``{language}`` (default ``en``),
  82. ``{model}`` (empty when unset)."""
  83. from tools.transcription_tools import _resolve_stt_language
  84. if prompt:
  85. _log_prompt_unsupported(f"Command STT provider '{provider_name}'")
  86. def fail(error: str) -> dict[str, Any]:
  87. return _error_result(error, provider=provider_name)
  88. command_template = str(config.get("command") or "").strip()
  89. if not command_template:
  90. return fail(f"stt.providers.{provider_name}.command is not configured")
  91. audio = Path(file_path).expanduser()
  92. if not audio.exists():
  93. return fail(f"Audio file not found: {file_path}")
  94. timeout = _get_command_stt_timeout(config)
  95. output_format = _get_command_stt_output_format(config)
  96. language = (language_override or config.get("language")
  97. or _resolve_stt_language(provider_name, stt_config) or DEFAULT_COMMAND_STT_LANGUAGE)
  98. try:
  99. with tempfile.TemporaryDirectory(prefix=f"hermes-cmd-stt-{provider_name}-") as tmpdir:
  100. input_path, normalize_error = _normalize_command_stt_input(audio, tmpdir, provider_name, config)
  101. if normalize_error:
  102. return fail(normalize_error)
  103. output_path = Path(tmpdir) / f"transcript.{output_format}"
  104. command = _render_command_stt_template(command_template, {
  105. "input_path": input_path, "output_path": str(output_path),
  106. "output_dir": str(output_path.parent), "format": output_format,
  107. "language": str(language), "model": str(model_override or config.get("model") or ""),
  108. })
  109. logger.info("Transcribing %s via command STT provider '%s'...", audio.name, provider_name)
  110. result = _run_command_stt(command, timeout, env_passthrough=_command_stt_env_passthrough(config))
  111. transcript_text = _read_command_stt_output(output_path, result.stdout or "", output_format)
  112. except subprocess.TimeoutExpired:
  113. return fail(f"STT command provider '{provider_name}' timed out after {timeout:g}s")
  114. except subprocess.CalledProcessError as exc:
  115. return fail(
  116. f"STT command provider '{provider_name}' exited with code {exc.returncode}: {command_failure_detail(exc)}"
  117. )
  118. except RuntimeError as exc:
  119. return fail(str(exc))
  120. except OSError as exc:
  121. return fail(f"STT command provider '{provider_name}' failed: {exc}")
  122. logger.info("Transcribed %s via command STT provider '%s' (%d chars)", audio.name, provider_name, len(transcript_text))
  123. return _ok_result(transcript_text, provider_name)
  124. def _unregistered_stt_provider_error(provider: str) -> dict[str, Any]:
  125. key = str(provider or "").strip()
  126. return _error_result(
  127. f"stt.provider='{key}' is set but no built-in, command, or plugin "
  128. "provider registered that name. Run `hermes plugins list` to see "
  129. "installed STT plugins, or configure a command provider under "
  130. f"`stt.providers.{key}.command`.",
  131. provider=key,
  132. error_type="provider_not_registered")
  133. # --------------------------------------------------------------------------- Plugin provider dispatch
  134. # (issue follow-up to #30398 — STT pluggability)
  135. # ---------------------------------------------------------------------------
  136. def _dispatch_to_plugin_provider(
  137. file_path: str, provider: str, stt_config: Optional[dict[str, Any]] = None, *,
  138. model: Optional[str] = None, language: Optional[str] = None, prompt: Optional[str] = None,
  139. ) -> Optional[dict[str, Any]]:
  140. """Route to a plugin-registered transcription provider; None when no plugin claims the name.
  141. Invariants re-verified here so a caller refactor can't break them: built-in names never reach
  142. the registry; a same-name command provider wins over a plugin. A matched plugin with
  143. ``is_available() == False`` returns an error envelope — not None — because the user
  144. explicitly opted in via ``stt.provider``. Provider exceptions become the error envelope."""
  145. key = (provider or "").lower().strip()
  146. if not key or key in _NON_COMMAND_STT_NAMES:
  147. return None
  148. if stt_config is not None and _is_command_stt_provider_config(_get_named_stt_provider_config(stt_config, key)):
  149. return None
  150. try:
  151. from agent.transcription_registry import get_provider
  152. from hermes_cli.plugins import _ensure_plugins_discovered
  153. _ensure_plugins_discovered()
  154. plugin_provider = get_provider(key)
  155. if plugin_provider is None:
  156. # Long-lived sessions may have discovered plugins before a backend
  157. # was patched in or config changed — retry once with a forced refresh.
  158. _ensure_plugins_discovered(force=True)
  159. plugin_provider = get_provider(key)
  160. except Exception as exc:
  161. logger.debug("STT plugin dispatch skipped (discovery failed): %s", exc)
  162. return None
  163. if plugin_provider is None:
  164. return None
  165. # ``is_available()`` MUST NOT raise per the ABC contract; defend anyway so
  166. # a buggy plugin can't break dispatch for everyone.
  167. try:
  168. available = plugin_provider.is_available()
  169. except Exception as exc:
  170. logger.warning(
  171. "STT plugin provider '%s' is_available() raised: %s — treating as unavailable", key, exc, exc_info=True,
  172. )
  173. available = False
  174. if not available:
  175. logger.info("STT plugin provider '%s' reports not available; returning unavailability envelope.", key)
  176. return _error_result(
  177. f"STT plugin '{key}' is not available — check that its required credentials / dependencies are configured.",
  178. provider=key)
  179. logger.info("Transcribing with plugin STT provider '%s'...", key)
  180. # The prompt travels via the ABC's ``**extra`` kwargs and is only sent when
  181. # set, so pre-prompt providers see byte-identical calls on the no-prompt path.
  182. extra_kwargs: dict[str, Any] = {} if prompt is None else {"prompt": prompt}
  183. try:
  184. result = plugin_provider.transcribe(file_path, model=model, language=language, **extra_kwargs)
  185. except Exception as exc:
  186. logger.warning("STT plugin provider '%s' raised: %s", key, exc, exc_info=True)
  187. return _error_result(f"STT plugin '{key}' raised: {exc}", provider=key)
  188. if not isinstance(result, dict):
  189. return _error_result(f"STT plugin '{key}' returned a non-dict result", provider=key)
  190. result.setdefault("provider", key)
  191. return result
  192. # Fields a pre_transcription hook may mutate; ``file_path`` is read-only (logged and dropped).
  193. # --------------------------------------------------------------------------- pre_transcription plugin hook
  194. # (issue #64168 — STT prompt/vocab threading)
  195. # ---------------------------------------------------------------------------
  196. _PRE_TRANSCRIPTION_MUTABLE_FIELDS = ("prompt", "language", "model")
  197. # Whisper-family models only use the final ~224 tokens of the prompt; longer values
  198. # waste upload bytes and can trip stricter OpenAI-compatible servers. Enforced
  199. # client-side (truncate with a warning, never error), ~4 chars/token.
  200. _WHISPER_PROMPT_TOKEN_CAP = 224
  201. _PROMPT_CHARS_PER_TOKEN = CHARS_PER_TOKEN
  202. _WHISPER_PROMPT_CAPPED_PROVIDERS = frozenset({"local", "openai", "groq", "deepinfra"})
  203. def _enforce_prompt_length_limit(prompt: Optional[str], provider: str) -> Optional[str]:
  204. """Truncate *prompt* to the whisper-family token cap, keeping the TAIL (whisper conditions
  205. on the final context window, so the newest hints survive). Other providers self-validate."""
  206. max_chars = _WHISPER_PROMPT_TOKEN_CAP * _PROMPT_CHARS_PER_TOKEN
  207. if not prompt or provider not in _WHISPER_PROMPT_CAPPED_PROVIDERS or len(prompt) <= max_chars:
  208. return prompt
  209. logger.warning(
  210. "Transcription prompt is ~%d tokens; whisper-family provider '%s' "
  211. "only uses the final ~%d — truncating to the last %d characters.",
  212. len(prompt) // _PROMPT_CHARS_PER_TOKEN, provider, _WHISPER_PROMPT_TOKEN_CAP, max_chars)
  213. return prompt[-max_chars:]
  214. def _apply_pre_transcription_hook(
  215. *, file_path: str, provider: str, model: Optional[str], language: Optional[str],
  216. prompt: Optional[str], source: Optional[str],
  217. ) -> tuple[Optional[str], Optional[str], Optional[str]]:
  218. """Fire the ``pre_transcription`` plugin hook; returns ``(model, language_override, prompt)``.
  219. Gated on ``has_hook`` (the no-hook path never builds kwargs) and fail-open: any plumbing error
  220. leaves the dispatch untouched. Results apply field-by-field in registration order (last hook
  221. wins). ``language_override`` is None unless a hook explicitly set ``language``, so backends keep
  222. their own config/env resolution."""
  223. try:
  224. from hermes_cli.plugins import has_hook, invoke_hook
  225. if not has_hook("pre_transcription"):
  226. return model, None, prompt
  227. hook_results = invoke_hook(
  228. "pre_transcription", file_path=file_path, provider=provider,
  229. model=model, language=language, prompt=prompt, source=source)
  230. overrides: dict[str, Any] = {}
  231. for hook_result in hook_results:
  232. for key, value in (hook_result.items() if isinstance(hook_result, dict) else ()):
  233. if key == "file_path":
  234. logger.warning(
  235. "pre_transcription hook attempted to change "
  236. "file_path (read-only) — ignoring the attempt.")
  237. elif key not in _PRE_TRANSCRIPTION_MUTABLE_FIELDS:
  238. logger.debug("pre_transcription hook returned unsupported field %r — ignoring.", key)
  239. elif not isinstance(value, str):
  240. logger.debug(
  241. "pre_transcription hook returned non-string value %r for field %r — ignoring.", value, key,
  242. )
  243. else:
  244. overrides[key] = value
  245. # Hooks win over the static ``stt.prompt`` config; "" clears it.
  246. if "prompt" in overrides:
  247. prompt = overrides["prompt"] or None
  248. return overrides.get("model", model), overrides.get("language") or None, prompt
  249. except Exception as _hook_err:
  250. logger.debug("pre_transcription hook error: %s", _hook_err)
  251. return model, None, prompt