transcription_audio.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279
  1. """Audio preprocessing for STT.
  2. Binary discovery, the shared ffmpeg m4a encode (transcode + silence trim),
  3. source/format validation, WeChat .silk decoding, CAF conversion and the
  4. best-effort cloud pre-upload silence trim. Facade-owned state (``_HAS_PILK``,
  5. ``_safe_find_spec``) is read lazily from ``tools.transcription_tools``.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import os
  10. import shutil
  11. import subprocess
  12. import tempfile
  13. from functools import partial
  14. from pathlib import Path
  15. from typing import Any, Dict, Optional
  16. from hermes_cli._subprocess_compat import windows_hide_flags
  17. from utils import is_truthy_value
  18. from tools.transcription_common import (
  19. COMMON_LOCAL_BIN_DIRS, LOCAL_NATIVE_AUDIO_FORMATS, MAX_FILE_SIZE, SUPPORTED_FORMATS,
  20. _config_number, _error_result, _lazy_ensure_quietly, _process_error_detail)
  21. # Log-record parity with the origin module.
  22. logger = logging.getLogger("tools.transcription_tools")
  23. def _find_binary(binary_name: str) -> Optional[str]:
  24. """Find a local binary, checking common Homebrew/local prefixes as well as PATH."""
  25. for directory in COMMON_LOCAL_BIN_DIRS:
  26. candidate = Path(directory) / binary_name
  27. if candidate.exists() and os.access(candidate, os.X_OK):
  28. return str(candidate)
  29. return shutil.which(binary_name)
  30. _find_ffmpeg_binary = partial(_find_binary, "ffmpeg")
  31. _find_ffprobe_binary = partial(_find_binary, "ffprobe")
  32. _find_whisper_binary = partial(_find_binary, "whisper")
  33. def _run_quiet(command: list, *, timeout: float, env: Optional[dict] = None) -> subprocess.CompletedProcess:
  34. """``subprocess.run`` for STT helper binaries: checked, captured, utf-8 text, no stdin, hidden window."""
  35. return subprocess.run(
  36. command, check=True, capture_output=True, text=True,
  37. encoding="utf-8", errors="replace", timeout=timeout,
  38. stdin=subprocess.DEVNULL, env=env, creationflags=windows_hide_flags())
  39. # Shared encode profile for every STT-bound m4a (transcode and silence-trim):
  40. # 16 kHz mono 32 kbps AAC, faststart. One owner so codec/bitrate never drift.
  41. _STT_M4A_ENCODE_ARGS = ("-vn", "-ac", "1", "-ar", "16000", "-c:a", "aac", "-b:a", "32k", "-movflags", "+faststart")
  42. def _run_ffmpeg_stt_encode(ffmpeg: str, input_path: str, output_path: str, *, audio_filter: Optional[str] = None) -> None:
  43. """Run the shared STT m4a encode, optionally with an ``-af`` filter. Raises on failure; callers own the semantics."""
  44. filter_args = ["-af", audio_filter] if audio_filter else []
  45. _run_quiet([ffmpeg, "-y", "-i", input_path, *filter_args, *_STT_M4A_ENCODE_ARGS, output_path],
  46. timeout=120)
  47. def _transcode_audio_for_stt(file_path: str, work_dir: str) -> tuple[Optional[str], Optional[str]]:
  48. """Transcode to a compact 16 kHz mono AAC/m4a for STT upload; ``(converted_path, None)`` or ``(None, error)``.
  49. Newer OpenAI models reject containers ``whisper-1`` accepted (notably Ogg/Opus voice notes) and
  50. gateway downloads may carry a misleading extension."""
  51. ffmpeg = _find_ffmpeg_binary()
  52. if not ffmpeg:
  53. return None, "audio needs transcoding for the STT API, but ffmpeg was not found"
  54. converted_path = os.path.join(work_dir, f"{Path(file_path).stem or 'audio'}-stt.m4a")
  55. try:
  56. _run_ffmpeg_stt_encode(ffmpeg, file_path, converted_path)
  57. return converted_path, None
  58. except subprocess.CalledProcessError as exc:
  59. details = _process_error_detail(exc)
  60. logger.error("ffmpeg STT transcode failed for %s: %s", file_path, details)
  61. return None, f"failed to transcode audio for the STT API: {details}"
  62. except Exception as exc:
  63. logger.error("unexpected STT transcode failure for %s: %s", file_path, exc, exc_info=True)
  64. return None, f"failed to transcode audio for the STT API: {exc}"
  65. def _validate_audio_file_size(audio_path: Path, *, enforce_size_limit: bool = True) -> Optional[dict[str, Any]]:
  66. """Return an error when *audio_path* is inaccessible or (if enforced) exceeds the remote upload cap."""
  67. try:
  68. file_size = audio_path.stat().st_size
  69. except OSError as e:
  70. return _error_result(f"Failed to access file: {e}")
  71. if enforce_size_limit and file_size > MAX_FILE_SIZE:
  72. return _error_result(f"File too large: {file_size / (1024*1024):.1f}MB (max {MAX_FILE_SIZE / (1024*1024):.0f}MB)")
  73. return None
  74. def _validate_audio_source_file(file_path: str, *, enforce_size_limit: bool = True) -> Optional[dict[str, Any]]:
  75. """Validate source path safety (and optionally size) before any decoder runs."""
  76. audio_path = Path(file_path)
  77. if os.path.islink(audio_path):
  78. return _error_result(f"Path is a symbolic link: {file_path}")
  79. if not audio_path.exists():
  80. return _error_result(f"Audio file not found: {file_path}")
  81. if not audio_path.is_file():
  82. return _error_result(f"Path is not a file: {file_path}")
  83. return _validate_audio_file_size(audio_path, enforce_size_limit=enforce_size_limit)
  84. def _validate_audio_file(file_path: str, *, enforce_size_limit: bool = True) -> Optional[dict[str, Any]]:
  85. """Validate a supported, decoder-safe audio file."""
  86. source_error = _validate_audio_source_file(file_path, enforce_size_limit=enforce_size_limit)
  87. suffix = Path(file_path).suffix
  88. if source_error or suffix.lower() in SUPPORTED_FORMATS:
  89. return source_error
  90. return _error_result(f"Unsupported format: {suffix}. Supported: {', '.join(sorted(SUPPORTED_FORMATS))}")
  91. def _prepare_audio_for_transcription(file_path: str) -> tuple[Optional[str], Optional[str], Optional[dict[str, Any]]]:
  92. """Convert a decoder-safe .silk source to a temporary supported WAV file."""
  93. from tools.transcription_tools import _HAS_PILK, _safe_find_spec
  94. audio_path = Path(file_path)
  95. if audio_path.suffix.lower() != ".silk":
  96. return file_path, None, None
  97. if not _HAS_PILK:
  98. # pilk is a tiny silk-v3 codec binding — lazy-installed on first .silk voice note.
  99. _lazy_ensure_quietly("silk")
  100. if not _safe_find_spec("pilk"):
  101. return None, None, _error_result(
  102. "Unsupported format: .silk. Install the optional 'pilk' dependency to enable WeChat voice transcription."
  103. )
  104. temp_dir = tempfile.mkdtemp(prefix="hermes-silk-")
  105. converted_path = os.path.join(temp_dir, f"{audio_path.stem}.wav")
  106. try:
  107. import pilk
  108. pilk.silk_to_wav(file_path, converted_path)
  109. if not Path(converted_path).is_file() or Path(converted_path).stat().st_size == 0:
  110. raise RuntimeError("pilk did not produce a readable WAV file")
  111. return converted_path, temp_dir, None
  112. except Exception as exc:
  113. shutil.rmtree(temp_dir, ignore_errors=True)
  114. logger.error("Failed to convert .silk audio %s: %s", file_path, exc, exc_info=True)
  115. return None, None, _error_result(f"Failed to convert .silk audio for transcription: {exc}")
  116. def _prepare_local_audio(file_path: str, work_dir: str) -> tuple[Optional[str], Optional[str]]:
  117. """Normalize audio for local CLI STT when needed."""
  118. audio_path = Path(file_path)
  119. if audio_path.suffix.lower() in LOCAL_NATIVE_AUDIO_FORMATS:
  120. return file_path, None
  121. ffmpeg = _find_ffmpeg_binary()
  122. if not ffmpeg:
  123. return None, "Local STT fallback requires ffmpeg for non-WAV inputs, but ffmpeg was not found"
  124. converted_path = os.path.join(work_dir, f"{audio_path.stem}.wav")
  125. try:
  126. _run_quiet([ffmpeg, "-y", "-i", file_path, converted_path], timeout=300)
  127. return converted_path, None
  128. except subprocess.TimeoutExpired:
  129. logger.error("ffmpeg conversion timed out for %s", file_path)
  130. return None, "Audio conversion for local STT timed out"
  131. except subprocess.CalledProcessError as e:
  132. details = _process_error_detail(e)
  133. logger.error("ffmpeg conversion failed for %s: %s", file_path, details)
  134. return None, f"Failed to convert audio for local STT: {details}"
  135. def _convert_caf_to_wav(file_path: str, work_dir: str) -> Optional[str]:
  136. """Convert CAF to WAV in a caller-owned directory using ffmpeg or afconvert."""
  137. audio_path = Path(file_path)
  138. wav_path = os.path.join(work_dir, f"{audio_path.stem}.wav")
  139. ffmpeg = _find_ffmpeg_binary()
  140. afconvert = shutil.which("afconvert")
  141. candidates = (
  142. ("ffmpeg", [ffmpeg, "-y", "-i", file_path, wav_path] if ffmpeg else None),
  143. ("afconvert", [afconvert, file_path, wav_path, "-d", "LEI16", "-f", "WAVE"] if afconvert else None),
  144. )
  145. for label, command in ((label, cmd) for label, cmd in candidates if cmd):
  146. try:
  147. _run_quiet(command, timeout=300)
  148. return wav_path
  149. except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
  150. logger.warning("%s CAF to WAV failed for %s: %s", label, file_path, e)
  151. return None
  152. # ---- Cloud pre-upload silence trim --------------------------------------
  153. #
  154. # Local faster-whisper gets Silero VAD; cloud providers get the raw file, so silence
  155. # is paid for twice (upload + per-minute billing) and cloud Whisper hallucinates on
  156. # it. Before uploading to a built-in cloud provider we collapse long pauses with
  157. # ffmpeg's silenceremove, keeping ``stt.cloud_trim_keep_ms`` of each pause so word
  158. # boundaries survive. Purely best-effort — ANY of these uploads the original:
  159. # ``stt.cloud_trim_silence: false``, ffmpeg/ffprobe missing, trim failure/timeout, a
  160. # ~empty result (the provider, not a dB heuristic, decides "no speech"), or <10%
  161. # saving. Command-type and plugin providers are NOT trimmed: they may wrap local
  162. # CLIs that want the original bytes.
  163. _CLOUD_TRIM_THRESHOLD_DB_DEFAULT = -40 # audio below this level counts as silence
  164. _CLOUD_TRIM_KEEP_MS_DEFAULT = 300 # how much of each pause survives the trim
  165. _CLOUD_TRIM_MIN_SAVING = 0.10 # use the trimmed file only when >=10% shorter
  166. _CLOUD_TRIM_MIN_RESULT_SECONDS = 0.3 # all-silence guard floor: never upload ~empty audio
  167. # Below this the trim can't pay for itself (several providers bill a 10s minimum)
  168. # and the encode would sit on the synchronous voice-note path.
  169. _CLOUD_TRIM_MIN_INPUT_SECONDS = 12.0
  170. def _probe_audio_duration(file_path: str) -> Optional[float]:
  171. """Return the audio duration in seconds via ffprobe, or None. Canonical sync probe;
  172. ``gateway/run.py._probe_audio_duration`` and the Telegram adapter carry local variants — keep
  173. the command shape in sync."""
  174. ffprobe = _find_ffprobe_binary()
  175. if not ffprobe:
  176. return None
  177. try:
  178. probe = _run_quiet([ffprobe, "-v", "error", "-show_entries", "format=duration", "-of",
  179. "default=noprint_wrappers=1:nokey=1", file_path], timeout=30)
  180. return float(probe.stdout.strip())
  181. except Exception:
  182. return None
  183. def _cloud_trim_settings(stt_config: dict[str, Any]) -> tuple[bool, int, int]:
  184. """Resolve (enabled, threshold_db, keep_ms) for the cloud silence trim."""
  185. cfg = stt_config if isinstance(stt_config, dict) else {}
  186. # is_truthy_value: a YAML string "false" must disable, exactly like is_stt_enabled.
  187. enabled = is_truthy_value(cfg.get("cloud_trim_silence", True), default=True)
  188. threshold_db = _config_number(cfg, "cloud_trim_threshold_db", _CLOUD_TRIM_THRESHOLD_DB_DEFAULT, int)
  189. keep_ms = _config_number(cfg, "cloud_trim_keep_ms", _CLOUD_TRIM_KEEP_MS_DEFAULT, int)
  190. return enabled, threshold_db, max(keep_ms, 0)
  191. def _trim_silence_for_cloud_stt(file_path: str, stt_config: dict[str, Any]) -> Optional[str]:
  192. """Return a silence-trimmed copy of *file_path* for cloud upload, or None (= upload the original).
  193. On success the caller owns deleting the returned file's parent directory."""
  194. enabled, threshold_db, keep_ms = _cloud_trim_settings(stt_config)
  195. if not enabled:
  196. return None
  197. ffmpeg = _find_ffmpeg_binary()
  198. if not ffmpeg:
  199. logger.debug("Cloud STT silence trim skipped: ffmpeg not found")
  200. return None
  201. original_duration = _probe_audio_duration(file_path)
  202. if not original_duration or original_duration <= 0:
  203. logger.debug("Cloud STT silence trim skipped: could not probe %s", file_path)
  204. return None
  205. name = Path(file_path).name
  206. if original_duration < _CLOUD_TRIM_MIN_INPUT_SECONDS:
  207. logger.debug("Cloud STT silence trim skipped for %s: %.1fs is below the %.0fs gate",
  208. name, original_duration, _CLOUD_TRIM_MIN_INPUT_SECONDS)
  209. return None
  210. keep_seconds = keep_ms / 1000.0
  211. # start_periods=1 strips leading silence; stop_periods=-1 collapses every interior/trailing silence.
  212. filter_expr = (
  213. f"silenceremove="
  214. f"start_periods=1:start_threshold={threshold_db}dB:start_silence={keep_seconds}:"
  215. f"stop_periods=-1:stop_threshold={threshold_db}dB:stop_silence={keep_seconds}")
  216. work_dir = tempfile.mkdtemp(prefix="hermes-stt-trim-")
  217. trimmed_path = os.path.join(work_dir, f"{Path(file_path).stem or 'audio'}-trimmed.m4a")
  218. # Scale the all-silence guard with keep_ms: output that is solely kept pause must never upload as "speech".
  219. min_result_seconds = max(_CLOUD_TRIM_MIN_RESULT_SECONDS, 2 * keep_seconds)
  220. keep_result = False
  221. try:
  222. _run_ffmpeg_stt_encode(ffmpeg, file_path, trimmed_path, audio_filter=filter_expr)
  223. trimmed_duration = _probe_audio_duration(trimmed_path)
  224. if not trimmed_duration or trimmed_duration < min_result_seconds:
  225. logger.debug("Cloud STT silence trim discarded for %s: trimmed result ~empty (%.2fs)",
  226. name, trimmed_duration or 0.0)
  227. return None
  228. if trimmed_duration > original_duration * (1 - _CLOUD_TRIM_MIN_SAVING):
  229. logger.debug("Cloud STT silence trim discarded for %s: saves <%.0f%% (%.1fs -> %.1fs)",
  230. name, _CLOUD_TRIM_MIN_SAVING * 100, original_duration, trimmed_duration)
  231. return None
  232. logger.info("Trimmed silence from %s before cloud STT upload (%.1fs -> %.1fs, -%d%%)", name,
  233. original_duration, trimmed_duration, round((1 - trimmed_duration / original_duration) * 100))
  234. keep_result = True
  235. return trimmed_path
  236. except Exception as exc:
  237. logger.debug("Cloud STT silence trim failed for %s: %s", file_path, exc)
  238. return None
  239. finally:
  240. if not keep_result:
  241. shutil.rmtree(work_dir, ignore_errors=True)