voice_mode.py 69 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549
  1. """Voice Mode -- push-to-talk recording and playback for the CLI.
  2. Capture via sounddevice, WAV via stdlib wave, STT via tools.transcription_tools,
  3. playback via sounddevice or system players. Optional deps: the ``audio-io`` / ``stt-whisper``
  4. extras, installed through PM (``hermes tools`` configures speech-to-text).
  5. """
  6. import logging
  7. import math
  8. import os
  9. import platform
  10. import shlex
  11. import shutil
  12. import subprocess
  13. import sys
  14. from collections import deque
  15. from contextlib import suppress
  16. from pathlib import Path
  17. import tempfile
  18. import threading
  19. import time
  20. import wave
  21. from typing import Any, Callable, Dict, List, Optional
  22. logger = logging.getLogger(__name__)
  23. from hermes_constants import is_termux as _is_termux_environment
  24. from hermes_platform.host.runtime import is_wsl
  25. from tools.voice_mode_transcript import _voice_config, is_voice_stop_phrase, is_whisper_hallucination
  26. # ── Recording parameters ──
  27. SAMPLE_RATE = 16000 # Whisper native rate
  28. CHANNELS = 1
  29. DTYPE = "int16"
  30. SAMPLE_WIDTH = 2 # bytes per sample (int16)
  31. SILENCE_RMS_THRESHOLD = 200 # RMS below this = silence (int16 range 0-32767)
  32. SILENCE_DURATION_SECONDS = 3.0 # continuous silence before auto-stop
  33. _TEMP_DIR = os.path.join(tempfile.gettempdir(), "hermes_voice")
  34. # ── Lazy audio imports ──
  35. # Never imported at module level: crashes headless environments (SSH, Docker,
  36. # WSL, no PortAudio).
  37. def _import_audio():
  38. """Lazy-import (sounddevice, numpy), enabling the ``audio-io`` extra through PM first.
  39. Raises ImportError when the extra cannot be enabled here (lazy installs off, platform
  40. gate, or installed-but-needs-restart) and OSError when PortAudio's shared library is
  41. missing — pip can't fix that one, so it is reported separately.
  42. """
  43. import pm
  44. if not pm.available("audio-io"):
  45. try:
  46. pm.ensure_import("audio-io")
  47. except pm.InstallError as exc:
  48. raise ImportError(str(exc)) from exc
  49. import sounddevice as sd
  50. import numpy as np
  51. return sd, np
  52. def _sounddevice_output_allowed() -> bool:
  53. """False on macOS: PortAudio/CoreAudio OUTPUT init triggers a kTCCServiceMediaLibrary
  54. prompt, so output goes through ``afplay`` there. Input (recording) is unaffected.
  55. Returns False on macOS: importing/initializing sounddevice (PortAudio/CoreAudio) for output triggers a
  56. kTCCServiceMediaLibrary permission prompt, even though playback needs no media-library access. This does
  57. NOT affect audio *input* (recording), which legitimately needs microphone permission. See PR #62601 /
  58. #13291.
  59. """
  60. return platform.system() != "Darwin"
  61. def _play_int16_via_tempfile(audio, sample_rate: int) -> None:
  62. """Play int16 mono PCM via a temp WAV + play_audio_file (macOS: afplay, no TCC prompt)."""
  63. tmp_path = None
  64. try:
  65. tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
  66. tmp_path = tmp.name
  67. _write_wav_frames(tmp, audio.tobytes(), sample_rate)
  68. play_audio_file(tmp_path)
  69. except Exception as e:
  70. logger.debug("Tone tempfile playback failed: %s", e)
  71. finally:
  72. _unlink_quietly(tmp_path)
  73. def _write_wav_frames(dest, frames: bytes, sample_rate: int) -> None:
  74. """Write raw 16-bit mono PCM *frames* as a WAV to *dest* (path or file object)."""
  75. with wave.open(dest, "wb") as wf:
  76. wf.setnchannels(CHANNELS)
  77. wf.setsampwidth(SAMPLE_WIDTH)
  78. wf.setframerate(sample_rate)
  79. wf.writeframes(frames)
  80. def _unlink_quietly(path: Optional[str]) -> None:
  81. """Best-effort unlink; missing/undeletable files are ignored."""
  82. if path:
  83. with suppress(OSError):
  84. os.unlink(path)
  85. def _audio_unavailable_reason() -> str:
  86. try:
  87. _import_audio()
  88. except ImportError as exc:
  89. return _voice_capture_install_hint(exc)
  90. except OSError:
  91. return _portaudio_missing_message().splitlines()[0]
  92. return ""
  93. def _audio_available() -> bool:
  94. try:
  95. _import_audio()
  96. return True
  97. except (ImportError, OSError):
  98. return False
  99. def _rms(np, data) -> float:
  100. return float(np.sqrt(np.mean(data.astype(np.float64) ** 2)))
  101. def _default_input_samplerate(sd) -> int:
  102. """Default input device rate, else the Whisper-friendly SAMPLE_RATE."""
  103. with suppress(Exception):
  104. info = sd.query_devices(None, "input")
  105. rate = info.get("default_samplerate") if isinstance(info, dict) else getattr(info, "default_samplerate", None)
  106. if isinstance(rate, (int, float)) and rate > 0:
  107. return round(rate)
  108. return SAMPLE_RATE
  109. # ── Environment detection ──
  110. def _voice_capture_install_hint(error: BaseException | None = None) -> str:
  111. """Why audio capture is unavailable. ``_import_audio`` already tried to enable the
  112. ``audio-io`` extra through PM, so the ImportError it raised IS the remediation."""
  113. # On Termux PortAudio is a system package a pip install can't provide (#18432).
  114. if _is_termux_environment():
  115. return "pkg install python-numpy portaudio && python -m pip install sounddevice"
  116. return str(error) if error else "audio-io extra unavailable"
  117. def _portaudio_missing_message() -> str:
  118. """sounddevice imports but PortAudio's .so is missing — pip can't fix that."""
  119. hint = (" Termux: pkg install portaudio" if _is_termux_environment()
  120. else " Linux: sudo apt-get install libportaudio2\n macOS: brew install portaudio")
  121. return f"PortAudio system library not found -- install it first:\n{hint}\nThen retry /voice on."
  122. _TERMUX_APP_MISSING_WARNING = (
  123. "Termux:API Android app is not installed. Install/update the Termux:API app to use termux-microphone-record."
  124. )
  125. def _termux_microphone_command() -> Optional[str]:
  126. return shutil.which("termux-microphone-record") if _is_termux_environment() else None
  127. def _run_quiet(cmd: list[str], *, timeout: float, check: bool) -> subprocess.CompletedProcess:
  128. """subprocess.run with captured, utf-8-decoded output and no stdin."""
  129. return subprocess.run(
  130. cmd, capture_output=True, text=True, encoding='utf-8', errors='replace',
  131. timeout=timeout, check=check, stdin=subprocess.DEVNULL)
  132. # `pm list packages` is canonical, but on some ROMs `pm` isn't on Termux's PATH
  133. # while `cmd package` is, and on others `pm` returns nothing for the calling
  134. # user even when the app is present — so both are tried.
  135. _TERMUX_API_PACKAGE_PROBES = (
  136. ("pm", "list", "packages", "com.termux.api"),
  137. ("cmd", "package", "list", "packages", "com.termux.api"),
  138. )
  139. def _termux_api_app_installed() -> bool:
  140. """True iff the Termux:API Android app is installed.
  141. A probe reporting ``package:com.termux.api`` is authoritative. If EVERY probe
  142. is inconclusive (binary missing, denied, timeout, non-zero exit) trust the
  143. ``termux-microphone-record`` binary on PATH: a false negative blocks ``/voice
  144. on``, a false positive only surfaces a precise runtime error.
  145. """
  146. if not _is_termux_environment():
  147. return False
  148. inconclusive = False
  149. for cmd in _TERMUX_API_PACKAGE_PROBES:
  150. try:
  151. result = _run_quiet(list(cmd), timeout=5, check=False)
  152. except (OSError, subprocess.TimeoutExpired):
  153. result = None
  154. if result is None or result.returncode != 0:
  155. inconclusive = True
  156. continue
  157. if "package:com.termux.api" in (result.stdout or "").lower():
  158. return True
  159. if inconclusive and shutil.which("termux-microphone-record") is not None:
  160. logger.debug(
  161. "Termux package-manager probes inconclusive; trusting "
  162. "termux-microphone-record binary on PATH (issue #31015).")
  163. return True
  164. return False
  165. def _termux_voice_capture_available() -> bool:
  166. return _termux_microphone_command() is not None and _termux_api_app_installed()
  167. def _pulse_socket_candidates() -> list[str]:
  168. """Socket paths a PulseAudio/PipeWire client would try by default."""
  169. env = os.environ.get
  170. # PULSE_SERVER may be "unix:/path", "unix:/path;..." or a bare path.
  171. parts = [p.strip() for p in env('PULSE_SERVER', '').split(';')]
  172. candidates = [p[len('unix:'):] for p in parts if p.startswith('unix:')]
  173. if env('PULSE_RUNTIME_PATH'):
  174. candidates.append(os.path.join(env('PULSE_RUNTIME_PATH'), 'native'))
  175. if env('XDG_RUNTIME_DIR'):
  176. candidates += [os.path.join(env('XDG_RUNTIME_DIR'), 'pulse', 'native'),
  177. os.path.join(env('XDG_RUNTIME_DIR'), 'pipewire-0')]
  178. return [c for c in candidates if c]
  179. def _pulse_socket_reachable() -> bool:
  180. """True if a PulseAudio/PipeWire socket on disk accepts a connection (a stale socket of
  181. a dead server does not count). Covers a local sound server without PULSE_SERVER set.
  182. Covers the common case where a sound server runs locally (e.g. on a remote SSH host) without
  183. ``PULSE_SERVER``/``PIPEWIRE_REMOTE`` being set -- the client just connects to the default socket under
  184. the runtime dir. We look at ``PULSE_SERVER`` unix paths, ``PULSE_RUNTIME_PATH``, and ``XDG_RUNTIME_DIR``
  185. for a ``pulse/native`` or ``pipewire-0`` socket (issue #35622).
  186. """
  187. import socket
  188. import stat
  189. for path in _pulse_socket_candidates():
  190. try:
  191. if not stat.S_ISSOCK(os.stat(path).st_mode):
  192. continue
  193. with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
  194. sock.settimeout(0.5)
  195. sock.connect(path)
  196. return True
  197. except OSError:
  198. continue
  199. return False
  200. def _probe_audio_libraries(warnings: list[str], notices: list[str], *, has_forwarded_audio: bool,
  201. termux_mic_cmd: Optional[str], termux_app_installed: bool) -> None:
  202. """Import sounddevice and query devices; append the outcome to warnings/notices.
  203. Host audio forwarding or Termux:API capture downgrade "no devices" / "query
  204. failed" to notices — in WSL with PulseAudio the query can fail yet audio works.
  205. """
  206. termux_capture = bool(termux_mic_cmd and termux_app_installed)
  207. def outcome(termux_notice, warning, *, forwarded_notice="", import_failed=False):
  208. if forwarded_notice and has_forwarded_audio:
  209. notices.append(forwarded_notice)
  210. elif termux_capture:
  211. notices.append(termux_notice)
  212. elif import_failed and termux_mic_cmd and not termux_app_installed:
  213. warnings.append(_TERMUX_APP_MISSING_WARNING)
  214. else:
  215. warnings.append(warning)
  216. try:
  217. sd, _ = _import_audio()
  218. except ImportError as exc:
  219. return outcome("Termux:API microphone recording available (sounddevice not required)",
  220. f"Audio libraries not installed ({_voice_capture_install_hint(exc)})", import_failed=True)
  221. except OSError:
  222. return outcome("Termux:API microphone recording available (PortAudio not required)",
  223. _portaudio_missing_message(), import_failed=True)
  224. try:
  225. if sd.query_devices():
  226. return
  227. outcome("No PortAudio devices detected, but Termux:API microphone capture is available",
  228. "No audio input/output devices detected",
  229. forwarded_notice="No PortAudio devices detected but host audio forwarding is configured -- continuing")
  230. except Exception:
  231. outcome("PortAudio device query failed, but Termux:API microphone capture is available",
  232. "Audio subsystem error (PortAudio cannot query devices)",
  233. forwarded_notice="Audio device query failed but host audio forwarding is configured -- continuing")
  234. def detect_audio_environment() -> dict:
  235. """Return ``{'available', 'warnings' (hard-fail, block voice), 'notices' (informational)}``.
  236. SSH, containers and WSL normally have no audio devices, but a reachable sound
  237. server (PulseAudio/PipeWire socket or forwarding env vars) is honored.
  238. """
  239. warnings, notices = [], []
  240. termux_mic_cmd = _termux_microphone_command()
  241. termux_app_installed = _termux_api_app_installed()
  242. has_forwarded_audio = bool(
  243. os.environ.get('PULSE_SERVER') or os.environ.get('PIPEWIRE_REMOTE') or _pulse_socket_reachable())
  244. def report(notice: str, warning: str) -> None:
  245. (notices if has_forwarded_audio else warnings).append(notice if has_forwarded_audio else warning)
  246. # SSH detection -- normally no audio devices, but honor a reachable sound server (PulseAudio/PipeWire
  247. # socket or forwarding env vars), which works fine over SSH (issue #35622).
  248. if any(os.environ.get(v) for v in ('SSH_CLIENT', 'SSH_TTY', 'SSH_CONNECTION')):
  249. report("Running over SSH with a reachable PulseAudio/PipeWire sound server",
  250. "Running over SSH -- no audio devices available.\n"
  251. " If a sound server (PulseAudio/PipeWire) is running on this host,\n"
  252. " point Hermes at it, e.g.:\n"
  253. " export XDG_RUNTIME_DIR=/run/user/$(id -u)\n"
  254. " # or: export PULSE_SERVER=unix:$XDG_RUNTIME_DIR/pulse/native")
  255. # Docker/Podman container detection — honor host audio forwarding. When the user mounts a
  256. # PulseAudio/PipeWire socket into the container and points PULSE_SERVER / PIPEWIRE_REMOTE at it, audio
  257. # works fine (issue #21203). Only block when no forwarding is configured.
  258. from hermes_constants import is_container
  259. if is_container():
  260. report("Running inside container (Docker/Podman/LXC) with host audio forwarding",
  261. "Running inside container (Docker/Podman/LXC) -- no audio devices.\n"
  262. " Forward host audio with one of (substitute $XDG_RUNTIME_DIR for your runtime dir,\n"
  263. " typically /run/user/$UID):\n"
  264. " PulseAudio: -v $XDG_RUNTIME_DIR/pulse/native:$XDG_RUNTIME_DIR/pulse/native \\\n"
  265. " -e PULSE_SERVER=unix:$XDG_RUNTIME_DIR/pulse/native\n"
  266. " PipeWire: -e PIPEWIRE_REMOTE=$XDG_RUNTIME_DIR/pipewire-0")
  267. # WSL: the PowerShell/Media.SoundPlayer fallback only covers OUTPUT, so when
  268. # it is all that's available downgrade to a notice (recording guidance stays
  269. # visible, TTS-only usage isn't blocked).
  270. if is_wsl():
  271. if has_forwarded_audio:
  272. notices.append("Running in WSL with a reachable PulseAudio/PipeWire sound server")
  273. elif _wsl_powershell_tts_available():
  274. notices.append(
  275. "Running in WSL without a PulseAudio bridge -- TTS playback "
  276. "will use the PowerShell/Media.SoundPlayer fallback. "
  277. "Voice INPUT (recording) still requires a PulseAudio bridge:\n"
  278. " 1. Set PULSE_SERVER=unix:/mnt/wslg/PulseServer\n"
  279. " 2. Create ~/.asoundrc pointing ALSA at PulseAudio\n"
  280. " 3. Verify with: arecord -d 3 test.wav && aplay test.wav")
  281. else:
  282. warnings.append(
  283. "Running in WSL -- audio requires a forwarded sound server.\n"
  284. " PulseAudio: export PULSE_SERVER=unix:/mnt/wslg/PulseServer\n"
  285. " PipeWire: export PIPEWIRE_REMOTE=$XDG_RUNTIME_DIR/pipewire-0\n"
  286. " Then verify: arecord -d 3 test.wav && aplay test.wav")
  287. _probe_audio_libraries(warnings, notices, has_forwarded_audio=has_forwarded_audio,
  288. termux_mic_cmd=termux_mic_cmd, termux_app_installed=termux_app_installed)
  289. return {"available": not warnings, "warnings": warnings, "notices": notices}
  290. # ── Audio cues (beep tones) ──
  291. _DEFAULT_BEEP_VOLUME = 0.3
  292. def _get_beep_volume() -> float:
  293. """``voice.beep_volume`` clamped to 0.0-1.0; 0.3 when missing/invalid."""
  294. raw = _voice_config().get("beep_volume", _DEFAULT_BEEP_VOLUME)
  295. try:
  296. volume = float(raw)
  297. except (TypeError, ValueError):
  298. return _DEFAULT_BEEP_VOLUME
  299. if isinstance(raw, bool) or volume < 0.0 or volume > 1.0 or math.isnan(volume):
  300. return _DEFAULT_BEEP_VOLUME
  301. return volume
  302. def _sd_play_blocking(sd, audio, sample_rate: int, *, timeout: float, blocksize: int = 0) -> None:
  303. """``sd.play`` then poll until idle or *timeout* (``sd.wait()`` has no timeout and
  304. hangs forever if the device stalls)."""
  305. sd.play(audio, samplerate=sample_rate, blocksize=blocksize)
  306. from agent.deadline import poll_until
  307. poll_until(lambda: not (sd.get_stream() and sd.get_stream().active), timeout, 0.01)
  308. sd.stop()
  309. def play_beep(frequency: int = 880, duration: float = 0.12, count: int = 1) -> None:
  310. """Play *count* short beeps of *frequency* Hz, *duration* s each. Synthesized with
  311. numpy only (no sounddevice import => no macOS TCC prompt); macOS output via afplay."""
  312. try:
  313. import numpy as np
  314. except ImportError:
  315. return
  316. try:
  317. samples_per_beep = int(SAMPLE_RATE * duration)
  318. gap = np.zeros(int(SAMPLE_RATE * 0.06), dtype=np.int16) # 60ms between beeps
  319. beep_volume = _get_beep_volume()
  320. parts = []
  321. for i in range(count):
  322. t = np.linspace(0, duration, samples_per_beep, endpoint=False)
  323. tone = np.sin(2 * np.pi * frequency * t)
  324. fade_len = min(int(SAMPLE_RATE * 0.01), samples_per_beep // 4) # avoid clicks
  325. tone[:fade_len] *= np.linspace(0, 1, fade_len)
  326. tone[-fade_len:] *= np.linspace(1, 0, fade_len)
  327. parts.append((tone * beep_volume * 32767).astype(np.int16))
  328. if i < count - 1:
  329. parts.append(gap)
  330. audio = np.concatenate(parts)
  331. if not _sounddevice_output_allowed():
  332. _play_int16_via_tempfile(audio, SAMPLE_RATE)
  333. return
  334. try:
  335. sd, _ = _import_audio()
  336. except (ImportError, OSError):
  337. return
  338. _sd_play_blocking(sd, audio, SAMPLE_RATE, timeout=2.0)
  339. except Exception as e:
  340. logger.debug("Beep playback failed: %s", e)
  341. # ── Thinking sound — calm ambient "blub blub" while the agent works ──
  342. # Minutes of silent tool use reads as "it died". Scaled by voice.beep_volume, gated
  343. # by voice.thinking_sound (default on); the host's *should_play* callback decides
  344. # when blips are allowed, using the output ref-count below as its signal.
  345. _audio_output_active_count = 0
  346. _audio_output_lock = threading.Lock()
  347. def mark_audio_output_active(active: bool) -> None:
  348. """Ref-count real audio output; playback paths bracket with (True)/(False) so
  349. ``is_audio_output_active()`` means speech is leaving the speakers RIGHT NOW —
  350. unlike per-turn TTS-done events, which stay 'busy' while waiting for text."""
  351. global _audio_output_active_count
  352. with _audio_output_lock:
  353. _audio_output_active_count = max(0, _audio_output_active_count + (1 if active else -1))
  354. def is_audio_output_active() -> bool:
  355. """True while TTS/file audio is actually playing on the speakers."""
  356. with _audio_output_lock:
  357. return _audio_output_active_count > 0
  358. _thinking_lock = threading.Lock()
  359. _thinking_stop: Optional[threading.Event] = None
  360. def thinking_sound_enabled() -> bool:
  361. """Config gate: ``voice.thinking_sound`` (default True)."""
  362. try:
  363. from utils import is_truthy_value
  364. return is_truthy_value(_voice_config().get("thinking_sound", True), default=True)
  365. except Exception:
  366. return True
  367. def _synth_thinking_blip(np, frequency: float) -> Any:
  368. """One soft 'blub': short sine with a downward glide and a click-free envelope."""
  369. duration = 0.16
  370. n = int(SAMPLE_RATE * duration)
  371. t = np.linspace(0, duration, n, endpoint=False)
  372. glide = np.linspace(1.0, 0.72, n) # water-drop feel: freq → 0.72*freq
  373. phase = 2 * np.pi * np.cumsum(frequency * glide) / SAMPLE_RATE
  374. tone = 0.8 * np.sin(phase) + 0.2 * np.sin(phase / 2.0) # octave-down softens harmonics
  375. attack = int(0.02 * SAMPLE_RATE)
  376. env = np.ones(n)
  377. env[:attack] = np.linspace(0.0, 1.0, attack)
  378. env *= np.exp(-t * 14.0)
  379. volume = _get_beep_volume() * 0.5 # deliberately quieter than the beeps
  380. return (tone * env * volume * 32767).astype(np.int16)
  381. def _thinking_sound_loop(stop: threading.Event, should_play) -> None:
  382. """Daemon loop: alternating-pitch blips every ~0.8-1.2s until *stop*; skips (without
  383. stopping) while *should_play* is False. macOS exits immediately: sounddevice output
  384. is TCC-gated and per-second afplay churn is worse than silence."""
  385. if not _sounddevice_output_allowed():
  386. return
  387. try:
  388. sd, np = _import_audio()
  389. except (ImportError, OSError):
  390. return
  391. import random
  392. blips = [_synth_thinking_blip(np, p) for p in (392.0, 329.6)] # G4 / E4
  393. i = 0
  394. while not stop.is_set():
  395. try:
  396. if should_play is None or should_play():
  397. blip = blips[i % len(blips)]
  398. i += 1
  399. sd.play(blip, samplerate=SAMPLE_RATE)
  400. stop.wait(len(blip) / SAMPLE_RATE + 0.02)
  401. sd.stop()
  402. except Exception as e:
  403. logger.debug("Thinking sound blip failed: %s", e)
  404. return
  405. stop.wait(0.8 + random.random() * 0.4)
  406. def start_thinking_sound(should_play=None) -> bool:
  407. """Start the ambient thinking sound (idempotent); *should_play* is polled before each
  408. blip. True when running (or already running), False when disabled/unavailable."""
  409. global _thinking_stop
  410. if not thinking_sound_enabled():
  411. return False
  412. with _thinking_lock:
  413. if _thinking_stop is not None and not _thinking_stop.is_set():
  414. return True
  415. stop = threading.Event()
  416. _thinking_stop = stop
  417. threading.Thread(target=_thinking_sound_loop, args=(stop, should_play),
  418. daemon=True, name="voice-thinking-sound").start()
  419. return True
  420. def stop_thinking_sound() -> None:
  421. """Stop the ambient thinking sound instantly (idempotent)."""
  422. global _thinking_stop
  423. with _thinking_lock:
  424. stop, _thinking_stop = _thinking_stop, None
  425. if stop is not None:
  426. stop.set()
  427. # ── Recorders ──
  428. def _new_recording_path(ext: str) -> str:
  429. """Timestamped ``recording_*.<ext>`` path under _TEMP_DIR (created on demand)."""
  430. os.makedirs(_TEMP_DIR, exist_ok=True)
  431. return os.path.join(_TEMP_DIR, f"recording_{time.strftime('%Y%m%d_%H%M%S')}.{ext}")
  432. # WAV path -> live STT session that heard the same take (``stt.streaming``). transcribe_recording
  433. # pops it, so every consumer of a recorder's WAV gets the live transcript with no call-site change.
  434. _LIVE_SESSIONS: dict[str, Any] = {}
  435. _LIVE_SESSIONS_MAX = 8
  436. _LIVE_LOCK = threading.Lock()
  437. def _park_live_session(wav_path: Optional[str], session: Any) -> None:
  438. if session is None:
  439. return
  440. if not wav_path:
  441. session.cancel()
  442. return
  443. session.end_audio() # the provider flushes while the caller is still handling the WAV
  444. with _LIVE_LOCK:
  445. _LIVE_SESSIONS[wav_path] = session
  446. while len(_LIVE_SESSIONS) > _LIVE_SESSIONS_MAX:
  447. _LIVE_SESSIONS.pop(next(iter(_LIVE_SESSIONS))).cancel()
  448. def _take_live_session(wav_path: str) -> Any:
  449. with _LIVE_LOCK:
  450. return _LIVE_SESSIONS.pop(wav_path, None)
  451. class _RecorderBase:
  452. """Lock, recording flag, start time and live RMS shared by both recorder backends."""
  453. def __init__(self) -> None:
  454. self._lock = threading.Lock()
  455. self._recording, self._start_time, self._current_rms = False, 0.0, 0
  456. # Live STT (``stt.streaming``): hosts set ``on_live_partial`` to render partial text.
  457. self.on_live_partial: Optional[Callable[[str], None]] = None
  458. self._live: Any = None
  459. def _open_live_session(self, sample_rate: int) -> None:
  460. from tools.transcription_streaming import open_streaming_session
  461. try:
  462. self._live = open_streaming_session(on_partial=self.on_live_partial)
  463. except Exception:
  464. logger.debug("Live STT session did not open", exc_info=True)
  465. self._live = None
  466. if self._live is not None:
  467. self._live.set_input_rate(sample_rate)
  468. logger.info("Live STT streaming (%s)", self._live.provider)
  469. def _detach_live(self) -> Any:
  470. live, self._live = self._live, None
  471. return live
  472. @property
  473. def is_recording(self) -> bool:
  474. return self._recording
  475. @property
  476. def elapsed_seconds(self) -> float:
  477. return time.monotonic() - self._start_time if self._recording else 0.0
  478. @property
  479. def current_rms(self) -> int:
  480. """Current input RMS level (0-32767), updated each audio chunk."""
  481. return self._current_rms
  482. class TermuxAudioRecorder(_RecorderBase):
  483. """Recorder backend that uses Termux:API microphone capture commands."""
  484. supports_silence_autostop = False
  485. def __init__(self) -> None:
  486. super().__init__()
  487. self._recording_path: Optional[str] = None
  488. def start(self, on_silence_stop=None) -> None:
  489. del on_silence_stop # Termux:API does not expose live silence callbacks.
  490. mic_cmd = _termux_microphone_command()
  491. if not mic_cmd:
  492. raise RuntimeError(
  493. "Termux voice capture requires the termux-api package and app.\n"
  494. "Install with: pkg install termux-api\n"
  495. "Then install/update the Termux:API Android app.")
  496. if not _termux_api_app_installed():
  497. raise RuntimeError(
  498. "Termux voice capture requires the Termux:API Android app.\n"
  499. "Install/update the Termux:API app, then retry /voice on.")
  500. with self._lock:
  501. if self._recording:
  502. return
  503. self._recording_path = _new_recording_path("aac")
  504. command = [mic_cmd, "-f", self._recording_path, "-l", "0", "-e", "aac",
  505. "-r", str(SAMPLE_RATE), "-c", str(CHANNELS)]
  506. try:
  507. _run_quiet(command, timeout=15, check=True)
  508. except subprocess.CalledProcessError as e:
  509. details = (e.stderr or e.stdout or str(e)).strip()
  510. raise RuntimeError(f"Termux microphone start failed: {details}") from e
  511. except Exception as e:
  512. raise RuntimeError(f"Termux microphone start failed: {e}") from e
  513. with self._lock:
  514. self._start_time, self._recording, self._current_rms = time.monotonic(), True, 0
  515. logger.info("Termux voice recording started")
  516. def _stop_termux_recording(self) -> None:
  517. mic_cmd = _termux_microphone_command()
  518. if mic_cmd:
  519. _run_quiet([mic_cmd, "-q"], timeout=15, check=False)
  520. def _reset_state(self) -> tuple:
  521. """Clear recording state under the lock; return (was_recording, path, started_at)."""
  522. with self._lock:
  523. was_recording, path, started_at = self._recording, self._recording_path, self._start_time
  524. self._recording, self._recording_path, self._current_rms = False, None, 0
  525. return was_recording, path, started_at
  526. def stop(self) -> Optional[str]:
  527. was_recording, path, started_at = self._reset_state()
  528. if not was_recording:
  529. return None
  530. self._stop_termux_recording()
  531. if not path or not os.path.isfile(path):
  532. return None
  533. if time.monotonic() - started_at < 0.3 or os.path.getsize(path) <= 0: # sub-0.3s taps / empty
  534. _unlink_quietly(path)
  535. return None
  536. logger.info("Termux voice recording stopped: %s", path)
  537. return path
  538. def cancel(self) -> None:
  539. _, path, _ = self._reset_state()
  540. with suppress(Exception):
  541. self._stop_termux_recording()
  542. _unlink_quietly(path)
  543. logger.info("Termux voice recording cancelled")
  544. def shutdown(self) -> None:
  545. self.cancel()
  546. class AudioRecorder(_RecorderBase):
  547. """Thread-safe sounddevice.InputStream recorder: ``start(on_silence_stop=cb)`` ...
  548. ``stop()`` -> WAV path or None; ``cancel()`` discards. With a callback the recording
  549. auto-stops after ``silence_duration`` seconds of silence following speech."""
  550. supports_silence_autostop = True
  551. def __init__(self) -> None:
  552. super().__init__()
  553. self._stream: Any = None
  554. self._frames: list[Any] = []
  555. self._sample_rate: int = SAMPLE_RATE
  556. self._on_silence_stop = None
  557. self._silence_threshold: int = SILENCE_RMS_THRESHOLD
  558. self._silence_duration: float = SILENCE_DURATION_SECONDS
  559. self._min_speech_duration: float = 0.3 # seconds above threshold to confirm speech
  560. self._max_dip_tolerance: float = 0.3 # max dip before resetting a speech attempt
  561. self._max_wait: float = 15.0 # seconds to wait for speech before auto-stop
  562. # Hard cap, wired from voice.max_recording_seconds by the CLI before each recording; 0 = none.
  563. self._max_recording_seconds: float = 0.0
  564. self._peak_rms: int = 0 # for the speech-presence check in stop()
  565. self._reset_detection_state()
  566. def _reset_detection_state(self) -> None:
  567. self._has_spoken = False
  568. # speech attempt / its dip / silence run / sustained resume after silence / resume dip
  569. self._speech_start = self._dip_start = self._silence_start = 0.0
  570. self._resume_start = self._resume_dip_start = 0.0
  571. def _max_duration_reached(self, elapsed: float) -> bool:
  572. """``voice.max_recording_seconds`` cap elapsed (<= 0 / unset disables it)."""
  573. cap = self._max_recording_seconds
  574. return bool(cap and cap > 0 and elapsed >= cap)
  575. def _track_speech(self, rms: int, now: float) -> None:
  576. """Advance the speech/dip trackers for one block. Speech is confirmed after
  577. ``_min_speech_duration`` above threshold, tolerating dips < ``_max_dip_tolerance``
  578. (micro-pauses); afterwards only SUSTAINED resumed speech resets the silence timer."""
  579. if rms > self._silence_threshold:
  580. self._dip_start = 0.0
  581. if self._speech_start == 0.0:
  582. self._speech_start = now
  583. elif not self._has_spoken and now - self._speech_start >= self._min_speech_duration:
  584. self._has_spoken = True
  585. logger.debug("Speech confirmed (%.2fs above threshold)", now - self._speech_start)
  586. if not self._has_spoken:
  587. self._silence_start = 0.0
  588. else:
  589. # Resumed speech mirrors initial detection: track, tolerate dips, confirm.
  590. self._resume_dip_start = 0.0
  591. if self._resume_start == 0.0:
  592. self._resume_start = now
  593. elif now - self._resume_start >= self._min_speech_duration:
  594. self._silence_start = 0.0
  595. self._resume_start = 0.0
  596. elif self._has_spoken:
  597. if self._resume_start > 0: # dip-tolerant resume reset
  598. if self._resume_dip_start == 0.0:
  599. self._resume_dip_start = now
  600. elif now - self._resume_dip_start >= self._max_dip_tolerance:
  601. self._resume_start = 0.0
  602. self._resume_dip_start = 0.0
  603. elif self._speech_start > 0:
  604. # Speech attempt dipped; a long enough dip is genuine silence.
  605. if self._dip_start == 0.0:
  606. self._dip_start = now
  607. elif now - self._dip_start >= self._max_dip_tolerance:
  608. logger.debug("Speech attempt reset (dip lasted %.2fs)", now - self._dip_start)
  609. self._speech_start = 0.0
  610. self._dip_start = 0.0
  611. def _should_auto_stop(self, rms: int, now: float) -> bool:
  612. """Spoke then silent for ``_silence_duration``; no speech for ``_max_wait``;
  613. or the hard cap elapsed (independent of speech)."""
  614. elapsed = now - self._start_time
  615. if self._has_spoken and rms <= self._silence_threshold:
  616. if self._silence_start == 0.0:
  617. self._silence_start = now
  618. elif now - self._silence_start >= self._silence_duration:
  619. logger.info("Silence detected (%.1fs), auto-stopping", self._silence_duration)
  620. return True
  621. elif not self._has_spoken and elapsed >= self._max_wait:
  622. logger.info("No speech within %.0fs, auto-stopping", self._max_wait)
  623. return True
  624. if self._max_duration_reached(elapsed):
  625. logger.info("Max recording length reached (%.0fs), auto-stopping", self._max_recording_seconds)
  626. return True
  627. return False
  628. def _fire_silence_callback(self) -> None:
  629. """Invoke ``on_silence_stop`` once, in a daemon thread."""
  630. with self._lock:
  631. cb, self._on_silence_stop = self._on_silence_stop, None # fire only once
  632. if not cb:
  633. return
  634. def _safe_cb():
  635. try:
  636. cb()
  637. except Exception as e:
  638. logger.error("Silence callback failed: %s", e, exc_info=True)
  639. threading.Thread(target=_safe_cb, daemon=True).start()
  640. def _on_audio_block(self, np, indata) -> None:
  641. self._frames.append(indata.copy())
  642. live = self._live
  643. if live is not None:
  644. live.push_audio(indata.tobytes())
  645. rms = int(_rms(np, indata))
  646. self._current_rms = rms
  647. self._peak_rms = max(self._peak_rms, rms)
  648. if self._on_silence_stop is None:
  649. return
  650. now = time.monotonic()
  651. self._track_speech(rms, now)
  652. if self._should_auto_stop(rms, now):
  653. self._fire_silence_callback()
  654. def _ensure_stream(self) -> None:
  655. """Create the audio InputStream and keep it alive while usable.
  656. The stream stays open for the lifetime of the recorder. Between
  657. recordings the callback simply discards audio chunks (``_recording``
  658. is ``False``). This avoids the CoreAudio bug where closing and
  659. re-opening an ``InputStream`` hangs indefinitely on macOS. CoreAudio
  660. can still deactivate the stream when another input stream opens; in
  661. that case the dead object must be closed and rebuilt before capture.
  662. """
  663. if self._stream is not None:
  664. try:
  665. if self._stream.active:
  666. return
  667. except Exception:
  668. logger.debug("Audio input stream liveness probe failed", exc_info=True)
  669. logger.debug("Rebuilding inactive audio input stream")
  670. self._close_stream_with_timeout()
  671. sd, np = _import_audio()
  672. def _callback(indata, frames, time_info, status):
  673. if status:
  674. logger.debug("sounddevice status: %s", status)
  675. if self._recording:
  676. self._on_audio_block(np, indata)
  677. stream = None
  678. for attempt in range(2):
  679. try: # may block on CoreAudio (first call only)
  680. stream = sd.InputStream(samplerate=self._sample_rate, channels=CHANNELS, dtype=DTYPE,
  681. callback=_callback)
  682. stream.start()
  683. break
  684. except Exception as e:
  685. with suppress(Exception):
  686. stream.close()
  687. stream = None
  688. # PortAudio paTimedOut (-9987): a cold host-API bridge (WSLg ALSA->Pulse
  689. # with a SUSPENDED RDP source) missed the 1 s thread-start window. The
  690. # failed open itself wakes the bridge, so one immediate retry succeeds
  691. # where the user's second key press would have (#109303).
  692. if attempt or "timed out" not in str(e).lower():
  693. raise RuntimeError(
  694. f"Failed to open audio input stream: {e}. "
  695. "Check that a microphone is connected and accessible.") from e
  696. logger.info("Audio input stream start timed out; retrying once")
  697. self._stream = stream
  698. def start(self, on_silence_stop=None) -> None:
  699. """Start capturing; *on_silence_stop* is invoked (daemon thread, no args) when
  700. silence follows speech. Raises ``RuntimeError`` if sounddevice/numpy are missing."""
  701. try:
  702. sd, _ = _import_audio()
  703. except OSError as e:
  704. raise RuntimeError(_portaudio_missing_message()) from e
  705. except ImportError as e:
  706. raise RuntimeError(f"Voice mode requires sounddevice and numpy.\n{_voice_capture_install_hint(e)}") from e
  707. with self._lock:
  708. if self._recording:
  709. return
  710. self._frames = []
  711. self._start_time = time.monotonic()
  712. self._reset_detection_state()
  713. self._peak_rms = 0
  714. self._current_rms = 0
  715. self._on_silence_stop = on_silence_stop
  716. self._sample_rate = _default_input_samplerate(sd)
  717. self._ensure_stream()
  718. self._open_live_session(self._sample_rate)
  719. with self._lock:
  720. self._recording = True
  721. logger.info("Voice recording started (rate=%d, channels=%d)", self._sample_rate, CHANNELS)
  722. def _close_stream_with_timeout(self, timeout: float = 3.0) -> None:
  723. """Close the stream with a timeout to prevent CoreAudio hangs."""
  724. if self._stream is None:
  725. return
  726. stream, self._stream = self._stream, None
  727. def _do_close():
  728. with suppress(Exception):
  729. stream.stop()
  730. with suppress(Exception):
  731. stream.close()
  732. t = threading.Thread(target=_do_close, daemon=True)
  733. t.start()
  734. clock = __import__("time") # real clock even when tests patch this module's ``time``
  735. deadline = clock.monotonic() + timeout
  736. while t.is_alive() and clock.monotonic() < deadline: # short joins keep Ctrl+C responsive
  737. t.join(timeout=0.1)
  738. if t.is_alive():
  739. logger.warning("Audio stream close timed out after %.1fs — forcing ahead", timeout)
  740. def stop(self) -> Optional[str]:
  741. """Stop recording (stream stays alive) and return the WAV path, or None if unusable."""
  742. wav_path = self._stop_capture()
  743. _park_live_session(wav_path, self._detach_live())
  744. return wav_path
  745. def _stop_capture(self) -> Optional[str]:
  746. with self._lock:
  747. if not self._recording:
  748. return None
  749. self._recording, self._current_rms = False, 0
  750. if not self._frames:
  751. return None
  752. _, np = _import_audio()
  753. audio_data = np.concatenate(self._frames, axis=0)
  754. self._frames = []
  755. logger.info("Voice recording stopped (%.1fs, %d samples)",
  756. time.monotonic() - self._start_time, len(audio_data))
  757. if len(audio_data) < int(self._sample_rate * 0.3):
  758. logger.debug("Recording too short (%d samples), discarding", len(audio_data))
  759. return None
  760. # Peak RMS, not the average (which trailing silence dilutes). Same
  761. # configured floor as the VAD above — the hardcoded module default
  762. # here discarded valid speech on low-threshold setups (mic peaking
  763. # at RMS ~160 with voice.silence_threshold: 80, #84046).
  764. if self._peak_rms < self._silence_threshold:
  765. logger.info("Recording too quiet (peak RMS=%d < %d), discarding",
  766. self._peak_rms, self._silence_threshold)
  767. return None
  768. return self._write_wav(audio_data, sample_rate=self._sample_rate)
  769. def _discard(self) -> None:
  770. with self._lock:
  771. self._recording, self._frames, self._on_silence_stop, self._current_rms = False, [], None, 0
  772. live = self._detach_live()
  773. if live is not None:
  774. live.cancel()
  775. def cancel(self) -> None:
  776. """Stop recording and discard all captured audio (stream stays alive)."""
  777. self._discard()
  778. logger.info("Voice recording cancelled")
  779. def shutdown(self) -> None:
  780. """Release the audio stream. Call when voice mode is disabled."""
  781. self._discard()
  782. self._close_stream_with_timeout() # outside the lock: avoids deadlock with the callback
  783. logger.info("AudioRecorder shut down")
  784. @staticmethod
  785. def _write_wav(audio_data, *, sample_rate: int = SAMPLE_RATE) -> str:
  786. """Write numpy int16 audio to a WAV file; returns the path."""
  787. wav_path = _new_recording_path("wav")
  788. _write_wav_frames(wav_path, audio_data.tobytes(), sample_rate)
  789. logger.info("WAV written: %s (%d bytes)", wav_path, os.path.getsize(wav_path))
  790. return wav_path
  791. def create_audio_recorder() -> AudioRecorder | TermuxAudioRecorder:
  792. """Return the best recorder backend for the current environment."""
  793. return TermuxAudioRecorder() if _termux_voice_capture_available() else AudioRecorder()
  794. # ── STT dispatch ──
  795. def _live_result(wav_path: str) -> Optional[dict[str, Any]]:
  796. """The live session's transcript for this take, or None to transcribe the WAV instead.
  797. A failed or empty live result falls back to the file: an empty live transcript over a take the
  798. recorder accepted as speech is more likely a dropped socket than silence."""
  799. session = _take_live_session(wav_path)
  800. if session is None:
  801. return None
  802. result = session.finalize()
  803. if result.get("success") and (result.get("transcript") or "").strip():
  804. return result
  805. logger.info("Live STT gave no transcript (%s); transcribing the recording",
  806. result.get("error") or "empty")
  807. return None
  808. def transcribe_recording(wav_path: str, model: Optional[str] = None) -> dict[str, Any]:
  809. """Transcribe a WAV via ``transcribe_audio()``, filtering Whisper hallucinations;
  810. returns ``{success, transcript[, error]}``."""
  811. from tools.transcription_tools import transcribe_audio
  812. result = _live_result(wav_path)
  813. if result is None:
  814. # transcribe_audio fits oversized recordings under the provider's upload cap itself.
  815. result = transcribe_audio(wav_path, model=model, source="voice_mode")
  816. # A configured stop phrase always survives: "bye"/"okay" overlap the
  817. # hallucination blocklist, and swallowing them would make "bye" fail to end the chat.
  818. if result.get("success"):
  819. raw_transcript = result.get("transcript", "")
  820. if is_whisper_hallucination(raw_transcript) and not is_voice_stop_phrase(raw_transcript):
  821. logger.info("Filtered Whisper hallucination: %r", result["transcript"])
  822. return {"success": True, "transcript": "", "filtered": True}
  823. # no_speech = heard no words, not a failure: re-listen quietly instead of
  824. # surfacing "Transcription failed".
  825. if result.get("no_speech"):
  826. return {"success": True, "transcript": "", "no_speech": True}
  827. return result
  828. # ── Audio playback (interruptable) ──
  829. _active_playback: Optional[subprocess.Popen] = None # so stop_playback can interrupt it
  830. _playback_lock = threading.Lock()
  831. def _set_active_playback(proc) -> None:
  832. global _active_playback
  833. with _playback_lock:
  834. _active_playback = proc
  835. def stop_playback() -> None:
  836. """Interrupt the currently playing audio (if any)."""
  837. global _active_playback
  838. with _playback_lock:
  839. proc = _active_playback
  840. _active_playback = None
  841. if proc and proc.poll() is None:
  842. with suppress(Exception):
  843. proc.terminate()
  844. logger.info("Audio playback interrupted")
  845. with suppress(Exception): # also stop sounddevice playback if active
  846. sd, _ = _import_audio()
  847. sd.stop()
  848. def _wsl_powershell_tts_available() -> bool:
  849. """WSL2 PowerShell TTS fallback usable. OUTPUT only (Media.SoundPlayer on the host) —
  850. recording still needs a PulseAudio bridge, so callers keep surfacing that guidance."""
  851. return bool(is_wsl() and shutil.which("powershell.exe") and shutil.which("ffmpeg"))
  852. def play_audio_file(file_path: str) -> bool:
  853. """Play an audio file; True on success. WAV via ``sounddevice.play()`` when allowed,
  854. else system players: afplay (macOS), WSL2 PowerShell bridge, ffplay, aplay (Linux).
  855. Interruptible via ``stop_playback()``."""
  856. mark_audio_output_active(True) # ref-count real speaker output for the whole call
  857. try:
  858. return _play_audio_file_impl(file_path)
  859. finally:
  860. mark_audio_output_active(False)
  861. def _play_wav_via_sounddevice(file_path: str) -> bool:
  862. """Play a WAV through sounddevice; False when unavailable/failed (caller falls through)."""
  863. try:
  864. sd, np = _import_audio()
  865. with wave.open(file_path, "rb") as wf:
  866. frames = wf.readframes(wf.getnframes())
  867. audio_data = np.frombuffer(frames, dtype=np.int16)
  868. sample_rate = wf.getframerate()
  869. # WSLg RDP audio needs a warmup to avoid crackling: the RDP channel takes
  870. # ~100 ms to stabilise and the small default blocksize worsens
  871. # clock-adjustment jitter (microsoft/wslg#1257).
  872. blocksize = 0 # default (auto)
  873. if is_wsl():
  874. fade_samples = int(0.1 * sample_rate)
  875. audio_float = audio_data.astype(np.float64)
  876. audio_float[:fade_samples] *= np.linspace(0.0, 1.0, fade_samples, dtype=np.float64)
  877. audio_data = np.concatenate([
  878. np.zeros(int(0.1 * sample_rate), dtype=np.int16), audio_float.astype(np.int16),
  879. np.zeros(int(0.05 * sample_rate), dtype=np.int16)])
  880. blocksize = 4096
  881. _sd_play_blocking(sd, audio_data, sample_rate,
  882. timeout=len(audio_data) / sample_rate + 2.0, blocksize=blocksize)
  883. return True
  884. except (ImportError, OSError):
  885. return False
  886. except Exception as e:
  887. logger.debug("sounddevice playback failed: %s", e)
  888. return False
  889. def _wsl_powershell_player_cmd(file_path: str) -> Optional[list[str]]:
  890. """WSL2 PowerShell fallback player command, or None. Without a PulseAudio bridge
  891. ffplay/aplay have no device, but Media.SoundPlayer on the host does: convert to a
  892. uniquely-named WAV in Windows %TEMP% (concurrent TTS must not collide), play, always
  893. delete, and re-raise the ORIGINAL exit status past the cleanup (rm -f exits 0)."""
  894. if not (shutil.which("powershell.exe") and shutil.which("ffmpeg") and is_wsl()):
  895. return None
  896. try:
  897. import uuid
  898. def _out(cmd):
  899. return subprocess.check_output(cmd, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
  900. timeout=3).decode(errors="replace").strip()
  901. win_tmp_wsl = _out(["wslpath", "-u", _out(["cmd.exe", "/c", "echo %TEMP%"])])
  902. if not win_tmp_wsl:
  903. return None
  904. wsl_wav = os.path.join(win_tmp_wsl, f"hermes-tts-{uuid.uuid4().hex[:8]}.wav")
  905. win_wav = _out(["wslpath", "-w", wsl_wav])
  906. if not win_wav:
  907. return None
  908. win_wav_safe = win_wav.replace("'", "''")
  909. ps_script = f"(New-Object Media.SoundPlayer '{win_wav_safe}').PlaySync()"
  910. ps_cmd = " && ".join([
  911. shlex.join(["ffmpeg", "-i", file_path, "-f", "wav", wsl_wav, "-loglevel", "quiet", "-y"]),
  912. shlex.join(["powershell.exe", "-NoProfile", "-Command", ps_script])])
  913. # /bin/sh full path so the which(cmd[0]) check in the player loop passes.
  914. return ["/bin/sh", "-c", f"( {ps_cmd} ); rc=$?; {shlex.join(['rm', '-f', wsl_wav])}; exit $rc"]
  915. except Exception:
  916. return None # WSL path resolution failed; fall through to ffplay/aplay
  917. def _system_player_candidates(file_path: str) -> list[list[str]]:
  918. """Ordered system-player commands for this platform."""
  919. system = platform.system()
  920. players: list[list[str]] = [["afplay", file_path]] if system == "Darwin" else []
  921. ps_cmd = _wsl_powershell_player_cmd(file_path) if system == "Linux" else None
  922. if ps_cmd:
  923. players.append(ps_cmd)
  924. players.append(["ffplay", "-nodisp", "-autoexit", "-loglevel", "quiet", file_path])
  925. if system == "Linux":
  926. players.append(["aplay", "-q", file_path])
  927. return players
  928. def _run_system_player(cmd: list[str]) -> bool:
  929. """Run one player to completion (interruptible via stop_playback)."""
  930. proc = None
  931. try:
  932. # Sibling of the TTS/STT credential scrub: players must not inherit tokens/keys.
  933. # See #56332, #70342.
  934. from tools.environments.local import hermes_subprocess_env
  935. proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
  936. env=hermes_subprocess_env(inherit_credentials=False))
  937. _set_active_playback(proc)
  938. proc.wait(timeout=300)
  939. rc = proc.returncode
  940. if rc == 0:
  941. return True
  942. # e.g. WSL ffplay/aplay with no audio device — fall through to the next player.
  943. logger.debug("System player %s exited with code %d, trying next", cmd[0], rc)
  944. except subprocess.TimeoutExpired:
  945. logger.warning("System player %s timed out, killing process", cmd[0])
  946. if proc is not None:
  947. proc.kill()
  948. proc.wait()
  949. except Exception as e:
  950. logger.debug("System player %s failed: %s", cmd[0], e)
  951. finally:
  952. _set_active_playback(None)
  953. return False
  954. def _play_audio_file_impl(file_path: str) -> bool:
  955. if not os.path.isfile(file_path):
  956. logger.warning("Audio file not found: %s", file_path)
  957. return False
  958. # macOS skips sounddevice output (TCC media-library prompt); afplay handles all formats.
  959. if file_path.endswith(".wav") and _sounddevice_output_allowed() and _play_wav_via_sounddevice(file_path):
  960. return True
  961. for cmd in _system_player_candidates(file_path):
  962. if shutil.which(cmd[0]) and _run_system_player(cmd):
  963. return True
  964. logger.warning("No audio player available for %s", file_path)
  965. return False
  966. # ── Barge-in — detect the user speaking over TTS playback ──
  967. # Public API kept from main: the per-playback barge monitor (rolling-floor VAD). No in-tree
  968. # caller since full_duplex_listen took over the voice path, but plugins may import it.
  969. def listen_for_speech(
  970. should_stop: Callable[[], bool],
  971. threshold: Optional[int] = None,
  972. sustained_ms: int = 300,
  973. calibration_ms: int = 400,
  974. capture: bool = False,
  975. on_trigger: Optional[Callable[[], None]] = None,
  976. pre_roll_ms: int = 1200,
  977. endpoint_silence_ms: int = 1250,
  978. max_utterance_ms: int = 30_000,
  979. ):
  980. """Block until sustained speech is heard on the mic, or *should_stop*.
  981. Barge-in monitor: run in a side thread while TTS is playing. Without
  982. *capture* it returns ``True`` when the user started talking (cut playback).
  983. With ``capture=True`` it ALSO records the interruption — a rolling
  984. *pre_roll_ms* buffer means the utterance is kept from its first syllable,
  985. not from the moment detection tripped — and keeps rolling until the user
  986. goes quiet for *endpoint_silence_ms*, then returns the WAV path (or
  987. ``None`` if speech never tripped). *on_trigger* fires at the moment of
  988. detection so the caller can stop playback while capture continues.
  989. The noise floor is calibrated from the first *calibration_ms* of input —
  990. playback is already audible then, so speaker bleed is baked into the
  991. floor and only louder-than-playback speech trips the trigger. Requiring
  992. *sustained_ms* of consecutive above-threshold blocks filters out coughs,
  993. keyboard thumps, and playback transients.
  994. """
  995. try:
  996. sd, np = _import_audio()
  997. except (ImportError, OSError):
  998. return None if capture else False
  999. from collections import deque
  1000. block = int(SAMPLE_RATE * 0.03) # 30ms blocks
  1001. calib_blocks = max(1, calibration_ms // 30)
  1002. trip_blocks = max(1, sustained_ms // 30)
  1003. endpoint_blocks = max(1, endpoint_silence_ms // 30)
  1004. max_blocks = max(1, max_utterance_ms // 30)
  1005. # Rolling floor window: continuously tracks TTS speaker-bleed volume
  1006. # throughout playback, not just the first calibration_ms. This is the
  1007. # key fix for false barge-in — a one-shot calibration freezes a floor
  1008. # from the opening TTS passage, but later louder passages exceed the
  1009. # stale floor and false-trigger. The rolling window keeps the floor
  1010. # current so only genuinely louder-than-playback speech trips the VAD.
  1011. floor_window: deque[float] = deque(maxlen=max(calib_blocks, 100)) # ~3s rolling
  1012. pre_roll: deque = deque(maxlen=max(1, pre_roll_ms // 30))
  1013. consecutive = 0
  1014. min_floor = 0.0 # baseline from initial calibration; floor never drops below this
  1015. block_idx = 0 # block counter for diagnostic logging
  1016. try:
  1017. with sd.InputStream(samplerate=SAMPLE_RATE, channels=1, dtype="int16", blocksize=block) as stream:
  1018. while not should_stop():
  1019. data, _ = stream.read(block)
  1020. rms = float(np.sqrt(np.mean(data.astype(np.float64) ** 2)))
  1021. if capture:
  1022. pre_roll.append(data.copy())
  1023. block_idx += 1
  1024. # Wait for at least calib_blocks before evaluating. During
  1025. # the initial warmup we always feed the window so calibration
  1026. # has data to work with.
  1027. if len(floor_window) < calib_blocks:
  1028. floor_window.append(rms)
  1029. continue
  1030. # Lock a minimum floor from the initial calibration samples.
  1031. # During inter-sentence pauses the rolling window can flush
  1032. # with near-silence, collapsing the 90th-percentile floor
  1033. # toward zero and false-triggering on the next rising
  1034. # sentence. min_floor keeps the trigger from ever dropping
  1035. # below the baseline TTS playback level established during
  1036. # the initial calibration_ms window.
  1037. #
  1038. # If the grace period ended during an inter-sentence gap the
  1039. # calibration samples near-silence. Locking a near-zero
  1040. # floor sets the trigger so low that TTS blocks exceed it,
  1041. # are excluded from the rolling window (rms >= trigger), and
  1042. # the floor freezes — guaranteeing a false trigger the moment
  1043. # TTS resumes. Clamp min_floor to SILENCE_RMS_THRESHOLD * 2
  1044. # (400 RMS) so the 8x multiplier yields a trigger of at least
  1045. # (500-2000 RMS) stays below it and feeds the rolling window,
  1046. # while genuine speech (3000-8000 RMS) can still trip it.
  1047. if min_floor == 0.0 and len(floor_window) >= calib_blocks:
  1048. _pct90 = float(np.percentile(list(floor_window), 90))
  1049. min_floor = max(_pct90, SILENCE_RMS_THRESHOLD * 2)
  1050. else:
  1051. _pct90 = float(np.percentile(list(floor_window), 90))
  1052. # Use the 90th percentile of the ROLLING window for the
  1053. # noise floor so the trigger reflects the loudest parts of
  1054. # recent playback — not a frozen snapshot from TTS onset.
  1055. _floor = max(_pct90, min_floor)
  1056. # 8.0x multiplier: TTS speaker bleed has wide
  1057. # volume variation between sentences and within sentences.
  1058. # At 5x, louder TTS passages exceed the trigger, get
  1059. # excluded from the floor window, and create a low-stale
  1060. # floor that false-triggers on the next loud passage.
  1061. # 8x gives enough headroom for TTS dynamics to stay below
  1062. # the trigger and get absorbed into the rolling floor.
  1063. trigger = max(float(threshold or SILENCE_RMS_THRESHOLD * 2), _floor * 8.0)
  1064. # Ceiling: never let the trigger exceed 4000 RMS, otherwise
  1065. # a very loud TTS passage would push the trigger so high
  1066. # that genuine speech (which is typically 3000–8000 RMS)
  1067. # couldn't trip it.
  1068. trigger = min(trigger, 4000.0)
  1069. # Only feed the floor window with blocks that are NOT above
  1070. # the current trigger — speech blocks would inflate the floor
  1071. # and make the trigger unreachable.
  1072. if rms < trigger:
  1073. floor_window.append(rms)
  1074. consecutive = consecutive + 1 if rms >= trigger else 0
  1075. if consecutive > 0:
  1076. logger.debug(
  1077. "VAD above-trigger: block=%d rms=%.0f floor=%.0f trigger=%.0f "
  1078. "consec=%d/%d min_floor=%.0f window_len=%d",
  1079. block_idx, rms, _floor, trigger, consecutive,
  1080. trip_blocks, min_floor, len(floor_window),
  1081. )
  1082. if consecutive < trip_blocks:
  1083. continue
  1084. # Tripped — the user is talking over playback.
  1085. logger.info(
  1086. "VAD TRIPPED: block=%d rms=%.0f floor=%.0f trigger=%.0f "
  1087. "consec=%d min_floor=%.0f — cutting TTS playback",
  1088. block_idx, rms, _floor, trigger, consecutive, min_floor,
  1089. )
  1090. if on_trigger:
  1091. try:
  1092. on_trigger()
  1093. except Exception as e:
  1094. logger.debug("Barge-in trigger callback failed: %s", e)
  1095. if not capture:
  1096. return True
  1097. # Keep rolling until the user goes quiet. Playback is stopped
  1098. # now, so plain silence endpointing (recorder threshold) works.
  1099. frames: list[Any] = list(pre_roll)
  1100. quiet = 0
  1101. for _ in range(max_blocks):
  1102. data, _ = stream.read(block)
  1103. frames.append(data.copy())
  1104. rms = float(np.sqrt(np.mean(data.astype(np.float64) ** 2)))
  1105. quiet = quiet + 1 if rms < SILENCE_RMS_THRESHOLD else 0
  1106. if quiet >= endpoint_blocks:
  1107. break
  1108. return AudioRecorder._write_wav(np.concatenate(frames, axis=0))
  1109. except Exception as e:
  1110. logger.debug("Barge-in listener failed: %s", e)
  1111. return None if capture else False
  1112. # ── Full-duplex agent-turn listener ──
  1113. # One listener for the WHOLE agent turn (armed at utterance submit, disarmed when
  1114. # response + TTS are done): calibrates against the QUIET room at turn start,
  1115. # holds that baseline through playback (never speaker bleed), trips on a windowed
  1116. # majority of blocks — so the user can interject during generation, not just TTS.
  1117. # Speaker bleed at the mic is a few hundred RMS (~1000-1400 loud/close); direct
  1118. # speech is 3000-8000; a quiet-room floor is typically 50-300.
  1119. PLAYBACK_MIN_TRIGGER = 1500.0 # min trigger while TTS flows: bleed alone never trips
  1120. TRIGGER_CEILING = 4000.0 # a noisy room must never push the trigger past normal speech
  1121. DEFAULT_BARGE_MULTIPLIER = 3.0 # over the quiet floor: 300 * 3 = 900 vs 3000+ speech
  1122. def _vad_log(msg: str) -> None:
  1123. """VAD diagnostic: logger.debug, plus stderr when HERMES_VOICE_DEBUG=1 (live tuning)."""
  1124. logger.debug(msg)
  1125. if os.environ.get("HERMES_VOICE_DEBUG", "").strip() == "1":
  1126. with suppress(Exception):
  1127. print(f"[voice-vad] {msg}", file=sys.stderr, flush=True)
  1128. def _capture_until_quiet(stream, np, block: int, pre_roll, *, endpoint_blocks: int, max_blocks: int) -> str:
  1129. """After a trip, read until *endpoint_blocks* of quiet (or *max_blocks*) and write
  1130. pre-roll + capture to a WAV. Playback was cut by the trigger, so silence endpointing works."""
  1131. frames: list[Any] = list(pre_roll)
  1132. quiet = 0
  1133. for _ in range(max_blocks):
  1134. data, _ = stream.read(block)
  1135. frames.append(data.copy())
  1136. quiet = quiet + 1 if _rms(np, data) < SILENCE_RMS_THRESHOLD else 0
  1137. if quiet >= endpoint_blocks:
  1138. break
  1139. return AudioRecorder._write_wav(np.concatenate(frames, axis=0))
  1140. class _BargeDetector:
  1141. """Per-block barge-in state machine behind ``full_duplex_listen``."""
  1142. def __init__(self, np, *, mult: float, calib_blocks: int, trip_blocks: int, grace_blocks: int) -> None:
  1143. self._np, self.mult, self.calib_blocks, self.grace_blocks = np, mult, calib_blocks, grace_blocks
  1144. self.trip_needed = max(1, round(trip_blocks * 0.8))
  1145. self.ambient: deque = deque(maxlen=100) # ~3s of quiet-phase RMS
  1146. self.recent_above: deque = deque(maxlen=trip_blocks)
  1147. self.quiet_floor = float(SILENCE_RMS_THRESHOLD)
  1148. self.floor_locked = self.playing_prev = self.playback_seen = False
  1149. self.grace_remaining = self.block_idx = 0
  1150. self.blocks_since_playback = 10_000
  1151. def _floor(self) -> tuple:
  1152. """(pct90, floor): 90th percentile of the quiet window; floor never below the silence threshold."""
  1153. pct90 = float(self._np.percentile(list(self.ambient), 90)) if self.ambient else float(SILENCE_RMS_THRESHOLD)
  1154. return pct90, max(pct90, float(SILENCE_RMS_THRESHOLD))
  1155. def _calibrate(self, rms: float, playing: bool) -> bool:
  1156. """Lock the floor from the first calib_blocks: the listener arms at utterance
  1157. submit, before any TTS exists, so they sample the quiet room — NOT bleed."""
  1158. if not playing:
  1159. self.ambient.append(rms)
  1160. if len(self.ambient) >= self.calib_blocks or playing:
  1161. pct90, self.quiet_floor = self._floor()
  1162. self.floor_locked = True
  1163. _vad_log(f"calibrated quiet floor={self.quiet_floor:.0f} "
  1164. f"(pct90={pct90:.0f}, {len(self.ambient)} blocks, mult={self.mult:g})")
  1165. return self.floor_locked
  1166. def _track_playback(self, playing: bool) -> None:
  1167. """Grace only when playback starts after a real gap (>=1s), so inter-sentence
  1168. flapping of the audio-active flag can't chain windows and swallow an interjection."""
  1169. if playing and not self.playing_prev:
  1170. if not self.playback_seen or self.blocks_since_playback > 33:
  1171. self.grace_remaining = self.grace_blocks
  1172. _vad_log(f"playback started (block={self.block_idx}) — grace {self.grace_blocks * 30}ms")
  1173. self.playback_seen = True
  1174. self.playing_prev = playing
  1175. self.blocks_since_playback = 0 if playing else self.blocks_since_playback + 1
  1176. def feed(self, rms: float, playing: bool) -> Optional[str]:
  1177. """Consume one 30ms block; return the phase name when speech trips, else None."""
  1178. self.block_idx += 1
  1179. if not self.floor_locked and not self._calibrate(rms, playing):
  1180. return None
  1181. self._track_playback(playing)
  1182. # Trigger: quiet baseline x multiplier, phase-clamped.
  1183. trigger = max(self.quiet_floor * self.mult,
  1184. PLAYBACK_MIN_TRIGGER if playing else float(SILENCE_RMS_THRESHOLD) * 2)
  1185. trigger = min(trigger, TRIGGER_CEILING)
  1186. # Track ambient drift ONLY while nothing plays (never absorb bleed) and the block isn't speech.
  1187. if not playing and rms < trigger:
  1188. self.ambient.append(rms)
  1189. _, self.quiet_floor = self._floor()
  1190. above = rms >= trigger
  1191. if above and self.grace_remaining > 0:
  1192. _vad_log(f"grace suppression: block={self.block_idx} rms={rms:.0f} "
  1193. f"trigger={trigger:.0f} ({self.grace_remaining} blocks left)")
  1194. above = False
  1195. if self.grace_remaining > 0:
  1196. self.grace_remaining -= 1
  1197. self.recent_above.append(above)
  1198. phase = "playback" if playing else "generation"
  1199. if rms >= trigger * 0.5:
  1200. _vad_log(f"block={self.block_idx} rms={rms:.0f} floor={self.quiet_floor:.0f} "
  1201. f"trigger={trigger:.0f} above={above} "
  1202. f"window={sum(self.recent_above)}/{self.trip_needed} phase={phase}")
  1203. if not (above and sum(self.recent_above) >= self.trip_needed):
  1204. return None
  1205. _vad_log(f"TRIPPED ({phase}): block={self.block_idx} rms={rms:.0f} floor={self.quiet_floor:.0f} "
  1206. f"trigger={trigger:.0f} window={sum(self.recent_above)}/{len(self.recent_above)}")
  1207. return phase
  1208. def full_duplex_listen(
  1209. should_stop: Callable[[], bool], is_playing: Optional[Callable[[], bool]] = None,
  1210. on_trigger: Optional[Callable[[str], None]] = None, multiplier: Optional[float] = None,
  1211. sustained_ms: int = 300, calibration_ms: int = 450, grace_ms: int = 500, pre_roll_ms: int = 1200,
  1212. endpoint_silence_ms: int = 1250, max_utterance_ms: int = 30_000,
  1213. ) -> Optional[str]:
  1214. """Listen across an ENTIRE agent turn; return the captured interruption WAV path.
  1215. Phase per 30ms block from *is_playing* (usually ``is_audio_output_active``):
  1216. ``generation`` — first *calibration_ms* of quiet room set the floor, trigger =
  1217. floor x *multiplier*; ``playback`` — floor HELD, trigger clamped up to
  1218. ``PLAYBACK_MIN_TRIGGER``, *grace_ms* after playback start suppresses onset
  1219. transients. Trip = >=80% of the last *sustained_ms* blocks above trigger (so
  1220. intra-word dips don't reset). Then ``on_trigger(phase)`` fires and capture runs
  1221. from the *pre_roll_ms* buffer until *endpoint_silence_ms* of quiet. ``None``
  1222. when *should_stop* ends the turn without speech.
  1223. """
  1224. try:
  1225. sd, np = _import_audio()
  1226. except (ImportError, OSError):
  1227. return None
  1228. block = int(SAMPLE_RATE * 0.03) # 30ms blocks
  1229. endpoint_blocks = max(1, endpoint_silence_ms // 30)
  1230. detector = _BargeDetector(
  1231. np, mult=float(multiplier) if multiplier else DEFAULT_BARGE_MULTIPLIER,
  1232. calib_blocks=max(1, calibration_ms // 30), trip_blocks=max(1, sustained_ms // 30),
  1233. grace_blocks=max(0, grace_ms // 30))
  1234. pre_roll: deque = deque(maxlen=max(1, pre_roll_ms // 30))
  1235. try:
  1236. with sd.InputStream(samplerate=SAMPLE_RATE, channels=1, dtype="int16", blocksize=block) as stream:
  1237. while not should_stop():
  1238. data, _ = stream.read(block)
  1239. pre_roll.append(data.copy())
  1240. playing = bool(is_playing()) if is_playing is not None else False
  1241. phase = detector.feed(_rms(np, data), playing)
  1242. if phase is None:
  1243. continue
  1244. if on_trigger:
  1245. try:
  1246. on_trigger(phase)
  1247. except Exception as e:
  1248. logger.debug("full-duplex trigger callback failed: %s", e)
  1249. return _capture_until_quiet(stream, np, block, pre_roll, endpoint_blocks=endpoint_blocks,
  1250. max_blocks=max(1, max_utterance_ms // 30))
  1251. except Exception as e:
  1252. logger.debug("Full-duplex listener failed: %s", e)
  1253. return None
  1254. # ── Requirements check ──
  1255. def _check_plugin_stt_provider(provider: str) -> bool:
  1256. """True when *provider* resolves to an available STT plugin."""
  1257. key = (provider or "").lower().strip()
  1258. if not key or key == "none":
  1259. return False
  1260. try:
  1261. from agent.transcription_registry import get_provider
  1262. from hermes_cli.plugins import _ensure_plugins_discovered
  1263. _ensure_plugins_discovered()
  1264. plugin_provider = get_provider(key)
  1265. if plugin_provider is None:
  1266. # Match the transcription dispatcher: long-lived processes may need
  1267. # one refresh after plugins or configuration change.
  1268. _ensure_plugins_discovered(force=True)
  1269. plugin_provider = get_provider(key)
  1270. except Exception as exc:
  1271. logger.debug("STT plugin requirements check skipped for '%s': %s", key, exc)
  1272. return False
  1273. try:
  1274. return plugin_provider is not None and bool(plugin_provider.is_available())
  1275. except Exception as exc:
  1276. logger.warning(
  1277. "STT plugin provider '%s' is_available() raised during requirements "
  1278. "check: %s - treating as unavailable", key, exc, exc_info=True)
  1279. return False
  1280. # STT providers handled natively by tools.transcription_tools -> status label.
  1281. _NATIVE_STT_LABELS = {
  1282. "local": "local faster-whisper",
  1283. "local_command": "local command",
  1284. "groq": "Groq",
  1285. "openai": "OpenAI",
  1286. "mistral": "Mistral Voxtral",
  1287. "xai": "xAI Grok STT",
  1288. "elevenlabs": "ElevenLabs Scribe",
  1289. "deepinfra": "DeepInfra",
  1290. }
  1291. def check_voice_requirements() -> dict[str, Any]:
  1292. """Check voice mode requirements: ``{available, audio_available, stt_available,
  1293. missing_packages, details, environment}``."""
  1294. from tools.transcription_tools import (
  1295. _get_provider, _load_stt_config, _resolve_command_stt_provider_config, is_stt_enabled)
  1296. stt_config = _load_stt_config()
  1297. stt_enabled = is_stt_enabled(stt_config)
  1298. stt_provider = _get_provider(stt_config)
  1299. stt_label = None # "OK (...)" once a native / command / plugin provider resolves
  1300. if stt_provider in _NATIVE_STT_LABELS:
  1301. stt_label = f"OK ({_NATIVE_STT_LABELS[stt_provider]})"
  1302. elif stt_enabled and _resolve_command_stt_provider_config(stt_provider, stt_config) is not None:
  1303. stt_label = f"OK (command: {stt_provider})"
  1304. elif stt_enabled and _check_plugin_stt_provider(stt_provider):
  1305. stt_label = f"OK (plugin: {stt_provider})"
  1306. stt_available = stt_enabled and stt_label is not None
  1307. termux_capture = _termux_voice_capture_available()
  1308. has_audio = _audio_available() or termux_capture
  1309. env_check = detect_audio_environment()
  1310. details = [
  1311. "Audio capture: OK (Termux:API microphone)" if termux_capture
  1312. else "Audio capture: OK" if has_audio
  1313. else f"Audio capture: MISSING ({_audio_unavailable_reason()})",
  1314. "STT provider: DISABLED in config (stt.enabled: false)" if not stt_enabled
  1315. else f"STT provider: {stt_label}" if stt_label
  1316. else ("STT provider: MISSING (run `hermes tools` and configure "
  1317. "Speech-to-Text: Local Whisper or a cloud provider)"),
  1318. ]
  1319. details += [f"Environment: {w}" for w in env_check["warnings"]]
  1320. details += [f"Environment: {n}" for n in env_check.get("notices", [])]
  1321. return {
  1322. "available": has_audio and stt_available and env_check["available"],
  1323. "audio_available": has_audio,
  1324. "stt_available": stt_available,
  1325. "missing_packages": [] if has_audio else ["sounddevice", "numpy"],
  1326. "details": "\n".join(details),
  1327. "environment": env_check,
  1328. }
  1329. # ── Temp file cleanup ──
  1330. def cleanup_temp_recordings(max_age_seconds: int = 3600) -> int:
  1331. """Remove ``recording_*.wav`` temp files older than *max_age_seconds*; returns the count."""
  1332. from utils import unlink_files_older_than
  1333. deleted = unlink_files_older_than(_TEMP_DIR, "recording_*.wav", max_age_seconds)
  1334. if deleted:
  1335. logger.debug("Cleaned up %d old voice recordings", deleted)
  1336. return deleted