wake_word.py 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938
  1. """Wake-word ("Hey Hermes") detection — hands-free session trigger.
  2. One always-on hotword listener shared by CLI, TUI and desktop GUI (a single owner,
  3. gated by ``wake_surface_enabled``). Engines live in :mod:`tools.wake_word_engines`;
  4. this module owns config, the capture loop and the process-wide listener singleton.
  5. Capture reuses voice mode's 16 kHz mono int16 ``sounddevice`` path on a daemon
  6. thread; callers ``pause()`` while a voice turn holds the mic and ``resume()`` once
  7. idle (two input streams on one device is unreliable cross-platform).
  8. """
  9. from __future__ import annotations
  10. from pm import install_hint
  11. import logging
  12. import os
  13. import queue
  14. import sys
  15. import threading
  16. import time
  17. from contextlib import suppress
  18. from dataclasses import dataclass
  19. from pathlib import Path
  20. from typing import Any, Callable, Dict, Optional
  21. # The engine classes are re-exported on purpose: _build_engine resolves the
  22. # _PROVIDERS names on THIS module so a test (or plugin) can swap one engine.
  23. from tools.wake_word_engines import (
  24. _Engine,
  25. _OpenWakeWordEngine,
  26. _PorcupineEngine,
  27. _SherpaKwsEngine,
  28. )
  29. logger = logging.getLogger(__name__)
  30. SAMPLE_RATE = 16000 # 16 kHz mono int16 — Whisper-native and what every engine expects.
  31. # Minimum gap between two wake fires, so one "hey hermes" can't retrigger across
  32. # several frames while the caller is still reacting.
  33. _FIRE_COOLDOWN_SECONDS = 2.0
  34. _START_TIMEOUT_SECONDS = 5.0
  35. _READ_POLL_SECONDS = 0.05 # slice between read_available polls; bounds halt latency
  36. # Some hostapis (Windows DirectSound) never advance ``read_available`` even while
  37. # audio flows; after this much starvation the poll guard is abandoned for the
  38. # blocking read (see ``_Capture.read``).
  39. _READ_STALL_FALLBACK_SECONDS = 2.0
  40. _HALT_JOIN_SECONDS = 2.0
  41. # Ambient-speech rejection: N consecutive over-threshold frames before firing
  42. # (a stray phoneme spikes one frame; a real phrase holds several).
  43. _DEFAULT_CONFIRMATION_FRAMES = 3
  44. # Dead-mic detection: an int16 stream whose peak stays at/below _SILENCE_PEAK for
  45. # this many consecutive seconds is flagged silent (desktop push-to-talk and the
  46. # backend listener capture differently, so one can work while the other is all zeros).
  47. _SILENCE_PEAK = 10
  48. _SILENCE_ALERT_SECONDS = 10
  49. # provider alias -> (engine class name on this module, pm extra).
  50. # Unknown providers probe as openwakeword but fail to build.
  51. _PROVIDERS: dict[str, tuple[str, str]] = {
  52. "porcupine": ("_PorcupineEngine", "wake-porcupine"),
  53. **{k: ("_SherpaKwsEngine", "wake-sherpa") for k in ("sherpa", "sherpa-onnx", "kws", "open")},
  54. **{k: ("_OpenWakeWordEngine", "wake-openwakeword") for k in ("openwakeword", "oww", "local")},
  55. }
  56. _PROVIDER_PREFERENCE = ("openwakeword", "sherpa", "porcupine")
  57. class WakeWordInUse(RuntimeError):
  58. """Raised when another surface or process owns the wake-word listener."""
  59. # ── Config ──
  60. # capture: "local" (PortAudio on the backend host), "client" (desktop/TUI streams int16
  61. # frames via wake.feed), or "auto" (local when a device exists, else client).
  62. _DEFAULTS: dict[str, Any] = {
  63. "enabled": False, "surface": "auto", "input_device": None, "capture": "auto",
  64. "provider": "auto", "phrase": "hey hermes", "sensitivity": 0.6,
  65. # detached-spawn: ok — not a spawn: the wake-word config key (fresh chat session on wake)
  66. "confirmation_frames": _DEFAULT_CONFIRMATION_FRAMES, "start_new_session": True,
  67. }
  68. # Bundled "hey hermes" model (tools/wakewords/) — the default; alias names resolve
  69. # to it, not to an openWakeWord built-in.
  70. _BUNDLED_MODEL_NAME = "hey_hermes"
  71. _BUNDLED_MODEL_ALIASES = frozenset({"", "hey_hermes", "hey hermes", "hermes"})
  72. def _bundled_wakeword_path() -> str:
  73. """Path to the shipped hey_hermes.tflite — pyopen-wakeword runs TFLite only."""
  74. return os.path.join(os.path.dirname(__file__), "wakewords", f"{_BUNDLED_MODEL_NAME}.tflite")
  75. def load_wake_word_config() -> dict[str, Any]:
  76. """Return the ``wake_word`` config section, shape-guarded to a dict."""
  77. cfg = None
  78. with suppress(Exception):
  79. from hermes_cli.config import load_config
  80. cfg = load_config().get("wake_word")
  81. return cfg if isinstance(cfg, dict) else {}
  82. def _get(cfg: dict[str, Any], key: str) -> Any:
  83. val = cfg.get(key)
  84. return _DEFAULTS.get(key) if val is None else val
  85. def _clamped(cfg: dict[str, Any], key: str, cast, lo, hi):
  86. """Numeric config value via ``cast``, defaulting on junk, clamped to lo..hi."""
  87. try:
  88. n = cast(_get(cfg, key))
  89. except (TypeError, ValueError):
  90. n = cast(_DEFAULTS[key])
  91. return min(max(n, lo), hi)
  92. def _provider(cfg: dict[str, Any], *, supported: Callable[[str], bool] | None = None) -> str:
  93. provider = str(_get(cfg, "provider")).strip().lower() or "auto"
  94. if provider != "auto":
  95. return provider
  96. if supported is None:
  97. from pm.extras import extra_supported
  98. supported = extra_supported
  99. return next((name for name in _PROVIDER_PREFERENCE if supported(_PROVIDERS[name][1])), "porcupine")
  100. def _input_device(cfg: dict[str, Any]) -> int | str | None:
  101. """Configured PortAudio input selector, preserving indices and names."""
  102. raw = _get(cfg, "input_device")
  103. return None if isinstance(raw, bool) else raw if raw is None or isinstance(raw, int) else (str(raw).strip() or None)
  104. def _sensitivity(cfg: dict[str, Any]) -> float:
  105. return _clamped(cfg, "sensitivity", float, 0.0, 1.0)
  106. def _confirmation_frames(cfg: dict[str, Any]) -> int:
  107. """Consecutive over-threshold frames required to fire, clamped 1..10 (1 = single-frame)."""
  108. return _clamped(cfg, "confirmation_frames", int, 1, 10)
  109. def wake_phrase(cfg: Optional[dict[str, Any]] = None) -> str:
  110. """Human-facing wake phrase label (purely cosmetic; engine keys detection)."""
  111. cfg = cfg if cfg is not None else load_wake_word_config()
  112. return str(_get(cfg, "phrase")) or "hey hermes"
  113. def resolve_capture_mode(cfg: Optional[dict[str, Any]] = None, *, prefer_client: bool = False,
  114. force_local: bool = False) -> str:
  115. """Return ``local`` or ``client`` capture mode for this arm. ``prefer_client`` is set by remote
  116. desktop; ``force_local`` keeps CLI/TUI on the process mic. Under ``auto`` a working backend input
  117. always wins; client is the fallback only for a preferring surface with no usable backend mic —
  118. CLI/TUI stay local so status reports the real requirement rather than a path nothing will feed."""
  119. cfg = cfg if cfg is not None else load_wake_word_config()
  120. if force_local:
  121. return "local"
  122. raw = str(_get(cfg, "capture") or "auto").strip().lower()
  123. if raw in ("client", "remote", "external"):
  124. return "client"
  125. return "client" if raw != "local" and prefer_client and not _local_input_device_ready() else "local"
  126. def _input_channels(info: Any) -> int:
  127. ch = info.get("max_input_channels") if isinstance(info, dict) else getattr(info, "max_input_channels", 0)
  128. return int(ch or 0)
  129. def _local_input_device_ready() -> bool:
  130. """True when PortAudio is importable and at least one input device exists."""
  131. try:
  132. sd, _ = _import_audio()
  133. devices = sd.query_devices()
  134. if isinstance(devices, dict):
  135. return _input_channels(devices) > 0
  136. # Also accept a resolvable default input (some hosts list devices oddly).
  137. return (any(_input_channels(d) > 0 for d in devices)
  138. or _input_channels(sd.query_devices(None, "input")) > 0)
  139. except Exception:
  140. return False
  141. def wake_surface_enabled(surface: str, cfg: Optional[dict[str, Any]] = None) -> bool:
  142. """Should ``surface`` (cli/tui/gui) host the listener? True when enabled and the configured
  143. surface is ``auto`` or this one; ``auto`` only makes it eligible — the lock admits one claimant."""
  144. cfg = cfg if cfg is not None else load_wake_word_config()
  145. want = str(_get(cfg, "surface")).strip().lower() or "auto"
  146. return bool(cfg.get("enabled")) and want in ("auto", surface.strip().lower())
  147. # ── Multi-profile phrase enrollment (open-vocabulary routing) ──
  148. def _active_profile_name() -> str:
  149. with suppress(Exception):
  150. from hermes_cli.profiles import get_active_profile_name
  151. return get_active_profile_name() or "default"
  152. return "default"
  153. def enrolled_profile_phrases() -> dict[str, str]:
  154. """Map ``profile name -> wake phrase`` for every wake-enabled profile, reading each profile's own
  155. ``config.yaml`` raw (``load_config()`` targets only the ACTIVE profile). Phrase defaults to
  156. ``"hey <profile>"``; the sherpa engine listens for all and routes to the match. Unreadable → skipped."""
  157. phrases: dict[str, str] = {}
  158. with suppress(Exception):
  159. from hermes_cli.config import read_user_config_raw
  160. from hermes_cli.profiles import get_profile_dir, list_profiles
  161. for info in list_profiles():
  162. name = getattr(info, "name", None) or str(info)
  163. with suppress(Exception):
  164. wc = read_user_config_raw(Path(get_profile_dir(name)) / "config.yaml").get("wake_word") or {}
  165. if isinstance(wc, dict) and wc.get("enabled"):
  166. phrase = str(wc.get("phrase") or f"hey {name}").strip()
  167. if phrase:
  168. phrases[name] = phrase
  169. return phrases
  170. # ── Audio capture (lazy — never import sounddevice at module load) ──
  171. def _import_audio():
  172. import numpy as np
  173. import sounddevice as sd
  174. return sd, np
  175. def _audio_available() -> bool:
  176. with suppress(ImportError, OSError):
  177. return bool(_import_audio())
  178. return False
  179. def _describe_input_device(selector: int | str | None, sd=None) -> dict[str, Any]:
  180. """Resolve a PortAudio selector into JSON-safe diagnostics (``InputStream`` stays the
  181. authority on whether the device actually opens). Imports sounddevice unless ``sd`` is given."""
  182. details: dict[str, Any] = {"selector": selector}
  183. try:
  184. sd = sd or _import_audio()[0]
  185. info = sd.query_devices(selector, "input")
  186. except Exception as e:
  187. details["error"] = str(e)
  188. return details
  189. if not isinstance(info, dict):
  190. return details
  191. if info.get("name"):
  192. details["name"] = str(info["name"])
  193. for key, out_key, cast in (("max_input_channels", "max_input_channels", int),
  194. ("default_samplerate", "default_samplerate", float), ("hostapi", "hostapi_index", int)):
  195. if isinstance(info.get(key), (int, float)):
  196. details[out_key] = cast(info[key])
  197. if "hostapi_index" in details:
  198. with suppress(Exception):
  199. hostapi = sd.query_hostapis(details["hostapi_index"])
  200. if isinstance(hostapi, dict) and hostapi.get("name"):
  201. details["hostapi"] = str(hostapi["name"])
  202. return details
  203. def _resample_audio_frame(np, frame, output_length: int):
  204. """Convert one native-rate capture block to an exact engine frame."""
  205. source = np.asarray(frame, dtype=np.float64).reshape(-1)
  206. if source.size == output_length:
  207. return np.asarray(frame, dtype=np.int16).reshape(-1)
  208. if source.size == 0:
  209. return np.zeros(output_length, dtype=np.int16)
  210. if source.size > output_length:
  211. # Average each source window when reducing (matches the desktop wake capture
  212. # path) so speech energy is retained instead of decimated.
  213. edges = np.linspace(0, source.size, output_length + 1, dtype=np.int64)
  214. values = np.add.reduceat(source, edges[:-1]) / np.diff(edges)
  215. else:
  216. # Unusual low-rate devices: interpolate up to the 16 kHz frame size.
  217. source_positions = np.arange(source.size, dtype=np.float64)
  218. values = np.interp(np.linspace(0, source.size - 1, output_length), source_positions, source)
  219. return np.rint(values).clip(-32768, 32767).astype(np.int16)
  220. def silent_audio_hint(details: dict[str, Any]) -> str:
  221. """Platform-specific remediation for an armed stream delivering silence."""
  222. if sys.platform == "darwin":
  223. return ("Microphone delivers only silence. Grant the Hermes backend "
  224. "microphone access in System Settings > Privacy & Security > "
  225. "Microphone, then toggle the wake word.")
  226. fix = ("Set wake_word.input_device to a different PortAudio input device"
  227. if sys.platform == "win32" else "Check the selected input device")
  228. selector = details.get("selector")
  229. label = str(details.get("name") or "").strip() or ("system default" if selector is None else str(selector))
  230. hostapi = str(details.get("hostapi") or "").strip()
  231. label = f"{label} ({hostapi})" if hostapi else label
  232. return f"Microphone delivers only silence from {label}. {fix}, then toggle the wake word."
  233. def _build_engine(cfg: dict[str, Any]) -> _Engine:
  234. provider = _provider(cfg)
  235. if provider not in _PROVIDERS:
  236. raise ValueError(f"Unknown wake_word provider: {provider!r}")
  237. return globals()[_PROVIDERS[provider][0]](cfg)
  238. # ── Requirements probe (for /wake status + enable path) ──
  239. def _stt_ready() -> bool:
  240. """Is a speech-to-text provider configured and enabled? (A wake without STT arms the
  241. mic but every utterance dies at transcription — same bar as ``check_voice_requirements``.)"""
  242. with suppress(Exception):
  243. from tools.transcription_tools import _get_provider, _load_stt_config, is_stt_enabled
  244. stt_config = _load_stt_config()
  245. return is_stt_enabled(stt_config) and _get_provider(stt_config) != "none"
  246. return False
  247. def _tts_ready() -> bool:
  248. """Can the configured text-to-speech provider run (or install at first use)?
  249. The wake flow is fully hands-free (wake → speak → hear the reply); without
  250. TTS the reply is silent and the loop is pointless.
  251. PROBE, not an installer: ``check_tts_requirements`` lazily pip-installs the
  252. provider SDK via ``_import_*`` → ``pm.ensure_import`` — running that inside
  253. a status poll froze wake.status for the length of a pip install (and a
  254. failed install marked the wake word unavailable, unmounting the desktop
  255. ear). When the provider's deps aren't installed yet, "installable at first
  256. use" counts as ready and we never touch pip from here.
  257. """
  258. try:
  259. from tools.tts_tool import _get_provider, _load_tts_config, check_tts_requirements
  260. provider = _get_provider(_load_tts_config())
  261. except Exception:
  262. return False
  263. _LAZY_TTS_FEATURES = {
  264. "edge": "edge-tts",
  265. "elevenlabs": "tts-premium",
  266. "mistral": "mistral",
  267. }
  268. feature = _LAZY_TTS_FEATURES.get(provider)
  269. if feature is not None:
  270. try:
  271. import pm
  272. from pm.install import lazy_installs_allowed
  273. if not pm.available(feature):
  274. # Not installed: ready iff it can install at first speak.
  275. return lazy_installs_allowed()
  276. except Exception:
  277. return False
  278. try:
  279. from tools.tts_tool import check_tts_requirements
  280. return bool(check_tts_requirements())
  281. except Exception:
  282. return False
  283. def check_wake_word_requirements(cfg: Optional[dict[str, Any]] = None, *,
  284. supported: Callable[[str], bool] | None = None) -> dict[str, Any]:
  285. """Report whether wake-word detection can run, with a remediation hint."""
  286. cfg = cfg if cfg is not None else load_wake_word_config()
  287. import pm
  288. from pm.install import lazy_installs_allowed
  289. from pm.extras import extra_supported
  290. supported = supported or extra_supported
  291. provider = _provider(cfg, supported=supported)
  292. if provider == "porcupine":
  293. feature = "wake-porcupine"
  294. elif provider in ("sherpa", "sherpa-onnx", "kws", "open"):
  295. feature = "wake-sherpa"
  296. else:
  297. feature = "wake-openwakeword"
  298. deps_ok = pm.available(feature)
  299. platform_ok = deps_ok or supported(feature)
  300. lazy_ok = lazy_installs_allowed()
  301. # The audio probe imports sounddevice + numpy — two of the very packages
  302. # the lazy installer would fetch — so it can only be trusted once the
  303. # feature's deps are installed. On a fresh install (deps missing, lazy
  304. # installs allowed) we defer the mic check: the engine constructors call
  305. # ``pm.ensure_import()`` and the stream-open surfaces any real audio
  306. # problem. Gating ``available`` on the probe here made the lazy-install
  307. # path unreachable (the probe always failed before ensure() could run).
  308. audio_ok = _audio_available() if deps_ok else False
  309. key_ok = True
  310. # The full wake loop is wake → record → STT → agent → TTS. Arming without
  311. # either end configured gives a mic that hears you and then does nothing
  312. # the user can perceive — refuse with a pointer instead.
  313. stt_ok = _stt_ready()
  314. tts_ok = _tts_ready()
  315. hint = ""
  316. if not platform_ok:
  317. alternatives = [name for name in ("sherpa", "porcupine")
  318. if supported(_PROVIDERS[name][1])]
  319. hint = f"The {provider} wake engine is not supported on this platform."
  320. if alternatives:
  321. hint += f" Set wake_word.provider to {' or '.join(alternatives)}."
  322. elif provider == "porcupine" and not (os.getenv("PORCUPINE_ACCESS_KEY") or "").strip():
  323. key_ok = False
  324. hint = "Set PORCUPINE_ACCESS_KEY (free key at https://console.picovoice.ai)."
  325. elif not deps_ok and not lazy_ok:
  326. hint = install_hint(feature)
  327. elif deps_ok and not audio_ok and resolve_capture_mode(cfg) == "local":
  328. hint = "Microphone capture needs sounddevice + numpy and a working audio device."
  329. elif not stt_ok or not tts_ok:
  330. missing = " and ".join(
  331. name for name, ok in (("speech-to-text", stt_ok), ("text-to-speech", tts_ok)) if not ok
  332. )
  333. hint = (f"Wake word needs {missing} configured — run `hermes tools` "
  334. f"(Voice section) or see the voice-mode docs.")
  335. capture_mode = resolve_capture_mode(cfg)
  336. # Client capture needs deps (engine) but not a server-side PortAudio device.
  337. if capture_mode == "client":
  338. mic_ok = deps_ok or lazy_ok
  339. else:
  340. mic_ok = (deps_ok and audio_ok) or (not deps_ok and lazy_ok)
  341. if deps_ok and not audio_ok and not hint:
  342. hint = ("No local microphone on this backend. Remote desktop can stream "
  343. "the client mic — set wake_word.capture: client or use a desktop "
  344. "build with client-capture wake support.")
  345. return {
  346. "available": platform_ok and key_ok and stt_ok and tts_ok and mic_ok, "provider": provider,
  347. "deps_available": deps_ok, "audio_available": audio_ok,
  348. "local_input_available": _local_input_device_ready() if deps_ok else False,
  349. "capture": capture_mode, "access_key_set": key_ok, "stt_available": stt_ok, "tts_available": tts_ok,
  350. "phrase": wake_phrase(cfg), "hint": hint,
  351. }
  352. # ── Detector ──
  353. @dataclass
  354. class _Capture:
  355. """One armed audio source: a PortAudio stream (local) or the feed queue (client)."""
  356. stream: Any = None # sounddevice.InputStream, None in client mode
  357. queue: Any = None # client-capture frame queue, None in local mode
  358. np: Any = None
  359. rate: int = SAMPLE_RATE
  360. frame_length: int = 1280 # samples per read at ``rate``
  361. # Set once ``read_available`` proves unreliable for this stream: skip the poll
  362. # guard and go straight to the blocking read.
  363. _skip_available_poll: bool = False
  364. def read(self, stop: Optional[threading.Event] = None):
  365. """One raw block; None when nothing arrived within ~250 ms (client) or ``stop`` was
  366. set while waiting (local). Stream errors propagate.
  367. A PortAudio ``read(n)`` blocks until ``n`` samples exist and, on a wedged ALSA/
  368. PipeWire device, never returns — so the halting thread's ``join`` timed out and
  369. ``close()`` raced the still-pending read. Poll ``read_available`` in short slices
  370. against ``stop`` and only call ``read`` once the block is guaranteed to be there.
  371. A device that never advances ``read_available`` (Windows DirectSound) would spin
  372. here forever, so after ``_READ_STALL_FALLBACK_SECONDS`` of starvation the guard is
  373. abandoned (with a warning) and the blocking read takes over; a truly wedged
  374. blocking read is released by ``_halt_thread`` aborting the stream.
  375. """
  376. if self.stream is not None:
  377. available = getattr(self.stream, "read_available", None)
  378. if stop is not None and available is not None and not self._skip_available_poll:
  379. stalled = 0.0
  380. while self.stream.read_available < self.frame_length:
  381. if stop.wait(_READ_POLL_SECONDS):
  382. return None
  383. stalled += _READ_POLL_SECONDS
  384. if stalled >= _READ_STALL_FALLBACK_SECONDS:
  385. logger.warning(
  386. "wake word: read_available stuck below frame length for "
  387. "%.1fs — falling back to blocking reads", stalled)
  388. self._skip_available_poll = True
  389. break
  390. return self.stream.read(self.frame_length)[0]
  391. with suppress(Exception):
  392. return self.queue.get(timeout=0.25)
  393. return None
  394. def close(self) -> None:
  395. """``abort()`` first: it discards pending buffers and unblocks any in-flight read,
  396. which ``stop()`` (drains, waits) cannot do on a dead device."""
  397. if self.stream is None:
  398. return
  399. abort = getattr(self.stream, "abort", None)
  400. with suppress(Exception):
  401. if abort is not None:
  402. abort()
  403. else:
  404. self.stream.stop()
  405. with suppress(Exception):
  406. self.stream.close()
  407. class WakeWordDetector:
  408. """Background hotword listener; fires ``on_wake()`` when the phrase is heard. The engine is built
  409. once and kept across pause/resume — only the stream + reader thread cycle, so mic toggles are cheap."""
  410. def __init__(self, engine: _Engine, on_wake: Callable[[], None], cooldown: float = _FIRE_COOLDOWN_SECONDS,
  411. on_failure: Optional[Callable[[WakeWordDetector], None]] = None,
  412. input_device: int | str | None = None, external_audio: bool = False):
  413. self.engine, self.on_wake, self.cooldown, self.on_failure = engine, on_wake, cooldown, on_failure
  414. self.input_device, self.external_audio = input_device, bool(external_audio)
  415. self.input_device_details: dict[str, Any] = (
  416. {"selector": "client", "name": "client capture", "hostapi": "remote"}
  417. if self.external_audio else {"selector": input_device})
  418. self._thread: Optional[threading.Thread] = None
  419. # Armed capture of the current reader thread; lets _halt_thread abort a
  420. # blocking read the poll guard cannot return from. Written by _run.
  421. self._cap: Optional[_Capture] = None
  422. self._stop, self._callback_inflight = threading.Event(), threading.Event()
  423. self._lock, self._last_fire = threading.Lock(), 0.0
  424. # Client-capture PCM queue (int16 mono frames). Local mode ignores this.
  425. self._audio_q: queue.Queue[Any] = queue.Queue(maxsize=64)
  426. # True when the stream is open but every frame is (near-)silence, so status
  427. # surfaces can tell "armed" from "deaf".
  428. self.audio_silent, self._silent_frames = False, 0
  429. @property
  430. def running(self) -> bool:
  431. t = self._thread
  432. return t is not None and t.is_alive()
  433. def feed(self, pcm_int16) -> None:
  434. """Enqueue one int16 mono frame (or raw bytes) for client capture. Short frames are
  435. zero-padded to ``engine.frame_length``, long ones split; on overflow the oldest is dropped."""
  436. if not self.external_audio:
  437. return
  438. try:
  439. import numpy as np
  440. except Exception:
  441. return
  442. if isinstance(pcm_int16, (bytes, bytearray, memoryview)):
  443. arr = np.frombuffer(pcm_int16, dtype=np.int16)
  444. else:
  445. arr = np.asarray(pcm_int16, dtype=np.int16).reshape(-1)
  446. fl = int(self.engine.frame_length)
  447. if fl <= 0:
  448. return
  449. for offset in range(0, int(arr.shape[0]), fl):
  450. chunk = arr[offset : offset + fl]
  451. if chunk.shape[0] < fl:
  452. chunk = np.pad(chunk, (0, fl - chunk.shape[0]))
  453. try:
  454. self._audio_q.put_nowait(chunk)
  455. except Exception:
  456. # Drop oldest on overflow so we stay real-time
  457. with suppress(Exception):
  458. self._audio_q.get_nowait()
  459. with suppress(Exception):
  460. self._audio_q.put_nowait(chunk)
  461. def start(self) -> None:
  462. """Open the mic (or client feeder) and begin listening. Idempotent."""
  463. with self._lock:
  464. if self.running:
  465. return
  466. self._stop.clear()
  467. ready = threading.Event()
  468. startup_errors: list[BaseException] = []
  469. self._thread = threading.Thread(target=self._run, args=(ready, startup_errors),
  470. daemon=True, name="wake-word")
  471. self._thread.start()
  472. if not ready.wait(_START_TIMEOUT_SECONDS):
  473. self._halt_thread()
  474. raise TimeoutError("Timed out while opening the wake-word microphone.")
  475. if startup_errors:
  476. self._halt_thread()
  477. raise RuntimeError("Failed to open the wake-word microphone.") from startup_errors[0]
  478. # pause/resume keep the engine; stop tears it down.
  479. def pause(self) -> None:
  480. self._halt_thread()
  481. def resume(self) -> None:
  482. self.start()
  483. def stop(self) -> None:
  484. self._halt_thread()
  485. self.engine.close()
  486. def _halt_thread(self) -> None:
  487. with self._lock:
  488. self._stop.set()
  489. t = self._thread
  490. # Join OUTSIDE the lock: a reader wedged in PortAudio would otherwise pin the lock
  491. # for the whole timeout and stall every start()/pause() caller behind it.
  492. if t is not None and t is not threading.current_thread():
  493. t.join(timeout=_HALT_JOIN_SECONDS)
  494. if t.is_alive():
  495. # The reader is wedged in a blocking read (read_available proved
  496. # unreliable, or the device died mid-read). abort() discards pending
  497. # buffers and unblocks it; close() is idempotent and the reader's
  498. # own finally will close it again.
  499. cap = self._cap
  500. if cap is not None:
  501. cap.close()
  502. with self._lock:
  503. # Keep the handle while the thread is still alive (join timed out) so
  504. # ``running`` stays truthful and the next start() does not double-arm.
  505. if self._thread is t and (t is None or not t.is_alive()):
  506. self._thread = None
  507. def _dispatch_wake(self) -> None:
  508. try:
  509. self.on_wake()
  510. except Exception as e:
  511. logger.warning("wake word callback failed: %s", e)
  512. finally:
  513. self._callback_inflight.clear()
  514. def _open_capture(self, frame_length: int) -> _Capture:
  515. """Open the audio source; raises on any local-mic failure."""
  516. if self.external_audio:
  517. with suppress(Exception): # drain stale frames from a previous arm
  518. while True:
  519. self._audio_q.get_nowait()
  520. logger.info("wake word: client-capture mode (frame=%d, rate=%d) — waiting for wake.feed",
  521. frame_length, SAMPLE_RATE)
  522. return _Capture(queue=self._audio_q, frame_length=frame_length)
  523. try:
  524. sd, np = _import_audio()
  525. except (ImportError, OSError) as e:
  526. logger.error("wake word: audio libraries unavailable: %s", e)
  527. raise
  528. details = self.input_device_details = _describe_input_device(self.input_device, sd)
  529. # Capture at the device's native rate when PortAudio reports one; frames are resampled to the engine.
  530. cap, rate = _Capture(np=np), details.get("default_samplerate")
  531. if isinstance(rate, (int, float)) and not isinstance(rate, bool) and rate > 0:
  532. with suppress(OverflowError, ValueError):
  533. cap.rate = round(rate)
  534. cap.frame_length = max(1, round(frame_length * cap.rate / SAMPLE_RATE))
  535. logger.info("wake word: opening microphone device=%s selector=%r hostapi=%s "
  536. "default_rate=%s capture_rate=%d engine_rate=%d", details.get("name") or "system default",
  537. self.input_device, details.get("hostapi") or "unknown",
  538. details.get("default_samplerate") or "unknown", cap.rate, SAMPLE_RATE)
  539. try:
  540. cap.stream = sd.InputStream(device=self.input_device, samplerate=cap.rate, channels=1,
  541. dtype="int16", blocksize=cap.frame_length)
  542. cap.stream.start()
  543. except Exception as e:
  544. cap.close()
  545. logger.error("wake word: failed to open microphone: %s", e)
  546. raise
  547. return cap
  548. def _note_silence(self, frame, silent_alert_frames: int) -> None:
  549. """Track consecutive near-zero frames; flag/unflag ``audio_silent``. ``frame`` is None when
  550. no client frame arrived (counts as silence for status, but is never logged as a dead mic)."""
  551. try:
  552. peak = 0 if frame is None or not len(frame) else int(abs(frame).max())
  553. except Exception:
  554. peak = _SILENCE_PEAK + 1
  555. if peak <= _SILENCE_PEAK:
  556. self._silent_frames += 1
  557. if self._silent_frames == silent_alert_frames:
  558. self.audio_silent = True
  559. if frame is not None:
  560. logger.warning("wake word: mic delivers only silence (peak<=%d for %ds); %s",
  561. _SILENCE_PEAK, _SILENCE_ALERT_SECONDS,
  562. silent_audio_hint(self.input_device_details))
  563. elif self._silent_frames:
  564. if self.audio_silent:
  565. logger.info("wake word: mic audio detected — stream healthy")
  566. self._silent_frames, self.audio_silent = 0, False
  567. def _fire(self) -> None:
  568. """Honor the cooldown, then run ``on_wake`` on its own thread (once)."""
  569. now = time.monotonic()
  570. if now - self._last_fire < self.cooldown:
  571. logger.debug("wake word: detection within cooldown — ignored")
  572. return
  573. self._last_fire = now
  574. logger.info("wake word: phrase detected — firing callback")
  575. if not self._callback_inflight.is_set():
  576. self._callback_inflight.set()
  577. threading.Thread(target=self._dispatch_wake, daemon=True, name="wake-word-callback").start()
  578. def _recover_capture(self, frame_length: int, retries) -> Optional[_Capture]:
  579. """Retry only an already-started local capture, keeping its owner and engine."""
  580. if self.external_audio:
  581. return None
  582. for delay in retries:
  583. logger.warning("wake word: reopening microphone in %.1fs", delay)
  584. if self._stop.wait(delay):
  585. return None
  586. try:
  587. cap = self._open_capture(frame_length)
  588. except Exception as e:
  589. logger.warning("wake word: microphone reopen failed: %s", e)
  590. continue
  591. with suppress(Exception):
  592. self.engine.reset()
  593. self.audio_silent, self._silent_frames = False, 0
  594. return cap
  595. logger.warning("wake word: microphone recovery exhausted; toggle wake word to retry")
  596. return None
  597. def _run(self, ready: threading.Event, startup_errors: list[BaseException]) -> None:
  598. frame_length = self.engine.frame_length
  599. try:
  600. cap = self._open_capture(frame_length)
  601. self._cap = cap
  602. except Exception as e:
  603. startup_errors.append(e)
  604. ready.set()
  605. return
  606. # Drop buffered audio/feature state so a resume right after a voice turn can't
  607. # re-fire on audio captured before the pause (wake → voice → resume → wake loop).
  608. with suppress(Exception):
  609. self.engine.reset()
  610. logger.info("wake word: listening (frame=%d, rate=%d, external=%s)",
  611. frame_length, SAMPLE_RATE, self.external_audio)
  612. ready.set()
  613. failed = False
  614. # Bound the whole arm, not just consecutive open failures: a device can
  615. # reopen successfully and fail its very next read forever.
  616. retries = iter((0.5, 1.0, 2.0))
  617. silent_alert_frames = max(1, int(_SILENCE_ALERT_SECONDS * SAMPLE_RATE / max(1, frame_length)))
  618. try:
  619. while not self._stop.is_set():
  620. try:
  621. data = cap.read(self._stop)
  622. except Exception as e:
  623. logger.warning("wake word: stream read error: %s", e)
  624. cap.close()
  625. cap = None
  626. if not self._stop.is_set():
  627. cap = self._recover_capture(frame_length, retries)
  628. if cap is not None:
  629. # Re-arm the halt hook: _halt_thread aborts whatever
  630. # self._cap points at, and recovery replaced the capture.
  631. self._cap = cap
  632. continue
  633. failed = not self._stop.is_set()
  634. break
  635. if data is None: # no client frames yet — counts as silence for status
  636. self._note_silence(None, silent_alert_frames)
  637. continue
  638. frame = data[:, 0] if getattr(data, "ndim", 1) == 2 else data
  639. if cap.rate != SAMPLE_RATE:
  640. frame = _resample_audio_frame(cap.np, frame, frame_length)
  641. self._note_silence(frame, silent_alert_frames)
  642. try:
  643. if self.engine.process(frame):
  644. self._fire()
  645. except Exception as e:
  646. logger.debug("wake word: engine error: %s", e)
  647. finally:
  648. if cap is not None:
  649. cap.close()
  650. logger.info("wake word: stream closed")
  651. if failed and self.on_failure is not None:
  652. self.on_failure(self)
  653. # ── Process-wide singleton (mirrors hermes_cli.voice's continuous API) ──
  654. _detector: Optional[WakeWordDetector] = None
  655. _detector_owner: object | None = None
  656. _detector_file_lock = None
  657. _detector_lock = threading.Lock()
  658. def _lock_path() -> Path:
  659. from hermes_constants import get_default_hermes_root
  660. return get_default_hermes_root() / "runtime" / "wake-word.lock"
  661. def _flock(handle, acquire: bool) -> None:
  662. """Non-blocking exclusive lock (or unlock) of one byte / whole file, per OS."""
  663. if os.name == "nt":
  664. import msvcrt
  665. if acquire: # msvcrt needs at least one byte to lock
  666. handle.seek(0, os.SEEK_END)
  667. if handle.tell() == 0:
  668. handle.write(b"\0")
  669. handle.flush()
  670. handle.seek(0)
  671. msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK if acquire else msvcrt.LK_UNLCK, 1)
  672. else:
  673. import fcntl
  674. fcntl.flock(handle.fileno(), (fcntl.LOCK_EX | fcntl.LOCK_NB) if acquire else fcntl.LOCK_UN)
  675. def _acquire_machine_lock(path: Optional[Path] = None):
  676. """Acquire the cross-process microphone lease, or raise WakeWordInUse."""
  677. lock_path = path or _lock_path()
  678. lock_path.parent.mkdir(parents=True, exist_ok=True)
  679. handle = open(lock_path, "a+b")
  680. try:
  681. _flock(handle, True)
  682. except OSError as e: # BlockingIOError is an OSError: lock held elsewhere
  683. handle.close()
  684. raise WakeWordInUse("Wake-word microphone is already owned.") from e
  685. return handle
  686. def _release_machine_lock(handle) -> None:
  687. if handle is None:
  688. return
  689. try:
  690. _flock(handle, False)
  691. except OSError:
  692. pass
  693. finally:
  694. handle.close()
  695. def _teardown_locked(close: Callable[[], None]) -> None:
  696. """Forget the singleton and run ``close``, always releasing the machine lease (caller holds the lock)."""
  697. global _detector, _detector_owner, _detector_file_lock
  698. lock_handle = _detector_file_lock
  699. _detector = _detector_owner = _detector_file_lock = None
  700. try:
  701. close()
  702. finally:
  703. _release_machine_lock(lock_handle)
  704. def _owned_detector(owner: object) -> Optional[WakeWordDetector]:
  705. """The armed detector iff ``owner`` holds the lease (caller holds the lock)."""
  706. return _detector if _detector is not None and _detector_owner is owner else None
  707. def _detector_failed(detector: WakeWordDetector) -> None:
  708. """Release ownership if the active microphone stream dies unexpectedly."""
  709. with _detector_lock:
  710. if _detector is detector:
  711. _teardown_locked(detector.engine.close)
  712. def start_listening(on_wake: Callable[[], None], *, owner: object, config: Optional[dict[str, Any]] = None,
  713. external_audio: bool = False) -> WakeWordDetector:
  714. """Claim, build, and start the detector. Idempotent for the same owner; a different owner
  715. (or process) gets :class:`WakeWordInUse`. Raises if engine construction fails (missing deps /
  716. access key / model) — callers should probe :func:`check_wake_word_requirements` first."""
  717. if owner is None:
  718. raise ValueError("wake-word owner must not be None")
  719. global _detector, _detector_owner, _detector_file_lock
  720. with _detector_lock:
  721. if _detector is not None:
  722. if _detector_owner is not owner:
  723. raise WakeWordInUse("Wake-word microphone is already owned.")
  724. _detector.on_wake = on_wake
  725. _detector.resume()
  726. return _detector
  727. _detector_file_lock = _acquire_machine_lock()
  728. try:
  729. cfg = config if config is not None else load_wake_word_config()
  730. engine_cfg = {**cfg, "capture": "client" if external_audio else "local"}
  731. _detector = WakeWordDetector(_build_engine(engine_cfg), on_wake, on_failure=_detector_failed,
  732. input_device=_input_device(cfg), external_audio=external_audio)
  733. _detector_owner = owner
  734. _detector.start()
  735. return _detector
  736. except Exception:
  737. with suppress(Exception):
  738. _teardown_locked(_detector.stop if _detector is not None else lambda: None)
  739. raise
  740. def _owned_call(owner: object, action: Optional[Callable[[WakeWordDetector], None]] = None) -> bool:
  741. """Under the lock, True iff ``owner`` holds the lease; also runs ``action(detector)`` when given."""
  742. with _detector_lock:
  743. det = _owned_detector(owner)
  744. if det is None:
  745. return False
  746. if action is not None:
  747. action(det)
  748. return True
  749. def owns_listener(owner: object) -> bool:
  750. return _owned_call(owner)
  751. def pause_listening(*, owner: object) -> bool:
  752. """Release the microphone only when ``owner`` holds the lease."""
  753. return _owned_call(owner, WakeWordDetector.pause)
  754. def resume_listening(*, owner: object) -> bool:
  755. """Re-open the microphone only when ``owner`` holds the lease."""
  756. return _owned_call(owner, WakeWordDetector.resume)
  757. def stop_listening(*, owner: object) -> bool:
  758. """Fully stop the detector only when ``owner`` holds the lease."""
  759. return _owned_call(owner, lambda det: _teardown_locked(det.stop))
  760. def _current_detector() -> Optional[WakeWordDetector]:
  761. with _detector_lock:
  762. return _detector
  763. def is_listening() -> bool:
  764. return (det := _current_detector()) is not None and det.running
  765. def audio_is_silent() -> bool:
  766. """True when the armed stream opens fine but delivers only silence (dead mic), so
  767. detection can never fire; status shows "listening but the microphone appears silent"."""
  768. return (det := _current_detector()) is not None and det.audio_silent
  769. def get_input_device_status(cfg: Optional[dict[str, Any]] = None) -> dict[str, Any]:
  770. """Return configured/active PortAudio input diagnostics for status UIs."""
  771. if (det := _current_detector()) is not None:
  772. return dict(det.input_device_details)
  773. return _describe_input_device(_input_device(cfg if cfg is not None else load_wake_word_config()))
  774. def get_last_match() -> Optional[tuple[str, str]]:
  775. """(matched phrase, profile) of the most recent wake fire when the engine reports
  776. per-phrase matches (sherpa multi-profile routing); None otherwise."""
  777. return None if (det := _current_detector()) is None else getattr(det.engine, "last_match", None)
  778. def feed_audio(*, owner: object, pcm_int16) -> bool:
  779. """Push client-captured PCM into ``owner``'s armed detector; True when accepted."""
  780. with _detector_lock:
  781. det = _owned_detector(owner)
  782. if det is None or not det.external_audio:
  783. return False
  784. det.feed(pcm_int16)
  785. return True
  786. def detector_frame_info() -> dict[str, Any]:
  787. """Sample rate + frame length for client capture streamers."""
  788. if (det := _current_detector()) is None:
  789. return {"sample_rate": SAMPLE_RATE, "frame_length": 1280}
  790. return {"sample_rate": SAMPLE_RATE, "external_audio": bool(det.external_audio),
  791. "frame_length": int(getattr(det.engine, "frame_length", 1280) or 1280)}