wake_word_engines.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281
  1. """Wake-word hotword engines (pyopen-wakeword / sherpa-onnx KWS / Porcupine).
  2. All three run fully on-device. Config, platform probes and sensitivity accessors
  3. live in :mod:`tools.wake_word`; engines read them lazily through that module (import cycle).
  4. This module is the SINGLE owner of the engine implementations — ``tools.wake_word``
  5. imports these classes and must not shadow them with copies.
  6. Dependency admission: constructing an engine ensures its ``wake-*`` extra. The
  7. ``audio-io`` extra (sounddevice + numpy) is ensured only when the resolved
  8. capture mode is ``local`` — client capture (desktop/TUI streaming PCM via
  9. ``wake.feed``) never needs, and must never trigger installation of, local
  10. audio libraries.
  11. """
  12. from __future__ import annotations
  13. import logging
  14. import os
  15. from contextlib import suppress
  16. from pathlib import Path
  17. from typing import Any, Dict, Optional
  18. logger = logging.getLogger("tools.wake_word")
  19. def _ww():
  20. from tools import wake_word
  21. return wake_word
  22. def _ensure_dep(feature: str, cfg: dict[str, Any]) -> None:
  23. import pm
  24. pm.ensure_import(feature)
  25. # Only local capture needs the microphone dependencies.
  26. if _ww().resolve_capture_mode(cfg) == "local" and not pm.available("audio-io"):
  27. pm.ensure_import("audio-io")
  28. class _Engine:
  29. """Minimal hotword-engine contract: feed int16 frames, get a bool. Subclasses set ``feature``
  30. (the pm extra ensured before ``_build``) and their own ``cfg`` sub-section ``section``."""
  31. feature: str = ""
  32. section: str = ""
  33. frame_length: int = 1280 # 80 ms at 16 kHz
  34. #: (matched phrase, profile name) of the most recent fire. Multi-phrase engines
  35. #: (sherpa) set this for profile routing; single-phrase engines leave it None.
  36. last_match: Optional[tuple[str, str]] = None
  37. def __init__(self, cfg: dict[str, Any]):
  38. _ensure_dep(self.feature, cfg)
  39. self._build(cfg, _sub(cfg, self.section), _ww())
  40. def _build(self, cfg: dict[str, Any], sub: dict[str, Any], ww) -> None:
  41. raise NotImplementedError
  42. def process(self, frame) -> bool: # frame: 1-D int16 ndarray
  43. raise NotImplementedError
  44. def reset(self) -> None:
  45. """Clear any internal audio/feature buffer (called on every (re)start)."""
  46. def close(self) -> None:
  47. """Release engine resources (called once on stop)."""
  48. def _looks_like_path(value: str) -> bool:
  49. return os.sep in value or value.endswith((".onnx", ".tflite", ".ppn")) or os.path.exists(value)
  50. def _sub(cfg: dict[str, Any], key: str) -> dict[str, Any]:
  51. sub = cfg.get(key)
  52. return sub if isinstance(sub, dict) else {}
  53. class _OpenWakeWordEngine(_Engine):
  54. """pyopen-wakeword — free, local hotword detection (TFLite via a bundled
  55. tensorflowlite_c lib; no runtime download, no framework choice). Scores one
  56. ~80 ms frame at a time; ``sensitivity`` IS the raw 0..1 threshold (higher =
  57. stricter). A real utterance holds the score high across frames while a stray
  58. phoneme spikes one, so ``confirmation_frames`` hits are required."""
  59. feature, section = "wake-openwakeword", "openwakeword"
  60. def _build(self, cfg, sub, ww) -> None:
  61. from pyopen_wakeword import OpenWakeWord, OpenWakeWordFeatures
  62. model_ref = str(sub.get("model") or ww._BUNDLED_MODEL_NAME).strip()
  63. # Default (or explicit "hey_hermes") → the bundled model; a custom path
  64. # is used as-is. pyopen-wakeword bundles the shared feature models
  65. # (melspectrogram + embedding — byte-identical to the openWakeWord
  66. # v0.5.1 files) inside its wheel, so there is no download_models step.
  67. if model_ref.lower() in ww._BUNDLED_MODEL_ALIASES:
  68. model_ref = ww._bundled_wakeword_path()
  69. # pyopen-wakeword returns a 0..1 score per completed window; sensitivity
  70. # IS the raw threshold a score must clear. Higher = stricter (fewer
  71. # false fires). Default 0.6 sits above openWakeWord's permissive 0.5
  72. # baseline, which let near-misses like "hey hor" through.
  73. self._threshold = ww._sensitivity(cfg)
  74. self._confirm_needed = ww._confirmation_frames(cfg)
  75. self._confirm_streak = 0
  76. self._features = OpenWakeWordFeatures.from_builtin()
  77. self._model = OpenWakeWord.from_model(model_ref)
  78. self._labels = [self._model.id]
  79. def process(self, frame) -> bool:
  80. # frame is a 1-D int16 ndarray; the features pipeline consumes int16
  81. # bytes. process_streaming() yields embeddings as the window fills and
  82. # the model yields one 0..1 score per completed window.
  83. over = False
  84. for emb in self._features.process_streaming(frame.tobytes()):
  85. for score in self._model.process_streaming(emb):
  86. if score >= self._threshold:
  87. over = True
  88. # Require N consecutive over-threshold frames: a real phrase holds the
  89. # score high across frames, a stray ambient phoneme spikes just one.
  90. if over:
  91. self._confirm_streak += 1
  92. if self._confirm_streak >= self._confirm_needed:
  93. self._confirm_streak = 0
  94. return True
  95. return False
  96. self._confirm_streak = 0
  97. return False
  98. def reset(self) -> None:
  99. # Clears pyopen-wakeword's rolling feature/prediction buffer so stale
  100. # audio captured before a pause can't re-fire the moment we resume.
  101. self._confirm_streak = 0
  102. with suppress(Exception):
  103. self._features.reset()
  104. self._model.reset()
  105. def close(self) -> None:
  106. self.reset()
  107. with suppress(Exception):
  108. self._features.close()
  109. self._model.close()
  110. # sherpa-onnx open-vocabulary KWS model: small streaming zipformer transducer (English,
  111. # GigaSpeech), downloaded once under HERMES_HOME. Keywords are tokenized at RUNTIME.
  112. _SHERPA_KWS_MODEL_URL = (
  113. "https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/"
  114. "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.tar.bz2"
  115. )
  116. _SHERPA_KWS_MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"
  117. def _sherpa_model_root() -> Path:
  118. from hermes_constants import get_hermes_home
  119. return get_hermes_home() / "cache" / "wakewords"
  120. def _ensure_sherpa_model(root: Optional[Path] = None) -> Path:
  121. """Download + unpack the sherpa KWS model once; return its directory."""
  122. root = root or _sherpa_model_root()
  123. target = root / _SHERPA_KWS_MODEL_DIR
  124. if (target / "tokens.txt").exists():
  125. return target
  126. import tarfile
  127. import urllib.request
  128. root.mkdir(parents=True, exist_ok=True)
  129. archive = root / f"{_SHERPA_KWS_MODEL_DIR}.tar.bz2"
  130. logger.info("wake word: downloading sherpa KWS model (one-time, ~13 MB)")
  131. urllib.request.urlretrieve(_SHERPA_KWS_MODEL_URL, archive)
  132. with tarfile.open(archive, "r:bz2") as tf:
  133. tf.extractall(root, filter="data")
  134. archive.unlink(missing_ok=True)
  135. if not (target / "tokens.txt").exists():
  136. raise RuntimeError(f"sherpa KWS model unpack failed: {target}")
  137. return target
  138. class _SherpaKwsEngine(_Engine):
  139. """sherpa-onnx open-vocabulary keyword spotting — any typed phrase, zero training. ``wake_word.phrase``
  140. is BPE-tokenized at runtime against the model's vocabulary: DETECTION config, not a cosmetic label."""
  141. feature, section = "wake-sherpa", "sherpa"
  142. frame_length = 1280 # streaming zipformer accepts any chunk; match capture path.
  143. def _build(self, cfg, sub, ww) -> None:
  144. import sherpa_onnx
  145. import tempfile
  146. from sherpa_onnx import text2token
  147. model_dir = str(sub.get("model_dir") or "").strip()
  148. d = Path(model_dir) if model_dir else _ensure_sherpa_model()
  149. if not (d / "tokens.txt").exists():
  150. raise RuntimeError(f"sherpa KWS model not found at {d}")
  151. # Phrase set: this profile's phrase plus — with profile routing on — every other
  152. # wake-enabled profile's phrase, so ONE listener can wake any profile.
  153. phrase = str(ww._get(cfg, "phrase") or "hey hermes").strip()
  154. phrase_map: dict[str, str] = {phrase: ww._active_profile_name()}
  155. if bool(cfg.get("profile_routing", True)):
  156. for prof, p in ww.enrolled_profile_phrases().items():
  157. phrase_map.setdefault(p.strip(), prof)
  158. phrases = list(phrase_map)
  159. tokens = text2token([p.upper() for p in phrases], tokens=str(d / "tokens.txt"), tokens_type="bpe",
  160. bpe_model=str(d / "bpe.model"))
  161. # sherpa keyword entries reject spaces in the @display-name; underscore them and
  162. # map display → profile for match routing.
  163. self._display_to_profile: dict[str, str] = {}
  164. kw = tempfile.NamedTemporaryFile(mode="w", suffix=".txt", prefix="hermes-kws-", delete=False,
  165. encoding="utf-8")
  166. for p, toks in zip(phrases, tokens):
  167. display = p.upper().replace(" ", "_")
  168. self._display_to_profile[display] = phrase_map[p]
  169. kw.write(" ".join(toks) + f" @{display}\n")
  170. kw.close()
  171. self._keywords_file = kw.name
  172. # Shared 0..1 sensitivity → sherpa keywords_threshold. 0.5 lands on sherpa's
  173. # recommended 0.25; a stricter 0.35 missed ~12% of true positives in live TTS
  174. # matrix tests while 0.25 held zero false fires.
  175. threshold = 0.05 + 0.4 * ww._sensitivity(cfg)
  176. def _model_file(part: str) -> str:
  177. hits = sorted(d.glob(f"{part}-*[!8].onnx"))
  178. if not hits:
  179. raise RuntimeError(f"sherpa KWS model file missing: {d}/{part}-*[!8].onnx")
  180. return str(hits[0])
  181. self._spotter = sherpa_onnx.KeywordSpotter(
  182. tokens=str(d / "tokens.txt"), encoder=_model_file("encoder"), decoder=_model_file("decoder"),
  183. joiner=_model_file("joiner"), keywords_file=self._keywords_file, keywords_threshold=threshold,
  184. num_threads=1,
  185. )
  186. self._stream = self._spotter.create_stream()
  187. def process(self, frame) -> bool:
  188. import numpy as np
  189. self._stream.accept_waveform(_ww().SAMPLE_RATE, np.asarray(frame, dtype=np.float32) / 32768.0)
  190. fired = False
  191. while self._spotter.is_ready(self._stream):
  192. self._spotter.decode_stream(self._stream)
  193. result = self._spotter.get_result(self._stream)
  194. if result:
  195. fired, display = True, str(result)
  196. self.last_match = (display.replace("_", " ").lower(),
  197. self._display_to_profile.get(display, ""))
  198. self._spotter.reset_stream(self._stream) # one utterance must not fire repeatedly
  199. return fired
  200. def reset(self) -> None:
  201. # Fresh stream drops buffered audio/decoder state (pause → resume must not re-fire).
  202. with suppress(Exception):
  203. self._stream = self._spotter.create_stream()
  204. def close(self) -> None:
  205. with suppress(OSError):
  206. os.unlink(self._keywords_file)
  207. class _PorcupineEngine(_Engine):
  208. """Picovoice Porcupine — premium, on-device, needs an access key."""
  209. feature, section = "wake-porcupine", "porcupine"
  210. def _build(self, cfg, sub, ww) -> None:
  211. import pvporcupine
  212. access_key = (os.getenv("PORCUPINE_ACCESS_KEY") or "").strip()
  213. if not access_key:
  214. raise RuntimeError("Porcupine wake word requires PORCUPINE_ACCESS_KEY "
  215. "(get a free key at https://console.picovoice.ai).")
  216. keyword = str(sub.get("keyword") or "jarvis").strip()
  217. # Porcupine's `sensitivities` runs the OPPOSITE way to our shared knob (higher =
  218. # looser); invert so "higher = stricter" holds for every engine.
  219. kwargs: dict[str, Any] = {"access_key": access_key, "sensitivities": [1.0 - ww._sensitivity(cfg)]}
  220. kwargs["keyword_paths" if _looks_like_path(keyword) else "keywords"] = [keyword]
  221. self._porcupine = pvporcupine.create(**kwargs)
  222. self.frame_length = self._porcupine.frame_length
  223. def process(self, frame) -> bool:
  224. return self._porcupine.process(frame) >= 0 # pvporcupine wants a plain sequence of int16
  225. def close(self) -> None:
  226. with suppress(Exception):
  227. self._porcupine.delete()