voice.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681
  1. """Process-wide voice recording + TTS API for the TUI gateway."""
  2. from __future__ import annotations
  3. import contextlib
  4. import json
  5. import logging
  6. import os
  7. import re
  8. import sys
  9. import tempfile
  10. import threading
  11. import time
  12. from typing import Any, Callable, Optional
  13. # Modifier aliases mirrored from the TUI parser (``ui-tui/src/lib/platform.ts`` ``_MOD_ALIASES``)
  14. # so one config value binds the same shortcut in both runtimes. ``super``/``win``/``windows`` are
  15. # deliberately absent: prompt_toolkit has no super/meta modifier, so those spellings are TUI-only
  16. # and normalize to the default (a silent fallback beats a hard startup crash; cli.py warns).
  17. _VOICE_MOD_ALIASES = {"ctrl": "c-", "control": "c-", "alt": "a-", "option": "a-", "opt": "a-"}
  18. # Named keys prompt_toolkit accepts as ``c-<name>`` / ``a-<name>``; aliases collapse to canonical.
  19. # Aliases collapse to prompt_toolkit's canonical spelling so the same config value binds identically in both
  20. # runtimes (review round-10 on #19835).
  21. _VOICE_NAMED_KEYS = {
  22. "space": "space", "spc": "space", "enter": "enter", "return": "enter", "ret": "enter", "tab": "tab",
  23. "escape": "escape", "esc": "escape", "backspace": "backspace", "bs": "backspace", "delete": "delete", "del": "delete",
  24. }
  25. # ``useInputHandlers()`` intercepts ctrl+c/d/l (interrupt/quit/clear) before the voice check, so
  26. # such a binding would be advertised but never fire (same blocklist as the TUI parser). On macOS
  27. # the CLI's copy/exit/clear bindings also claim ``a-c``/``a-d``/``a-l`` (hermes-ink reports Alt as
  28. # ``key.meta``), mirroring the TUI's darwin-only reservation — alt is reserved on darwin only.
  29. _VOICE_RESERVED_CHARS = frozenset({"c", "d", "l"})
  30. # On macOS the classic CLI's prompt_toolkit bindings for copy / exit / clear also claim ``a-c`` / ``a-d`` /
  31. # ``a-l`` via the action-modifier lookup, and hermes-ink reports Alt as ``key.meta`` on many terminals.
  32. # Mirror the TUI parser's darwin-only reservation so ``option+c`` etc. don't bind Alt+C in the CLI while the
  33. # TUI silently falls back to Ctrl+B (review round-14 on #19835).
  34. _DEFAULT_PT_KEY = "c-b"
  35. def voice_record_key_from_config(cfg: Any) -> Any:
  36. """Shape-safe ``cfg.voice.record_key``: a hand-edited ``voice: true`` / ``voice: cmd+b`` leaves
  37. ``cfg["voice"]`` as a bool/str and the naive ``.get`` chain would raise before voice starts.
  38. ``load_config()`` deep-merges raw YAML and preserves scalar overrides, so a hand-edited ``voice: true``
  39. / ``voice: cmd+b`` leaves ``cfg["voice"]`` as a bool/str instead of a dict, and the naive
  40. ``.get("voice", {}).get("record_key")`` chain raises AttributeError before voice can even start (review
  41. round-11 on 19835). Return ``None`` for malformed shapes so call sites can feed the result straight into
  42. the normalizer/formatter and get the documented default. See #19835.
  43. """
  44. voice = cfg.get("voice") if isinstance(cfg, dict) else None
  45. return voice.get("record_key") if isinstance(voice, dict) else None
  46. def normalize_voice_record_key_for_prompt_toolkit(raw: Any) -> str:
  47. """Coerce ``voice.record_key`` into prompt_toolkit's ``c-x`` / ``a-x`` format.
  48. Mirrors the TUI parser contract (``ui-tui/src/lib/platform.ts``): non-string / empty / typo'd /
  49. bare-char / multi-modifier / reserved ``ctrl+c|d|l`` / ``super``-family → the documented default
  50. ``c-b``; named keys collapse to canonical spelling (``ctrl+return`` → ``c-enter``). Exactly one
  51. modifier: multi-modifier chords bind different shortcuts in prompt_toolkit (a-c-r) and
  52. hermes-ink rejects them; a bare key is refused by the TUI parser.
  53. * ``super`` / ``win`` / ``windows`` → ``c-b`` (TUI-only modifiers — prompt_toolkit has no super mod; the
  54. CLI binding site is expected to warn when this fallback fires so users see the cross-runtime split,
  55. review round-11 on #19835)
  56. """
  57. if not isinstance(raw, str):
  58. return _DEFAULT_PT_KEY
  59. parts = [p.strip() for p in raw.strip().lower().split("+") if p.strip()]
  60. if len(parts) != 2:
  61. return _DEFAULT_PT_KEY
  62. modifier_token, key_token = parts
  63. # ``super`` / ``win`` / ``windows`` are TUI-only (prompt_toolkit has no super modifier, so
  64. # ``@kb.add(super+b)`` crashes the CLI at startup). Fall back to the documented default here; the CLI
  65. # binding site is expected to log a warning when the configured value is one of these spellings so users
  66. # know the TUI+CLI runtimes diverge on that shortcut (review round-11 on #19835).
  67. normalized_mod = _VOICE_MOD_ALIASES.get(modifier_token)
  68. if not normalized_mod:
  69. return _DEFAULT_PT_KEY
  70. if len(key_token) == 1:
  71. reserved = (normalized_mod == "c-" or sys.platform == "darwin") and key_token in _VOICE_RESERVED_CHARS
  72. return _DEFAULT_PT_KEY if reserved else f"{normalized_mod}{key_token}"
  73. # Multi-char token must be a known named key; ``ctrl+spcae`` must not pass through as
  74. # ``c-spcae`` (prompt_toolkit would reject it).
  75. named = _VOICE_NAMED_KEYS.get(key_token)
  76. return f"{normalized_mod}{named}" if named else _DEFAULT_PT_KEY
  77. def pt_key_to_sequence(pt_key: str) -> tuple[str, ...]:
  78. """Convert a prompt_toolkit key specifier (e.g. 'c-b' or 'a-v') to a sequence tuple."""
  79. if isinstance(pt_key, str) and pt_key.startswith("a-"):
  80. return ("escape", pt_key[2:])
  81. return (pt_key,)
  82. def format_voice_record_key_for_status(raw: Any) -> str:
  83. """Render ``voice.record_key`` for ``/voice status`` as ``Ctrl+B`` / ``Alt+Space``; malformed
  84. configs surface as the default so status never advertises a shortcut that won't bind.
  85. Mirrors the TUI's ``formatVoiceRecordKey``: returns ``Ctrl+B`` / ``Alt+Space`` / ``Ctrl+Enter``. See
  86. #19835.
  87. """
  88. normalized = normalize_voice_record_key_for_prompt_toolkit(raw)
  89. prefix = "Alt+" if normalized.startswith("a-") else "Ctrl+"
  90. key = normalized[2:]
  91. return prefix + key[0].upper() + key[1:]
  92. from tools.voice_mode_transcript import is_voice_stop_phrase, is_whisper_hallucination
  93. from tools.voice_mode import create_audio_recorder, play_audio_file, transcribe_recording
  94. logger = logging.getLogger(__name__)
  95. def _debug(msg: str) -> None:
  96. """HERMES_VOICE_DEBUG=1 breadcrumb on stderr (the TUI gateway shows it as a gateway.stderr
  97. Activity line). Broken pipes are swallowed: this fires from background threads where a dead
  98. stderr must not kill the gateway — the stdin/stdout command pipe is what matters."""
  99. if os.environ.get("HERMES_VOICE_DEBUG", "").strip() == "1":
  100. with contextlib.suppress(BrokenPipeError, OSError):
  101. print(f"[voice] {msg}", file=sys.stderr, flush=True)
  102. def _beeps_enabled() -> bool:
  103. """CLI parity: voice.beep_enabled in config.yaml (default True)."""
  104. try:
  105. from hermes_cli.config import load_config
  106. from utils import is_truthy_value
  107. voice_cfg = load_config().get("voice", {})
  108. if isinstance(voice_cfg, dict):
  109. # is_truthy_value handles quoted YAML strings like "false" that bool() misreads.
  110. # See #49883.
  111. # See #49883.
  112. return is_truthy_value(voice_cfg.get("beep_enabled", True), default=True)
  113. except Exception:
  114. pass
  115. return True
  116. def _play_beep(frequency: int, count: int = 1) -> None:
  117. """Audible cue matching cli.py's beeps (880 Hz once on start, 660 Hz twice on stop).
  118. Best-effort — a missing speaker must never break the voice loop."""
  119. if not _beeps_enabled():
  120. return
  121. try:
  122. from tools.voice_mode import play_beep
  123. play_beep(frequency=frequency, count=count)
  124. except Exception as e:
  125. _debug(f"beep {frequency}Hz failed: {e}")
  126. def _safe_call(cb: Optional[Callable], *args: Any, warn: Optional[str] = None) -> None:
  127. """Invoke an optional callback, swallowing its exceptions. ``warn`` is a ``logger.warning``
  128. format with one ``%s`` slot; without it failures are silent (status callbacks are fire-and-forget)."""
  129. if not cb:
  130. return
  131. try:
  132. cb(*args)
  133. except Exception as e:
  134. if warn:
  135. logger.warning(warn, e)
  136. def _transcribe_wav(wav_path: str, fail_msg: str, debug_prefix: Optional[str] = None) -> Optional[str]:
  137. """Transcribe ``wav_path``, unlink it, and return the cleaned transcript (or None).
  138. transcribe_recording returns {"success", "transcript", "error"?} — NOT {"text"}; the wrong key
  139. silently masqueraded as "not hearing the user". Empty text and Whisper hallucinations are
  140. dropped; failures are logged with ``fail_msg``.
  141. """
  142. try:
  143. result = transcribe_recording(wav_path)
  144. success = bool(result.get("success"))
  145. text = (result.get("transcript") or "").strip()
  146. if debug_prefix:
  147. _debug(f"{debug_prefix}: transcribe -> success={success} text={text!r} err={result.get('error')!r}")
  148. if success and text and not is_whisper_hallucination(text):
  149. return text
  150. except Exception as e:
  151. logger.warning(fail_msg, e)
  152. if debug_prefix:
  153. _debug(f"{debug_prefix}: transcribe raised {type(e).__name__}: {e}")
  154. finally:
  155. with contextlib.suppress(Exception):
  156. if os.path.isfile(wav_path):
  157. os.unlink(wav_path)
  158. return None
  159. def _deactivate(on_status: Optional[Callable[[str], None]] = None) -> None:
  160. """Mark the continuous loop inactive and (optionally) report ``"idle"``."""
  161. global _continuous_active
  162. with _continuous_lock:
  163. _continuous_active = False
  164. _safe_call(on_status, "idle")
  165. # Push-to-talk state.
  166. _recorder = None
  167. _recorder_lock = threading.Lock()
  168. # Continuous (VAD) state.
  169. _continuous_lock = threading.Lock()
  170. _continuous_active = False
  171. _continuous_stopping = False
  172. _continuous_auto_restart: bool = True
  173. _continuous_recorder: Any = None
  174. # TTS-vs-STT feedback guard: TTS over the speakers lands in the live mic and gets transcribed as
  175. # user input — an infinite loop the agent happily joins. Mirrors cli.py:_voice_tts_done: cleared
  176. # while speak_text plays, set while silent. The silence callback waits on it before re-arming;
  177. # speak_text cancels live capture before playback so the previous utterance's tail doesn't leak.
  178. _tts_playing = threading.Event()
  179. _tts_playing.set() # initially "not playing"
  180. # Silence-count hold: while the agent is mid-turn (possibly minutes) or TTS plays, the user is
  181. # CORRECTLY silent — those cycles must not count toward the no-speech limit or a long tool run
  182. # ends the voice chat under the user. The host surface registers a probe reporting "agent busy".
  183. _voice_busy_probe: Optional[Callable[[], bool]] = None
  184. def set_voice_busy_probe(probe: Optional[Callable[[], bool]]) -> None:
  185. """Register a callable returning True while the agent is mid-turn; ``None`` clears it.
  186. Must be cheap and thread-safe — it runs on the silence-callback thread."""
  187. global _voice_busy_probe
  188. _voice_busy_probe = probe
  189. def _voice_activity_held() -> bool:
  190. """True while silent cycles must NOT count toward the no-speech limit (TTS playing or agent
  191. mid-turn). Fail-open to "not held" so a broken probe can never make the voice chat immortal."""
  192. if not _tts_playing.is_set():
  193. return True
  194. try:
  195. return _voice_busy_probe is not None and bool(_voice_busy_probe())
  196. except Exception:
  197. return False
  198. # (on_transcript, on_status, on_silent_limit, on_stop_phrase) of the active loop; guarded by
  199. # ``_continuous_lock``. on_stop_phrase is the explicit user-intent stop (the user SAYS a bare stop
  200. # phrase), distinct from on_silent_limit (a timeout) so consumers end the conversation like a
  201. # manual stop instead of reporting "no speech detected"; unset → on_silent_limit fires.
  202. _NO_CALLBACKS: tuple = (None, None, None, None)
  203. _continuous_callbacks: tuple = _NO_CALLBACKS
  204. _continuous_no_speech_count = 0
  205. _CONTINUOUS_NO_SPEECH_LIMIT = 3
  206. def _turn_transcript(
  207. wav_path: Optional[str], fail_msg: str, where: str, tail: str, trace: bool = False
  208. ) -> tuple[Optional[str], bool, str]:
  209. """Transcribe a finished capture → (deliverable text, is_stop_phrase, stop_text).
  210. A bare stop phrase ("stop") is explicit user intent to end the voice chat: it is never sent
  211. to the agent, so the deliverable text becomes None. ``trace`` adds transcription breadcrumbs.
  212. """
  213. transcript = _transcribe_wav(wav_path, fail_msg, where if trace else None) if wav_path else None
  214. if not (transcript and is_voice_stop_phrase(transcript)):
  215. return transcript, False, ""
  216. _debug(f"{where}: stop phrase {transcript!r} — {tail}")
  217. return None, True, transcript
  218. def _signal_halt(stop_phrase: bool, stop_text: str, on_stop_phrase, on_silent_limit) -> None:
  219. """Dedicated stop-phrase signal when wired, else the legacy on_silent_limit fallback."""
  220. if stop_phrase and on_stop_phrase is not None:
  221. _safe_call(on_stop_phrase, stop_text)
  222. else:
  223. _safe_call(on_silent_limit)
  224. def _tally_silence(spoke: bool, held: bool, where: str) -> tuple[bool, int]:
  225. """Update the no-speech counter (caller holds ``_continuous_lock``): speech resets it, a held
  226. cycle (agent busy / TTS playing) is ignored, otherwise it bumps. Returns (limit_hit,
  227. count_after_bump); the counter is reset when the limit is hit."""
  228. global _continuous_no_speech_count
  229. if spoke:
  230. _continuous_no_speech_count = 0
  231. return False, 0
  232. if held:
  233. _debug(f"{where}: silent cycle ignored (agent busy or TTS playing)")
  234. return False, _continuous_no_speech_count
  235. _continuous_no_speech_count += 1
  236. count = _continuous_no_speech_count
  237. if count >= _CONTINUOUS_NO_SPEECH_LIMIT:
  238. _continuous_no_speech_count = 0
  239. return True, count
  240. return False, count
  241. # ── Push-to-talk API ─────────────────────────────────────────────────
  242. def start_recording() -> None:
  243. """Begin capturing from the default input device (push-to-talk)."""
  244. global _recorder
  245. with _recorder_lock:
  246. if _recorder is not None and getattr(_recorder, "is_recording", False):
  247. return
  248. rec = create_audio_recorder()
  249. rec.start() # only publish a recorder that actually started
  250. _recorder = rec
  251. def stop_and_transcribe() -> Optional[str]:
  252. """Stop the active push-to-talk recording, transcribe, return text."""
  253. global _recorder
  254. with _recorder_lock:
  255. rec = _recorder
  256. _recorder = None
  257. if rec is None:
  258. return None
  259. wav_path = rec.stop()
  260. return _transcribe_wav(wav_path, "voice transcription failed: %s") if wav_path else None
  261. # ── Continuous (VAD) API ─────────────────────────────────────────────
  262. def start_continuous(
  263. on_transcript: Callable[[str], None],
  264. on_status: Optional[Callable[[str], None]] = None,
  265. on_silent_limit: Optional[Callable[[], None]] = None,
  266. silence_threshold: int = 200,
  267. silence_duration: float = 3.0,
  268. auto_restart: bool = True,
  269. max_recording_seconds: float = 0.0,
  270. on_stop_phrase: Optional[Callable[[str], None]] = None,
  271. on_partial: Optional[Callable[[str], None]] = None,
  272. ) -> bool:
  273. """Start a VAD-driven continuous recording loop.
  274. ``max_recording_seconds`` caps a single recording (``voice.max_recording_seconds``);
  275. non-positive / non-numeric disables the cap. ``on_stop_phrase`` receives the stripped
  276. transcript when the user utters a bare stop phrase (``voice.stop_phrases``); the loop halts
  277. first, so the consumer only reflects "voice off" — like the manual stop control.
  278. ``on_partial`` receives live STT text while the user speaks (``stt.streaming``).
  279. """
  280. global _continuous_active, _continuous_recorder, _continuous_auto_restart, _continuous_no_speech_count
  281. global _continuous_callbacks
  282. with _continuous_lock:
  283. if _continuous_active:
  284. _debug("start_continuous: already active — no-op")
  285. return True
  286. if _continuous_stopping:
  287. _debug("start_continuous: stop/transcribe in progress — busy")
  288. return False
  289. _continuous_active = True
  290. _continuous_auto_restart = auto_restart
  291. _continuous_callbacks = (on_transcript, on_status, on_silent_limit, on_stop_phrase)
  292. if auto_restart:
  293. _continuous_no_speech_count = 0
  294. if _continuous_recorder is None:
  295. _continuous_recorder = create_audio_recorder()
  296. rec = _continuous_recorder
  297. rec.on_live_partial = on_partial
  298. rec._silence_threshold = silence_threshold
  299. rec._silence_duration = silence_duration
  300. # Same numeric-with-bool-excluded guard as cli.py:_voice_start_recording.
  301. cap_ok = isinstance(max_recording_seconds, (int, float)) and not isinstance(max_recording_seconds, bool)
  302. rec._max_recording_seconds = max_recording_seconds if cap_ok and max_recording_seconds > 0 else 0.0
  303. _debug(f"start_continuous: begin (threshold={silence_threshold}, duration={silence_duration}s)")
  304. # CLI parity: beep *before* opening the stream — after stream.start() it triggers a CoreAudio
  305. # conflict on macOS.
  306. _play_beep(frequency=880, count=1)
  307. try:
  308. rec.start(on_silence_stop=_continuous_on_silence)
  309. except Exception as e:
  310. logger.error("failed to start continuous recording: %s", e)
  311. _debug(f"start_continuous: rec.start raised {type(e).__name__}: {e}")
  312. _deactivate()
  313. raise
  314. _safe_call(on_status, "listening")
  315. return True
  316. def stop_continuous(force_transcribe: bool = False) -> None:
  317. """Stop the active continuous loop and release the microphone.
  318. Idempotent. With ``force_transcribe`` the recorder stops synchronously, then
  319. transcription/cleanup runs on a background thread before reporting ``"idle"``; otherwise
  320. the buffer is discarded.
  321. """
  322. global _continuous_active, _continuous_stopping, _continuous_recorder, _continuous_no_speech_count
  323. global _continuous_callbacks
  324. with _continuous_lock:
  325. if not _continuous_active:
  326. return
  327. _continuous_active = False
  328. rec = _continuous_recorder
  329. callbacks = _continuous_callbacks
  330. track_no_speech = force_transcribe and not _continuous_auto_restart
  331. _continuous_stopping = rec is not None
  332. _continuous_callbacks = _NO_CALLBACKS
  333. if not track_no_speech:
  334. _continuous_no_speech_count = 0
  335. on_transcript, on_status = callbacks[0], callbacks[1]
  336. if rec is not None:
  337. if force_transcribe and on_transcript:
  338. _safe_call(on_status, "transcribing")
  339. try:
  340. wav_path = rec.stop()
  341. except Exception as e:
  342. logger.warning("failed to stop recorder: %s", e)
  343. _safe_call(rec.cancel, warn="failed to cancel recorder: %s")
  344. wav_path = None
  345. threading.Thread(
  346. target=_finish_forced_stop, args=(wav_path, callbacks, track_no_speech), daemon=True
  347. ).start()
  348. return
  349. # cancel() (not stop()) discards buffered frames — the loop is over, we don't want to
  350. # transcribe a half-captured turn.
  351. _safe_call(rec.cancel, warn="failed to cancel recorder: %s")
  352. _finish_stop(on_status)
  353. def _finish_forced_stop(wav_path: Optional[str], callbacks: tuple, track_no_speech: bool) -> None:
  354. """Background tail of ``stop_continuous(force_transcribe=True)``: transcribe, deliver, tally."""
  355. on_transcript, on_status, on_silent_limit, on_stop_phrase = callbacks
  356. # With auto_restart=False the CLIENT drives the loop, so a stop phrase must fire the stop
  357. # signal — discarding the transcript alone would leave the conversation running forever.
  358. transcript, stop_phrase, stop_text = _turn_transcript(
  359. wav_path, "failed to stop/transcribe recorder: %s", "stop_continuous", "ending voice chat"
  360. )
  361. if stop_phrase:
  362. _signal_halt(True, stop_text, on_stop_phrase, on_silent_limit)
  363. if transcript:
  364. _safe_call(on_transcript, transcript, warn="on_transcript callback raised: %s")
  365. if track_no_speech:
  366. held = _voice_activity_held()
  367. with _continuous_lock:
  368. should_halt, _ = _tally_silence(bool(transcript) or stop_phrase, held, "stop_continuous")
  369. if should_halt:
  370. _safe_call(on_silent_limit)
  371. _finish_stop(on_status)
  372. def _finish_stop(on_status) -> None:
  373. """Clear the stopping flag, play the CLI-parity 660 Hz × 2 "stopped" cue, report idle."""
  374. global _continuous_stopping
  375. with _continuous_lock:
  376. _continuous_stopping = False
  377. _play_beep(frequency=660, count=2)
  378. _safe_call(on_status, "idle")
  379. def is_continuous_active() -> bool:
  380. """Whether a continuous voice loop is currently running."""
  381. with _continuous_lock:
  382. return _continuous_active
  383. def _continuous_on_silence() -> None:
  384. """AudioRecorder silence callback — runs in a daemon thread.
  385. Stops the current capture, transcribes, delivers text via ``on_transcript``, and — if the
  386. loop is still active — starts the next capture. Three consecutive silent cycles end the loop.
  387. """
  388. global _continuous_active, _continuous_no_speech_count
  389. _debug("_continuous_on_silence: fired")
  390. with _continuous_lock:
  391. if not _continuous_active:
  392. _debug("_continuous_on_silence: loop inactive — abort")
  393. return
  394. rec = _continuous_recorder
  395. on_transcript, on_status, on_silent_limit, on_stop_phrase = _continuous_callbacks
  396. if rec is None:
  397. _debug("_continuous_on_silence: no recorder — abort")
  398. return
  399. _safe_call(on_status, "transcribing")
  400. wav_path = rec.stop()
  401. # Peak RMS tells at a glance whether the mic was too quiet for SILENCE_RMS_THRESHOLD (200)
  402. # when stop() returns None despite the VAD firing.
  403. _debug(f"_continuous_on_silence: rec.stop -> {wav_path!r} (peak_rms={getattr(rec, '_peak_rms', -1)})")
  404. # CLI parity: double beep after the stream stops (safe from the CoreAudio conflict).
  405. _play_beep(frequency=660, count=2)
  406. transcript, stop_phrase, stop_text = _turn_transcript(
  407. wav_path, "continuous transcription failed: %s", "_continuous_on_silence", "ending loop", trace=True
  408. )
  409. # Held check runs outside the lock (the probe may call into the host surface).
  410. held = transcript is None and not stop_phrase and _voice_activity_held()
  411. with _continuous_lock:
  412. if not _continuous_active:
  413. _debug("_continuous_on_silence: stopped during transcribe — no restart")
  414. return
  415. limit_hit, no_speech = _tally_silence(bool(transcript) or stop_phrase, held, "_continuous_on_silence")
  416. if transcript:
  417. _safe_call(on_transcript, transcript, warn="on_transcript callback raised: %s")
  418. if stop_phrase or limit_hit:
  419. _debug(f"_continuous_on_silence: halting ({'stop phrase' if stop_phrase else f'{no_speech} silent cycles'})")
  420. with _continuous_lock:
  421. _continuous_active = False
  422. _continuous_no_speech_count = 0
  423. _signal_halt(stop_phrase, stop_text, on_stop_phrase, on_silent_limit)
  424. _safe_call(rec.cancel)
  425. _safe_call(on_status, "idle")
  426. return
  427. _rearm_after_turn(rec, on_status, no_speech)
  428. def _rearm_after_turn(rec: Any, on_status, no_speech: int) -> None:
  429. """Wait out in-flight TTS, then restart capture (auto_restart) or stop (client-driven loop).
  430. CLI parity: the mic waits for TTS and then leaves a small gap so the speaker tail isn't
  431. captured — otherwise the agent's spoken reply lands back in the mic and gets re-submitted.
  432. """
  433. if not _tts_playing.is_set():
  434. _debug("_continuous_on_silence: waiting for TTS to finish")
  435. _tts_playing.wait(timeout=60)
  436. time.sleep(0.3)
  437. with _continuous_lock:
  438. if not _continuous_active:
  439. _debug("_continuous_on_silence: stopped while waiting for TTS")
  440. return
  441. if not _continuous_auto_restart:
  442. _debug("_continuous_on_silence: auto_restart=False, stopping loop")
  443. _deactivate(on_status)
  444. return
  445. _debug(f"_continuous_on_silence: restarting loop (no_speech={no_speech})")
  446. _play_beep(frequency=880, count=1)
  447. try:
  448. rec.start(on_silence_stop=_continuous_on_silence)
  449. except Exception as e:
  450. logger.error("failed to restart continuous recording: %s", e)
  451. _debug(f"_continuous_on_silence: restart raised {type(e).__name__}: {e}")
  452. _deactivate(on_status)
  453. return
  454. _safe_call(on_status, "listening")
  455. # ── TTS API ──────────────────────────────────────────────────────────
  456. # Legacy markdown stripper used only when tools.tts_text_normalize is unavailable.
  457. _LEGACY_TTS_STRIP = [
  458. (re.compile(r'```[\s\S]*?```'), ' '), # fenced code blocks
  459. (re.compile(r'\[([^\]]+)\]\([^)]+\)'), r'\1'), # [text](url) → text
  460. (re.compile(r'https?://\S+'), ''), # bare URLs
  461. (re.compile(r'\*\*(.+?)\*\*'), r'\1'), # bold
  462. (re.compile(r'\*(.+?)\*'), r'\1'), # italic
  463. (re.compile(r'`(.+?)`'), r'\1'), # inline code
  464. (re.compile(r'^#+\s*', re.MULTILINE), ''), # headers
  465. (re.compile(r'^\s*[-*]\s+', re.MULTILINE), ''), # list bullets
  466. (re.compile(r'---+'), ''), # horizontal rules
  467. (re.compile(r'\n{3,}'), '\n\n'), # excess newlines
  468. ]
  469. def _speak_streaming(text: str, stop_event: Optional[threading.Event]) -> bool:
  470. """Speak via the CLI's ``stream_tts_to_speaker`` pipeline when the configured provider has a
  471. chunked streamer (audio starts on sentence one); False → caller uses the whole-file path.
  472. The full reply is fed as one delta + end-of-text sentinel and we block until the done event
  473. fires — same blocking contract as the sync path, just earlier first audio. ``stop_event`` is
  474. wired into the pipeline so external barge-in / stop paths can cut playback.
  475. """
  476. import queue
  477. # One dispatcher, zero parallel streaming implementations (#58930): when the configured provider has a
  478. # chunked streamer registered in tools.tts_streaming, route the whole reply through the same
  479. # stream_tts_to_speaker pipeline the CLI voice mode uses — audio starts on sentence one instead of after
  480. # full synthesis. Falls through to the legacy whole-file path when no streamer resolves.
  481. from tools.tts_streaming import resolve_streaming_provider
  482. from tools.tts_tool import _load_tts_config
  483. from tools.tts_tool_speaker import stream_tts_to_speaker
  484. if resolve_streaming_provider(_load_tts_config()) is None:
  485. return False
  486. text_queue: queue.Queue = queue.Queue()
  487. text_queue.put(text)
  488. text_queue.put(None) # end-of-text sentinel
  489. done_event = threading.Event()
  490. stream_tts_to_speaker(text_queue, stop_event or threading.Event(), done_event)
  491. return done_event.is_set()
  492. def _speak_whole_file(text: str) -> None:
  493. """Sync path: clean, synthesize to a temp MP3 via ``text_to_speech_tool``, play, unlink."""
  494. from tools.tts_tool import text_to_speech_tool
  495. # Shared cleaner (markdown, emoji, ⋗ blocks, verifier footer, units); the TTS tool owns
  496. # provider request limits and long-form chunking.
  497. try:
  498. from tools.tts_text_normalize import prepare_spoken_text
  499. tts_text = prepare_spoken_text(text, max_chars=None)
  500. except Exception:
  501. tts_text = text
  502. for pattern, repl in _LEGACY_TTS_STRIP:
  503. tts_text = pattern.sub(repl, tts_text)
  504. tts_text = tts_text.strip()
  505. if not tts_text:
  506. return
  507. # Pre-chosen MP3 path so we can play MP3 even when text_to_speech_tool auto-converts to OGG
  508. # for messaging platforms (afplay's OGG is flaky).
  509. os.makedirs(os.path.join(tempfile.gettempdir(), "hermes_voice"), exist_ok=True)
  510. mp3_path = os.path.join(tempfile.gettempdir(), "hermes_voice", f"tts_{time.strftime('%Y%m%d_%H%M%S')}.mp3")
  511. _debug(f"speak_text: synthesizing {len(tts_text)} chars -> {mp3_path}")
  512. raw_result = text_to_speech_tool(text=tts_text, output_path=mp3_path)
  513. try:
  514. tts_result = json.loads(raw_result) if isinstance(raw_result, str) else {}
  515. except Exception:
  516. tts_result = {}
  517. # The tool result is authoritative — long-form output may be several files.
  518. play_paths = tts_result.get("file_paths") or [tts_result.get("file_path") or mp3_path]
  519. played_any = False
  520. for play_path in play_paths if tts_result.get("success") else []:
  521. if os.path.isfile(play_path) and os.path.getsize(play_path) > 0:
  522. _debug(f"speak_text: playing {play_path} ({os.path.getsize(play_path)} bytes)")
  523. play_audio_file(play_path)
  524. played_any = True
  525. for path in set(play_paths + [mp3_path, mp3_path.rsplit(".", 1)[0] + ".ogg"]):
  526. if os.path.isfile(path):
  527. with contextlib.suppress(OSError):
  528. os.unlink(path)
  529. if not played_any:
  530. _debug(f"speak_text: TTS tool produced no audio at {mp3_path}")
  531. def speak_text(text: str, stop_event: Optional[threading.Event] = None) -> None:
  532. """Synthesize ``text`` with the configured TTS provider and play it.
  533. While playback is in flight ``_tts_playing`` is cleared so the continuous loop waits before
  534. re-arming the mic (otherwise the agent's reply feedback-loops through the microphone). Live
  535. capture is cancelled first — otherwise the user's turn tail + our first syllables both land
  536. in the next recording window — and resumed afterwards so the user can answer without
  537. pressing the record key.
  538. """
  539. if not text or not text.strip():
  540. return
  541. paused_recording = False
  542. with _continuous_lock:
  543. if _continuous_active and getattr(_continuous_recorder, "is_recording", False):
  544. try:
  545. _continuous_recorder.cancel()
  546. paused_recording = True
  547. except Exception as e:
  548. logger.warning("failed to pause recorder for TTS: %s", e)
  549. _tts_playing.clear()
  550. _debug(f"speak_text: TTS begin (paused_recording={paused_recording})")
  551. try:
  552. # One dispatcher: streaming when a chunked streamer resolves, else the whole-file path.
  553. try:
  554. if _speak_streaming(text, stop_event):
  555. return
  556. except Exception as e:
  557. _debug(f"speak_text: streaming dispatch unavailable ({e}); using sync path")
  558. _speak_whole_file(text)
  559. except Exception as e:
  560. logger.warning("Voice TTS playback failed: %s", e)
  561. _debug(f"speak_text raised {type(e).__name__}: {e}")
  562. finally:
  563. _tts_playing.set()
  564. _debug("speak_text: TTS done")
  565. # The delay lets afplay release the audio device before sounddevice re-opens.
  566. if paused_recording:
  567. time.sleep(0.3)
  568. with _continuous_lock:
  569. if _continuous_active and _continuous_recorder is not None:
  570. try:
  571. _continuous_recorder.start(on_silence_stop=_continuous_on_silence)
  572. _debug("speak_text: recording resumed after TTS")
  573. except Exception as e:
  574. logger.warning("failed to resume recorder after TTS: %s", e)