cli_voice_mixin.py 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843
  1. """Voice mode (recording, STT, TTS, full-duplex barge-in) and wake-word listener handlers for the interactive CLI
  2. Mixin split out of ``cli.py``; bound onto ``HermesCLI`` via the MRO. cli.py-internal
  3. symbols are imported LAZILY inside each method (``from cli import ...``) — the mixin
  4. never imports ``cli`` at module load time (import cycle).
  5. """
  6. from __future__ import annotations
  7. import json
  8. import os
  9. import re
  10. import tempfile
  11. import threading
  12. import time
  13. from agent.i18n import t
  14. from hermes_constants import is_termux as _is_termux_environment
  15. from typing import Optional
  16. def _config_section(name: str) -> dict:
  17. """``load_config()[name]`` coerced to a dict.
  18. Shape-safe: a hand-edited ``voice: true`` / ``voice: cmd+b`` leaves the section as a
  19. non-dict; return {} so callers fall back to documented defaults instead of crashing on
  20. ``.get()``. Config load failures also yield {}.
  21. """
  22. try:
  23. from hermes_cli.config import load_config
  24. section = load_config().get(name)
  25. return section if isinstance(section, dict) else {}
  26. except Exception:
  27. return {}
  28. def _numeric_or(value, default):
  29. """``value`` if it is a real int/float, else ``default``.
  30. ``bool`` is excluded — it subclasses int, so a hand-edited ``silence_threshold: true``
  31. would otherwise be forwarded as ``1`` instead of falling back.
  32. """
  33. return value if isinstance(value, (int, float)) and not isinstance(value, bool) else default
  34. def _unlink_quietly(path) -> None:
  35. try:
  36. if path and os.path.isfile(path):
  37. os.unlink(path)
  38. except OSError:
  39. pass
  40. class CLIVoiceMixin:
  41. """Voice mode (recording, STT, TTS, full-duplex barge-in) and wake-word listener handlers for the interactive CLI"""
  42. def _voice_invalidate(self) -> None:
  43. """Repaint the TUI (audio level indicator / status) when the app is live."""
  44. if hasattr(self, '_app') and self._app:
  45. self._app.invalidate()
  46. def _voice_start_recording(self):
  47. """Start capturing audio from the microphone."""
  48. from cli import _ACCENT, _DIM, _RST, _cprint
  49. if getattr(self, '_should_exit', False):
  50. return
  51. from tools.voice_mode import create_audio_recorder, check_voice_requirements
  52. reqs = check_voice_requirements()
  53. if not reqs["audio_available"]:
  54. if _is_termux_environment():
  55. # These RuntimeErrors are human copy: callers print ``{e}`` straight to the terminal.
  56. if "Termux:API Android app is not installed" in reqs.get("details", ""):
  57. raise RuntimeError(t("cli.voice.err_termux_app_missing"))
  58. raise RuntimeError(t("cli.voice.err_mic_access"))
  59. # check_voice_requirements already asked PM to enable audio-io; its detail line
  60. # says why that did not happen (lazy installs off, needs restart, platform gate).
  61. raise RuntimeError(t("cli.voice.err_audio_capture") + "\n" + reqs.get("details", ""))
  62. if not reqs.get("stt_available", reqs.get("stt_key_set")):
  63. raise RuntimeError(t("cli.voice.err_stt_provider"))
  64. # Prevent double-start from concurrent threads (atomic check-and-set)
  65. with self._voice_lock:
  66. if self._voice_recording:
  67. return
  68. self._voice_recording = True
  69. voice_cfg = _config_section("voice")
  70. # Recorder creation can fail (no input device, PortAudio init). Reset the flag on
  71. # failure or every future voice start is silently skipped by the guard above.
  72. if self._voice_recorder is None:
  73. try:
  74. self._voice_recorder = create_audio_recorder()
  75. except Exception:
  76. with self._voice_lock:
  77. self._voice_recording = False
  78. raise
  79. # Config-driven silence params, numeric-guarded against YAML scalar corruption.
  80. rec = self._voice_recorder
  81. self._voice_live_text = ""
  82. rec.on_live_partial = self._voice_on_live_partial
  83. rec._silence_threshold = _numeric_or(voice_cfg.get("silence_threshold"), 200)
  84. rec._silence_duration = _numeric_or(voice_cfg.get("silence_duration"), 3.0)
  85. # voice.max_recording_seconds — hard cap on one recording; explicit <= 0 disables it.
  86. _max_rec = _numeric_or(voice_cfg.get("max_recording_seconds"), None)
  87. rec._max_recording_seconds = (_max_rec if _max_rec > 0 else 0.0) if _max_rec is not None else 120.0
  88. def _on_silence():
  89. """Called by AudioRecorder when silence is detected after speech."""
  90. with self._voice_lock:
  91. if not self._voice_recording:
  92. return
  93. _cprint(f"\n{_DIM}{t('cli.voice.silence_autostop')}{_RST}")
  94. self._voice_invalidate()
  95. self._voice_stop_and_transcribe()
  96. # Audio cue: single beep BEFORE starting stream (avoid CoreAudio conflict)
  97. self._voice_beep(frequency=880, count=1)
  98. try:
  99. self._voice_recorder.start(on_silence_stop=_on_silence)
  100. except Exception:
  101. with self._voice_lock:
  102. self._voice_recording = False
  103. raise
  104. _label = self._voice_record_key_label()
  105. if getattr(self._voice_recorder, "supports_silence_autostop", True):
  106. _recording_hint = t("cli.voice.hint_continuous", shortcut=_label)
  107. elif _is_termux_environment():
  108. _recording_hint = t("cli.voice.hint_termux", shortcut=_label)
  109. else:
  110. _recording_hint = t("cli.voice.hint_stop", shortcut=_label)
  111. _cprint(f"\n{_ACCENT}{t('cli.voice.recording')}{_RST} {_DIM}({_recording_hint}){_RST}")
  112. # Periodically refresh prompt to update audio level indicator
  113. def _refresh_level():
  114. while True:
  115. with self._voice_lock:
  116. still_recording = self._voice_recording
  117. if not still_recording:
  118. break
  119. self._voice_invalidate()
  120. time.sleep(0.15)
  121. threading.Thread(target=_refresh_level, daemon=True).start()
  122. def _voice_beep(self, *, frequency: int, count: int) -> None:
  123. """Play a record start/stop beep when enabled; never raises."""
  124. if self._voice_beeps_enabled():
  125. try:
  126. from tools.voice_mode import play_beep
  127. play_beep(frequency=frequency, count=count)
  128. except Exception:
  129. pass
  130. def _voice_stt_model(self) -> Optional[str]:
  131. """STT model override from config, or None for the provider default.
  132. For the local provider, prefer stt.local.model (default ``base``) so the CLI passes
  133. a real model name into the local STT backend.
  134. """
  135. stt_config = _config_section("stt")
  136. if str(stt_config.get("provider") or "").strip().lower() == "local":
  137. local_config = stt_config.get("local") or {}
  138. if not isinstance(local_config, dict):
  139. local_config = {}
  140. return local_config.get("model") or "base"
  141. return stt_config.get("model")
  142. def _voice_stt_provider(self) -> str:
  143. """Configured STT provider name (lowercased), or empty string."""
  144. return str(_config_section("stt").get("provider") or "").strip().lower()
  145. def _voice_restart_recording_async(self) -> None:
  146. """Restart continuous-mode recording off-thread (start() can block)."""
  147. from cli import _DIM, _RST, _cprint
  148. def _restart_recording():
  149. try:
  150. self._voice_start_recording()
  151. self._voice_invalidate()
  152. except Exception as e:
  153. _cprint(f"{_DIM}{t('cli.voice.autorestart_failed', error=e)}{_RST}")
  154. threading.Thread(target=_restart_recording, daemon=True).start()
  155. def _voice_on_live_partial(self, text: str) -> None:
  156. """Live STT partial (``stt.streaming``): shown in the input placeholder while recording."""
  157. self._voice_live_text = text
  158. self._voice_invalidate()
  159. def _voice_stop_and_transcribe(self):
  160. """Stop recording, transcribe via STT, and queue the transcript as input."""
  161. from cli import _DIM, _RST, _VoiceInputMessage, _cprint
  162. # Atomic guard; _voice_processing is set immediately so concurrent Ctrl+B presses
  163. # don't race into the START path while recorder.stop() holds its lock.
  164. with self._voice_lock:
  165. if not self._voice_recording:
  166. return
  167. self._voice_recording = False
  168. self._voice_processing = True
  169. submitted = False
  170. transcription_failed = False
  171. wav_path = None
  172. try:
  173. if self._voice_recorder is None:
  174. return
  175. wav_path = self._voice_recorder.stop()
  176. # Audio cue: double beep after stream stopped (no CoreAudio conflict)
  177. self._voice_beep(frequency=660, count=2)
  178. if wav_path is None:
  179. _cprint(f"{_DIM}{t('cli.voice.no_speech')}{_RST}")
  180. return
  181. self._voice_invalidate()
  182. stt_model = self._voice_stt_model()
  183. if self._voice_stt_provider() == "local":
  184. _cprint(f"{_DIM}{t('cli.voice.preparing_stt', model=stt_model)}{_RST}")
  185. else:
  186. _cprint(f"{_DIM}{t('cli.voice.transcribing')}{_RST}")
  187. from tools.voice_mode_transcript import is_voice_stop_phrase
  188. from tools.voice_mode import transcribe_recording
  189. result = transcribe_recording(wav_path, model=stt_model)
  190. if result.get("success") and result.get("transcript", "").strip():
  191. transcript = result["transcript"].strip()
  192. if is_voice_stop_phrase(transcript):
  193. # Bare "stop" (or configured phrase) ends the voice chat, not a turn.
  194. _cprint(f"{_DIM}{t('cli.voice.stop_phrase')}{_RST}")
  195. self._disable_voice_mode()
  196. return
  197. self._attached_images.clear()
  198. self._voice_invalidate()
  199. self._pending_input.put(_VoiceInputMessage(transcript))
  200. submitted = True
  201. elif result.get("success"):
  202. _cprint(f"{_DIM}{t('cli.voice.no_speech')}{_RST}")
  203. else:
  204. _cprint(f"\n{_DIM}{t('cli.voice.transcription_failed', error=result.get('error') or t('cli.shared.unknown_error'))}{_RST}")
  205. transcription_failed = True
  206. except Exception as e:
  207. _cprint(f"\n{_DIM}{t('cli.voice.processing_error', error=e)}{_RST}")
  208. transcription_failed = wav_path is not None
  209. finally:
  210. with self._voice_lock:
  211. self._voice_processing = False
  212. self._voice_live_text = ""
  213. self._voice_invalidate()
  214. # On failure keep the source recording so long dictation is not lost.
  215. try:
  216. if wav_path and os.path.isfile(wav_path):
  217. if transcription_failed:
  218. _cprint(f"{_DIM}{t('cli.voice.recording_preserved', path=wav_path)}{_RST}")
  219. else:
  220. os.unlink(wav_path)
  221. except Exception:
  222. pass
  223. # Three consecutive no-speech cycles end continuous mode (no infinite restart
  224. # loop). While the agent is mid-turn or TTS is speaking the user is CORRECTLY
  225. # silent — those cycles must not count, or a multi-minute tool run ends the voice
  226. # chat under the user (stop phrase and barge-in still work during the hold).
  227. stop_continuous_restart = False
  228. _tts_done = getattr(self, "_voice_tts_done", None)
  229. _activity_hold = bool(
  230. getattr(self, "_agent_running", False)
  231. or (_tts_done is not None and not _tts_done.is_set()))
  232. if submitted:
  233. self._no_speech_count = 0
  234. elif not _activity_hold:
  235. self._no_speech_count = getattr(self, '_no_speech_count', 0) + 1
  236. if self._no_speech_count >= 3:
  237. self._voice_continuous = False
  238. self._no_speech_count = 0
  239. _cprint(f"{_DIM}{t('cli.voice.no_speech_stop')}{_RST}")
  240. stop_continuous_restart = True
  241. # No transcript but continuous mode active: restart so the user can keep talking
  242. # (when a transcript IS submitted, process_loop restarts after chat()).
  243. if (
  244. self._voice_continuous
  245. and not submitted
  246. and not self._voice_recording
  247. and not stop_continuous_restart):
  248. self._voice_restart_recording_async()
  249. def _voice_speak_response_async(self, text: str) -> None:
  250. """Schedule TTS and mark it pending before continuous recording can restart."""
  251. if not self._voice_tts or not text:
  252. return
  253. self._voice_tts_done.clear()
  254. threading.Thread(target=self._voice_speak_response, args=(text,), daemon=True).start()
  255. # Barge-in safety net for speak calls outside a chat turn (the agent-turn listener
  256. # armed in chat() normally covers playback); idempotent via _voice_fd_active.
  257. if self._voice_continuous:
  258. threading.Thread(target=self._voice_full_duplex_listener, daemon=True).start()
  259. def _voice_speak_response(self, text: str):
  260. """Speak the agent's response aloud using TTS (runs in background thread)."""
  261. from cli import _DIM, _RST, _cprint, logger
  262. if not self._voice_tts:
  263. return
  264. self._voice_tts_done.clear()
  265. try:
  266. from tools.tts_tool import text_to_speech_tool
  267. from tools.voice_mode import play_audio_file
  268. # Shared cleaner strips markdown/emoji/⋗ blocks/verifier footer; the TTS tool owns
  269. # provider request limits and long-form chunking.
  270. try:
  271. from tools.tts_text_normalize import prepare_spoken_text
  272. tts_text = prepare_spoken_text(text, max_chars=None)
  273. except Exception:
  274. # Legacy fallback pipeline — keep voice replies best-effort.
  275. tts_text = re.sub(r'```[\s\S]*?```', ' ', text) # fenced code blocks
  276. tts_text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', tts_text) # [text](url) -> text
  277. tts_text = re.sub(r'https?://\S+', '', tts_text) # URLs
  278. tts_text = re.sub(r'\*\*(.+?)\*\*', r'\1', tts_text) # bold
  279. tts_text = re.sub(r'\*(.+?)\*', r'\1', tts_text) # italic
  280. tts_text = re.sub(r'`(.+?)`', r'\1', tts_text) # inline code
  281. tts_text = re.sub(r'^#+\s*', '', tts_text, flags=re.MULTILINE) # headers
  282. tts_text = re.sub(r'^\s*[-*]\s+', '', tts_text, flags=re.MULTILINE) # list items
  283. tts_text = re.sub(r'---+', '', tts_text) # horizontal rules
  284. tts_text = re.sub(r'\n{3,}', '\n\n', tts_text) # excessive newlines
  285. tts_text = tts_text.strip()
  286. if not tts_text:
  287. return
  288. self._voice_last_tts_text = tts_text
  289. # MP3 for CLI playback (afplay doesn't handle OGG well); the TTS tool may
  290. # auto-convert MP3->OGG but the original MP3 remains.
  291. out_dir = os.path.join(tempfile.gettempdir(), "hermes_voice")
  292. os.makedirs(out_dir, exist_ok=True)
  293. mp3_path = os.path.join(out_dir, f"tts_{time.strftime('%Y%m%d_%H%M%S')}.mp3")
  294. raw_result = text_to_speech_tool(text=tts_text, output_path=mp3_path)
  295. try:
  296. tts_result = json.loads(raw_result) if isinstance(raw_result, str) else {}
  297. except Exception:
  298. tts_result = {}
  299. # The tool result is authoritative — chunked long-form output returns several files.
  300. play_paths = tts_result.get("file_paths") or [tts_result.get("file_path") or mp3_path]
  301. for play_path in play_paths if tts_result.get("success") else []:
  302. if os.path.isfile(play_path) and os.path.getsize(play_path) > 0:
  303. play_audio_file(play_path)
  304. # Clean up all generated files (play_paths + mp3_path + ogg variant)
  305. for path in set(play_paths + [mp3_path, mp3_path.rsplit(".", 1)[0] + ".ogg"]):
  306. _unlink_quietly(path)
  307. except Exception as e:
  308. logger.warning("Voice TTS playback failed: %s", e)
  309. _cprint(f"{_DIM}{t('cli.voice.tts_failed', error=e)}{_RST}")
  310. finally:
  311. self._voice_tts_done.set()
  312. def _voice_full_duplex_listener(self) -> None:
  313. """Full-duplex agent-turn listener: mic live for the WHOLE turn.
  314. Armed at utterance-submit (chat() start in continuous voice mode), disarmed when agent
  315. finished + TTS played, so the user can interject during generation too. Generation
  316. phase: speech interrupts the turn via ``self.agent.interrupt()`` (same seam as
  317. Ctrl+C); playback phase: speech cuts TTS. Either way the captured utterance is
  318. submitted as the next message; the stop phrase ends the voice chat in BOTH phases.
  319. """
  320. from cli import _DIM, _RST, _cprint, logger
  321. fd_active = getattr(self, "_voice_fd_active", None)
  322. if fd_active is None:
  323. fd_active = threading.Event()
  324. self._voice_fd_active = fd_active
  325. if fd_active.is_set():
  326. return # one listener owns the mic for this turn
  327. fd_active.set()
  328. try:
  329. from hermes_cli.config import load_config
  330. voice_cfg = load_config().get("voice") or {}
  331. if not (isinstance(voice_cfg, dict) and voice_cfg.get("barge_in", True)):
  332. return
  333. from tools.voice_mode import full_duplex_listen, is_audio_output_active, stop_playback
  334. try:
  335. _mult = float(voice_cfg.get("barge_in_threshold_multiplier", 0) or 0)
  336. except (TypeError, ValueError):
  337. _mult = 0.0
  338. try:
  339. _grace_ms = int(float(voice_cfg.get("barge_in_grace_seconds", 0.5)) * 1000)
  340. except (TypeError, ValueError):
  341. _grace_ms = 500
  342. tts_done = getattr(self, "_voice_tts_done", None)
  343. def _should_stop() -> bool:
  344. if not (getattr(self, "_voice_mode", False) and getattr(self, "_voice_continuous", False)):
  345. return True
  346. if getattr(self, "_agent_running", False):
  347. return False
  348. # Agent finished — keep listening until TTS fully played.
  349. if tts_done is not None and not tts_done.is_set():
  350. return False
  351. return not is_audio_output_active()
  352. def _on_trigger(phase: str) -> None:
  353. # Latch BEFORE cutting anything: suppresses process_loop's auto-restart until
  354. # the capture is submitted.
  355. self._voice_barge_capture.set()
  356. self._voice_barge_phase = phase
  357. _pipe_stop = getattr(self, "_voice_tts_stop", None)
  358. if phase == "playback":
  359. logger.debug("TTS CUT: full-duplex listener tripped during playback")
  360. from tools.tts_streaming import mark_speech_interrupted
  361. mark_speech_interrupted()
  362. if _pipe_stop is not None:
  363. _pipe_stop.set()
  364. stop_playback()
  365. else:
  366. # Generation phase: no audio to cut — interrupt the in-flight agent turn.
  367. logger.debug(
  368. "full-duplex listener tripped during generation — "
  369. "interrupting agent turn")
  370. if _pipe_stop is not None:
  371. _pipe_stop.set() # never let the stale reply speak
  372. try:
  373. if self.agent is not None and getattr(self, "_agent_running", False):
  374. _cprint(f"\n{_DIM}{t('cli.voice.interjection')}{_RST}")
  375. self.agent.interrupt()
  376. except Exception as e:
  377. logger.debug("voice interjection interrupt failed: %s", e)
  378. wav_path = full_duplex_listen(
  379. _should_stop, is_playing=is_audio_output_active, on_trigger=_on_trigger,
  380. multiplier=_mult or None, grace_ms=max(0, _grace_ms))
  381. if wav_path and self._voice_barge_capture.is_set():
  382. self._voice_submit_barge_utterance(wav_path)
  383. else:
  384. self._voice_barge_capture.clear()
  385. except Exception as e:
  386. self._voice_barge_capture.clear()
  387. logger.debug("Voice full-duplex listener failed: %s", e)
  388. finally:
  389. fd_active.clear()
  390. def _voice_submit_barge_utterance(self, wav_path: str) -> None:
  391. """Transcribe a barge-captured interruption and queue it as the next turn."""
  392. from cli import _DIM, _RST, _VoiceInputMessage, _cprint, logger
  393. submitted = False
  394. try:
  395. from tools.voice_mode import transcribe_recording
  396. result = transcribe_recording(wav_path, model=self._voice_stt_model())
  397. transcript = (result.get("transcript") or "").strip() if result.get("success") else ""
  398. if transcript:
  399. from tools.voice_mode_transcript import is_voice_stop_phrase
  400. if is_voice_stop_phrase(transcript):
  401. _cprint(f"\n{_DIM}{t('cli.voice.stop_phrase')}{_RST}")
  402. self._disable_voice_mode()
  403. return
  404. # Fail-closed echo guard: playback-phase capture has no echo cancellation, so
  405. # a close match for what Hermes just spoke is speaker bleed, not a user turn.
  406. if getattr(self, "_voice_barge_phase", None) == "playback":
  407. from tools.voice_mode_transcript import is_tts_echo
  408. if is_tts_echo(transcript, getattr(self, "_voice_last_tts_text", "")):
  409. logger.debug(
  410. "Dropping playback-phase barge transcript as TTS echo: %r", transcript)
  411. _cprint(f"\n{_DIM}{t('cli.voice.echo_ignored')}{_RST}")
  412. return
  413. self._pending_input.put(_VoiceInputMessage(transcript))
  414. submitted = True
  415. elif not result.get("success"):
  416. _cprint(f"\n{_DIM}{t('cli.voice.transcription_failed', error=result.get('error') or t('cli.shared.unknown_error'))}{_RST}")
  417. except Exception as e:
  418. _cprint(f"\n{_DIM}{t('cli.voice.processing_error', error=e)}{_RST}")
  419. finally:
  420. _unlink_quietly(wav_path)
  421. self._voice_barge_capture.clear()
  422. self._voice_barge_phase = None
  423. # No usable transcript: hand the mic back to the normal loop.
  424. if not submitted and self._voice_mode and self._voice_continuous and not self._voice_recording:
  425. self._voice_restart_recording_async()
  426. def _voice_beeps_enabled(self) -> bool:
  427. """Return whether CLI voice mode should play record start/stop beeps."""
  428. try:
  429. from utils import is_truthy_value # handles quoted YAML "false" (bool() would not)
  430. return is_truthy_value(_config_section("voice").get("beep_enabled", True), default=True)
  431. except Exception:
  432. return True
  433. def _enable_voice_mode(self):
  434. """Enable voice mode after checking requirements."""
  435. from cli import _ACCENT, _BOLD, _DIM, _RST, _cprint
  436. if self._voice_mode:
  437. _cprint(f"{_DIM}{t('cli.voice.already_enabled')}{_RST}")
  438. return
  439. from tools.voice_mode import check_voice_requirements, detect_audio_environment
  440. env_check = detect_audio_environment()
  441. if not env_check["available"]:
  442. _cprint(f"\n{_ACCENT}{t('cli.voice.unavailable_header')}{_RST}")
  443. for warning in env_check["warnings"]:
  444. _cprint(f" {_DIM}{warning}{_RST}")
  445. return
  446. reqs = check_voice_requirements()
  447. if not reqs["available"]:
  448. _cprint(f"\n{_ACCENT}{t('cli.voice.requirements_header')}{_RST}")
  449. for line in reqs["details"].split("\n"):
  450. _cprint(f" {_DIM}{line}{_RST}")
  451. if reqs["missing_packages"]:
  452. if _is_termux_environment():
  453. _cprint(f"\n {_BOLD}{t('cli.voice.termux_install_cmd')}{_RST}")
  454. _cprint(f" {_DIM}{t('cli.voice.termux_install_hint')}{_RST}")
  455. return
  456. with self._voice_lock:
  457. self._voice_mode = True
  458. if _config_section("voice").get("auto_tts", False):
  459. with self._voice_lock:
  460. self._voice_tts = True
  461. # The voice-mode instruction is injected as a user message prefix (not a system
  462. # prompt change) to avoid invalidating the prompt cache — see _voice_message_prefix.
  463. tts_status = t("cli.voice.tts_enabled_suffix") if self._voice_tts else ""
  464. if self._voice_tts:
  465. self._tts_lease_async(True) # warm the engine so the first reply isn't dead air
  466. # Startup-pinned label so the advertised shortcut always matches the live
  467. # prompt_toolkit binding (live config would drift after a mid-session edit).
  468. # See #19835.
  469. _cprint(f"\n{_ACCENT}{t('cli.voice.enabled', tts_status=tts_status)}{_RST}")
  470. _cprint(f" {_DIM}{t('cli.voice.record_key_hint', shortcut=self._voice_record_key_label())}{_RST}")
  471. # Spoken-stop hint from voice.stop_phrases (first entry); "" when disabled.
  472. try:
  473. from tools.voice_mode_transcript import voice_stop_hint
  474. _stop_hint = voice_stop_hint()
  475. except Exception:
  476. _stop_hint = ""
  477. if _stop_hint:
  478. _cprint(f" {_DIM}{_stop_hint}{_RST}")
  479. _cprint(f" {_DIM}{t('cli.voice.tts_toggle_hint')}{_RST}")
  480. _cprint(f" {_DIM}{t('cli.voice.disable_hint')}{_RST}")
  481. def _typed_voice_stop(self, user_input) -> bool:
  482. """Typed bare stop phrase during an active voice chat ends the chat (mirrors the spoken
  483. one; outside voice mode "stop" passes through to the agent). Exact-match via
  484. ``is_voice_stop_phrase``, so longer messages containing "stop" are never swallowed.
  485. Saying "stop" ends the voice chat (PR #73106); TYPING the same bare stop phrase while voice mode is
  486. on must behave identically instead of sending "stop" to the agent as a turn.
  487. """
  488. from cli import _DIM, _RST, _cprint
  489. if not isinstance(user_input, str):
  490. return False
  491. with self._voice_lock:
  492. voice_on = self._voice_mode or self._voice_continuous
  493. if not voice_on:
  494. return False
  495. try:
  496. from tools.voice_mode_transcript import is_voice_stop_phrase
  497. if not is_voice_stop_phrase(user_input):
  498. return False
  499. except Exception:
  500. return False
  501. _cprint(f"\n{_DIM}{t('cli.voice.stop_phrase_typed')}{_RST}")
  502. self._disable_voice_mode()
  503. return True
  504. def _disable_voice_mode(self):
  505. """Disable voice mode, cancel any active recording, and stop TTS."""
  506. from cli import _DIM, _RST, _cprint, logger
  507. with self._voice_lock:
  508. if self._voice_recording and self._voice_recorder:
  509. self._voice_recorder.cancel()
  510. self._voice_recording = False
  511. recorder = self._voice_recorder
  512. self._voice_mode = False
  513. self._voice_tts = False
  514. self._voice_continuous = False
  515. # Release the TTS lease so a resident local model (piper/kittentts) can be freed.
  516. self._tts_lease_async(False)
  517. # Shut down the persistent audio stream in background
  518. if recorder is not None:
  519. def _bg_shutdown(rec=recorder):
  520. try:
  521. rec.shutdown()
  522. except Exception:
  523. pass
  524. threading.Thread(target=_bg_shutdown, daemon=True).start()
  525. self._voice_recorder = None
  526. # Stop any active TTS playback (file player + streaming pipeline)
  527. try:
  528. if self._voice_tts_stop is not None:
  529. logger.info("TTS CUT: _disable_voice_mode setting stop event")
  530. self._voice_tts_stop.set()
  531. from tools.voice_mode import stop_playback
  532. stop_playback()
  533. except Exception:
  534. pass
  535. self._voice_tts_done.set()
  536. _cprint(f"\n{_DIM}{t('cli.voice.disabled')}{_RST}")
  537. def _maybe_start_wake_word(self):
  538. """Start the wake-word listener at CLI startup if this surface is eligible."""
  539. try:
  540. from tools.wake_word import wake_surface_enabled
  541. if not wake_surface_enabled("cli"):
  542. return
  543. except Exception:
  544. return
  545. self._start_wake_word_listener(announce=True)
  546. def _start_wake_word_listener(self, announce: bool = False) -> bool:
  547. """Build + start the hotword detector. Returns True on success."""
  548. from cli import _ACCENT, _DIM, _RST, _cprint
  549. say = _cprint if announce else (lambda *_a: None)
  550. try:
  551. from tools.wake_word import (
  552. check_wake_word_requirements, load_wake_word_config, owns_listener, start_listening)
  553. except Exception as e:
  554. say(f"{_DIM}{t('cli.voice.wake_unavailable', error=e)}{_RST}")
  555. return False
  556. if getattr(self, "_wake_word_active", False) and owns_listener(self):
  557. say(f"{_DIM}{t('cli.voice.wake_already')}{_RST}")
  558. return True
  559. self._wake_word_active = False
  560. cfg = load_wake_word_config()
  561. reqs = check_wake_word_requirements(cfg)
  562. if not reqs["available"]:
  563. say(f"\n{_ACCENT}{t('cli.voice.wake_requirements_header')}{_RST}")
  564. if reqs.get("hint"):
  565. say(f" {_DIM}{reqs['hint']}{_RST}")
  566. return False
  567. if not reqs.get("deps_available", True):
  568. # Fresh install: the engine constructor lazy-installs its deps (onnxruntime is
  569. # a large wheel) — tell the user why this is slow.
  570. say(f"{_DIM}{t('cli.voice.wake_installing')}{_RST}")
  571. self._wake_start_new_session = bool(cfg.get("start_new_session", True))
  572. try:
  573. start_listening(self._on_wake_word, owner=self, config=cfg)
  574. except Exception as e:
  575. say(f"\n{_DIM}{t('cli.voice.wake_start_failed', error=e)}{_RST}")
  576. return False
  577. self._wake_word_active = True
  578. self._wake_suspended = False
  579. import cli as _cli
  580. _cli._cli_wake_owner = self
  581. self._start_wake_watchdog()
  582. say(f"\n{_ACCENT}{t('cli.voice.wake_listening')}{_RST} "
  583. f"{_DIM}{t('cli.voice.wake_listening_hint', phrase=reqs['phrase'])}{_RST}")
  584. return True
  585. def _stop_wake_word_listener(self, announce: bool = False):
  586. """Stop and tear down the hotword detector."""
  587. from cli import _DIM, _RST, _cprint
  588. import cli as _cli
  589. was_active = getattr(self, "_wake_word_active", False)
  590. self._wake_word_active = False
  591. self._wake_suspended = False
  592. try:
  593. from tools.wake_word import stop_listening
  594. stop_listening(owner=self)
  595. except Exception:
  596. pass
  597. if _cli._cli_wake_owner is self:
  598. _cli._cli_wake_owner = None
  599. if announce:
  600. _cprint(f"{_DIM}{t('cli.voice.wake_stopped' if was_active else 'cli.voice.wake_not_running')}{_RST}")
  601. def _on_wake_word(self):
  602. """Fired after the detector hears the wake phrase."""
  603. from cli import _ACCENT, _DIM, _RST, _cprint, logger
  604. if getattr(self, "_should_exit", False):
  605. return
  606. # Ignore wake while a turn is in flight or the mic is already in use.
  607. if self._agent_running or self._voice_recording or getattr(self, "_voice_processing", False):
  608. return
  609. # Release the mic so STT can capture the command utterance.
  610. try:
  611. from tools.wake_word import pause_listening
  612. if not pause_listening(owner=self):
  613. self._wake_word_active = False
  614. return
  615. except Exception as e:
  616. logger.debug("wake word pause failed: %s", e)
  617. return
  618. self._wake_suspended = True
  619. # The CLI is single-profile: a phrase enrolled by ANOTHER profile can't be routed
  620. # here — print the switch command and re-arm rather than answer as the wrong profile.
  621. try:
  622. from tools.wake_word import get_last_match
  623. _match = get_last_match()
  624. except Exception:
  625. _match = None
  626. if _match and _match[1]:
  627. from tools.wake_word import _active_profile_name
  628. if _match[1] != _active_profile_name():
  629. _cprint(f"\n{_DIM}{t('cli.voice.wake_profile_hint', profile=_match[1])}{_RST}")
  630. self._wake_suspended = True # watchdog resumes the listener
  631. return
  632. _cprint(f"\n{_ACCENT}{t('cli.voice.wake_detected')}{_RST}")
  633. if getattr(self, "_app", None):
  634. try:
  635. self._app.invalidate()
  636. except Exception:
  637. pass
  638. if getattr(self, "_wake_start_new_session", True):
  639. try:
  640. self.new_session(silent=True)
  641. except Exception as e:
  642. logger.debug("wake word new_session failed: %s", e)
  643. # Single-utterance capture; VAD auto-stop transcribes and queues for process_loop.
  644. with self._voice_lock:
  645. self._voice_mode = True
  646. self._voice_continuous = False
  647. try:
  648. self._voice_start_recording()
  649. except Exception as e:
  650. _cprint(f"{_DIM}{t('cli.voice.wake_capture_failed', error=e)}{_RST}")
  651. def _start_wake_watchdog(self):
  652. """Resume the paused detector when the CLI returns to a stable idle."""
  653. from cli import logger
  654. if getattr(self, "_wake_watchdog_started", False):
  655. return
  656. self._wake_watchdog_started = True
  657. def _loop():
  658. idle_polls = 0
  659. try:
  660. while getattr(self, "_wake_word_active", False) and not getattr(self, "_should_exit", False):
  661. time.sleep(0.25)
  662. if not getattr(self, "_wake_suspended", False):
  663. idle_polls = 0
  664. continue
  665. busy = (
  666. self._agent_running
  667. or self._voice_recording
  668. or getattr(self, "_voice_processing", False)
  669. or not self._pending_input.empty())
  670. if busy:
  671. idle_polls = 0
  672. continue
  673. # Require a few consecutive idle polls (~0.75s) so we don't resume in
  674. # the gap between VAD stop and the agent starting.
  675. idle_polls += 1
  676. if idle_polls >= 3:
  677. idle_polls = 0
  678. try:
  679. from tools.wake_word import resume_listening
  680. if resume_listening(owner=self):
  681. self._wake_suspended = False
  682. else:
  683. self._wake_word_active = False
  684. except Exception as e:
  685. logger.debug("wake word resume failed: %s", e)
  686. finally:
  687. self._wake_watchdog_started = False
  688. threading.Thread(target=_loop, daemon=True, name="wake-watchdog").start()
  689. def _show_wake_word_status(self):
  690. """Show current wake-word listener status."""
  691. from cli import _ACCENT, _BOLD, _DIM, _RST, _cprint
  692. from tools.wake_word import (
  693. audio_is_silent, check_wake_word_requirements, is_listening, load_wake_word_config,
  694. owns_listener)
  695. cfg = load_wake_word_config()
  696. reqs = check_wake_word_requirements(cfg)
  697. owned = owns_listener(self)
  698. # ``state`` is the internal id; only its display label is localized.
  699. state = "listening" if owned and is_listening() else "paused" if owned else "off"
  700. _cprint(f"\n{_BOLD}{t('cli.voice.wake_status_title')}{_RST}")
  701. _cprint(t("cli.voice.label_state", state=t(f"cli.voice.state_{state}")))
  702. _cprint(t("cli.voice.label_phrase", phrase=reqs['phrase']))
  703. _cprint(t("cli.voice.label_provider", provider=reqs['provider']))
  704. _cprint(t("cli.voice.label_surface", surface=cfg.get('surface', 'auto')))
  705. _cprint(t("cli.voice.label_new_session",
  706. value=t("cli.shared.label_yes" if cfg.get('start_new_session', True) else "cli.shared.label_no")))
  707. if state == "listening" and audio_is_silent():
  708. _cprint(f" {_ACCENT}{t('cli.voice.mic_silent_warning')}{_RST}")
  709. _cprint(f" {_DIM}{t('cli.voice.mic_silent_macos_hint')}{_RST}")
  710. if not reqs["available"] and reqs.get("hint"):
  711. _cprint(f" {_DIM}{reqs['hint']}{_RST}")
  712. if not owned:
  713. _cprint(f" {_DIM}{t('cli.voice.wake_enable_hint')}{_RST}")
  714. def _tts_lease_async(self, active: bool) -> None:
  715. """Acquire/release this CLI's TTS engine lease in the background.
  716. Acquiring pre-loads the configured provider so the first reply starts hot; releasing
  717. lets the last-holder path unload resident local models. Never blocks or fails the toggle.
  718. """
  719. from cli import logger
  720. def _run():
  721. try:
  722. from tools.tts_tool_lifecycle import acquire_tts_lease, release_tts_lease
  723. if active:
  724. acquire_tts_lease("cli:voice-tts")
  725. else:
  726. release_tts_lease("cli:voice-tts")
  727. except Exception as e:
  728. logger.debug("voice: tts lease active=%s failed: %s", active, e)
  729. threading.Thread(target=_run, name="tts-lease-cli", daemon=True).start()
  730. def _toggle_voice_tts(self):
  731. """Toggle TTS output for voice mode."""
  732. from cli import _ACCENT, _DIM, _RST, _cprint
  733. if not self._voice_mode:
  734. _cprint(f"{_DIM}{t('cli.voice.enable_first_hint')}{_RST}")
  735. return
  736. with self._voice_lock:
  737. self._voice_tts = not self._voice_tts
  738. status = t("cli.shared.label_enabled" if self._voice_tts else "cli.shared.label_disabled")
  739. if self._voice_tts:
  740. from tools.tts_tool import check_tts_requirements
  741. if not check_tts_requirements():
  742. _cprint(f"{_DIM}{t('cli.voice.tts_no_provider')}{_RST}")
  743. self._tts_lease_async(self._voice_tts) # warm-up / release signal for the TTS engine
  744. _cprint(f"{_ACCENT}{t('cli.voice.tts_status', status=status)}{_RST}")
  745. def _show_voice_status(self):
  746. """Show current voice mode status."""
  747. from cli import _BOLD, _RST, _cprint
  748. from tools.voice_mode import check_voice_requirements
  749. reqs = check_voice_requirements()
  750. _on, _off = t("cli.voice.state_on"), t("cli.voice.state_off")
  751. _cprint(f"\n{_BOLD}{t('cli.voice.status_title')}{_RST}")
  752. _cprint(t("cli.voice.label_mode", value=_on if self._voice_mode else _off))
  753. _cprint(t("cli.voice.label_tts", value=_on if self._voice_tts else _off))
  754. _cprint(t("cli.voice.label_recording",
  755. value=t("cli.voice.state_yes") if self._voice_recording else t("cli.shared.label_no")))
  756. # Startup-pinned label so /voice status always matches the live prompt_toolkit
  757. # binding (live config would drift after a mid-session config edit).
  758. # See #19835.
  759. _cprint(t("cli.voice.label_record_key", shortcut=self._voice_record_key_label()))
  760. _cprint(f"\n {_BOLD}{t('cli.voice.label_requirements')}{_RST}")
  761. for line in reqs["details"].split("\n"):
  762. _cprint(f" {line}")