audio.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785
  1. """Audio dashboard routes: transcription upload + live transcribe-stream, voice config, ElevenLabs voices, TTS speak/lease and the speak-stream WebSocket.
  2. Extracted from ``hermes_cli.web_server``; helpers/state that tests monkeypatch on
  3. ``web_server`` stay there and are late-bound (cycle-safe).
  4. """
  5. import base64
  6. import binascii
  7. import contextlib
  8. import logging
  9. import queue
  10. import tempfile
  11. import threading
  12. import asyncio
  13. import json
  14. import os
  15. import urllib.parse
  16. import urllib.request
  17. from fastapi import APIRouter
  18. from hermes_cli.dashboard_auth.native_http import grant_home_scope
  19. from hermes_cli.web_routers._common import http_failure
  20. from hermes_cli.web_deps import late
  21. from hermes_cli.web_server_chat import _ws_auth_ok, _ws_request_is_allowed
  22. from hermes_cli.web_server_gateway import _read_dashboard_json_response, _split_text_for_speak_stream
  23. from hermes_cli.web_server_surface import policy
  24. from hermes_cli.web_session_sockets import SPEAK_STREAM_SOCKET, TRANSCRIBE_STREAM_SOCKET
  25. from fastapi import HTTPException, Request, WebSocket, WebSocketDisconnect
  26. from hermes_cli.web_models import (
  27. AudioTranscriptionRequest,
  28. STTLeaseRequest,
  29. TTSSpeakRequest,
  30. TTSLeaseRequest,
  31. VoiceLiveSessionRequest,
  32. )
  33. from typing import Any, Dict, Optional
  34. _log = logging.getLogger("hermes_cli.web_server")
  35. router = APIRouter()
  36. # Late-bound so a test's monkeypatch on the owning module wins at call time.
  37. _config_profile_scope = late("_config_profile_scope", "hermes_cli.web_server_profiles")
  38. _voice_list_error_logged_once = late("_voice_list_error_logged_once")
  39. load_env = late("load_env", "hermes_cli.config")
  40. _AUDIO_MIME_EXTENSIONS: dict[str, str] = {
  41. "audio/aac": ".aac", "audio/flac": ".flac", "audio/m4a": ".m4a", "audio/mp3": ".mp3",
  42. "audio/mp4": ".mp4", "audio/mpeg": ".mp3", "audio/ogg": ".ogg", "audio/wav": ".wav",
  43. "audio/wave": ".wav", "audio/webm": ".webm", "audio/x-m4a": ".m4a", "audio/x-wav": ".wav",
  44. "video/webm": ".webm",
  45. }
  46. _MAX_TRANSCRIPTION_UPLOAD_BYTES = 25 * 1024 * 1024
  47. _SPEAK_MIME_BY_EXT = {
  48. ".mp3": "audio/mpeg", ".ogg": "audio/ogg", ".opus": "audio/ogg", ".wav": "audio/wav",
  49. ".flac": "audio/flac",
  50. }
  51. def _unlink_quietly(path: str) -> None:
  52. try:
  53. os.unlink(path)
  54. except OSError:
  55. pass
  56. async def _run_config_scoped(profile: Optional[str], fn):
  57. """Run ``fn()`` on a worker thread under the config-only profile scope.
  58. Home-only contextvar scope, NOT ``_profile_scope``: these calls block for a
  59. provider round-trip and only need config/.env resolution, while
  60. ``_profile_scope`` holds a process-global skills lock for its entire body.
  61. """
  62. def _scoped():
  63. with _config_profile_scope(profile):
  64. return fn()
  65. # to_thread copies the request context: a secondary-profile native ticket's implicit
  66. # scope (native_profile_scope's home override) must reach the worker, which
  67. # run_in_executor drops, silently resolving the LAUNCH profile's config and keys.
  68. return await asyncio.to_thread(_scoped)
  69. def _audio_extension_for_mime(mime_type: str) -> str:
  70. normalized = (mime_type or "").split(";", 1)[0].strip().lower()
  71. return _AUDIO_MIME_EXTENSIONS.get(normalized, ".webm")
  72. @router.post("/api/audio/transcribe")
  73. async def transcribe_audio_upload(
  74. payload: AudioTranscriptionRequest, profile: Optional[str] = None
  75. ):
  76. data_url = (payload.data_url or "").strip()
  77. if not data_url.startswith("data:") or "," not in data_url:
  78. raise HTTPException(status_code=400, detail="Invalid audio payload")
  79. header, encoded = data_url.split(",", 1)
  80. if ";base64" not in header:
  81. raise HTTPException(status_code=400, detail="Audio payload must be base64 encoded")
  82. mime_type = (payload.mime_type or header[5:].split(";", 1)[0] or "audio/webm").strip()
  83. normalized_mime_type = mime_type.split(";", 1)[0].lower()
  84. if not (normalized_mime_type.startswith("audio/") or normalized_mime_type == "video/webm"):
  85. raise HTTPException(status_code=400, detail="Payload must be an audio recording")
  86. try:
  87. audio_bytes = base64.b64decode(encoded, validate=True)
  88. except (binascii.Error, ValueError):
  89. raise HTTPException(status_code=400, detail="Audio payload is not valid base64")
  90. if not audio_bytes:
  91. raise HTTPException(status_code=400, detail="Audio recording is empty")
  92. if len(audio_bytes) > _MAX_TRANSCRIPTION_UPLOAD_BYTES:
  93. raise HTTPException(status_code=413, detail="Audio recording is too large")
  94. temp_path = ""
  95. try:
  96. with http_failure("Desktop voice transcription failed", 500, "Transcription failed"):
  97. with tempfile.NamedTemporaryFile(
  98. prefix="hermes-desktop-voice-", suffix=_audio_extension_for_mime(mime_type),
  99. delete=False,
  100. ) as tmp:
  101. tmp.write(audio_bytes)
  102. temp_path = tmp.name
  103. # transcribe_recording (not raw transcribe_audio): filters Whisper
  104. # hallucinations and maps provider "empty transcript" errors to a
  105. # successful empty result — the live voice loop treats "" as silence
  106. # and re-listens instead of surfacing a 400 on every quiet turn.
  107. from tools.voice_mode import transcribe_recording
  108. result = await _run_config_scoped(profile, lambda: transcribe_recording(temp_path))
  109. finally:
  110. if temp_path:
  111. _unlink_quietly(temp_path)
  112. if not result.get("success"):
  113. err = result.get("error") or "Transcription failed"
  114. # No speech detected is a normal outcome for VAD/continuous voice loops
  115. # (re-listening on silence), not an error: return an empty transcript so
  116. # the client quietly re-listens instead of showing a failure toast.
  117. if "empty transcript" in err.lower():
  118. return {"ok": True, "transcript": "", "provider": result.get("provider")}
  119. raise HTTPException(status_code=400, detail=err)
  120. return {
  121. "ok": True, "transcript": str(result.get("transcript") or "").strip(),
  122. "provider": result.get("provider"),
  123. }
  124. @router.get("/api/audio/voice-config")
  125. async def get_client_voice_config(request: Request, profile: Optional[str] = None):
  126. """The active profile's STT/TTS config for CLIENT-DIRECT voice.
  127. Lets the desktop cut the audio relay hop: mic audio goes straight to the
  128. profile's STT provider and reply text is synthesized on the client with
  129. the profile's TTS provider — the desktop↔gateway link carries only text.
  130. Providers that can only run on this host (local whisper, edge-tts,
  131. command/plugin providers) resolve to ``{"mode": "relay"}`` and the
  132. desktop keeps using the /api/audio/* relay endpoints.
  133. Same trust boundary as every profile-scoped route: the caller is an
  134. authenticated client that can already drive the agent. Keys in the
  135. response are held in client memory only, never persisted client-side.
  136. Gate: ``voice.client_direct`` in config.yaml (default true), and the
  137. serving surface (``client_direct_voice``: the Webapp always relays).
  138. """
  139. from tools.voice_client_config import resolve_client_voice_config
  140. client_direct = policy(request.app.state).client_direct_voice
  141. try:
  142. result = await _run_config_scoped(profile, lambda: resolve_client_voice_config(client_direct))
  143. except HTTPException:
  144. raise # an unknown ?profile= is the scope's 404, not a reason to fall back to relay
  145. except Exception:
  146. _log.exception("Client voice-config resolution failed")
  147. fallback = {"mode": "relay", "reason": "resolution error"}
  148. return {"ok": True, "stt": fallback, "tts": dict(fallback)}
  149. return {"ok": True, **result}
  150. @router.get("/api/audio/voice-live/status")
  151. async def get_voice_live_status(profile: Optional[str] = None):
  152. """Which voice chat mode the profile selected (``chained`` | ``gpt-live``) and whether GPT-Live
  153. can start. Non-secret: the desktop decides which conversation engine to mount from this."""
  154. from tools.voice_live import resolve_gpt_live_status
  155. with http_failure("GPT-Live status resolution failed", 500, "GPT-Live status failed"):
  156. result = await _run_config_scoped(profile, resolve_gpt_live_status)
  157. return {"ok": True, **result}
  158. @router.post("/api/audio/voice-live/session")
  159. async def create_voice_live_session(payload: VoiceLiveSessionRequest, profile: Optional[str] = None):
  160. """Exchange the renderer's WebRTC SDP offer for a GPT-Live session answer.
  161. The project API key stays on this host; the renderer only receives the session id and the
  162. SDP answer. Client delegation is fixed at creation: every ``session.delegation.created`` the
  163. renderer receives becomes a Hermes turn on the session it belongs to.
  164. """
  165. from tools.voice_live import create_webrtc_session
  166. # Validate emptiness only: the vendor's SDP parser needs the offer byte-exact, including the
  167. # trailing CRLF (a stripped offer answers 400 "failed to unmarshal SDP: EOF").
  168. sdp = payload.sdp or ""
  169. if not sdp.strip():
  170. raise HTTPException(status_code=400, detail="An SDP offer is required")
  171. try:
  172. result = await _run_config_scoped(profile, lambda: create_webrtc_session(sdp, payload.history))
  173. except ValueError as exc:
  174. raise HTTPException(status_code=503, detail=str(exc))
  175. except RuntimeError as exc:
  176. raise HTTPException(status_code=502, detail=str(exc))
  177. return {"ok": True, **result}
  178. def _elevenlabs_voice_label(voice: dict[str, Any]) -> str:
  179. name = str(voice.get("name") or voice.get("voice_id") or "Voice").strip()
  180. category = str(voice.get("category") or "").strip()
  181. return f"{name} ({category})" if category else name
  182. @router.get("/api/audio/elevenlabs/voices")
  183. async def get_elevenlabs_voices(profile: Optional[str] = None):
  184. """Return ElevenLabs voices when an API key is configured.
  185. The desktop UI uses this for the ``tts.elevenlabs.voice_id`` dropdown.
  186. Only non-secret voice metadata is returned; the API key stays server-side.
  187. """
  188. # Config-only scope (await-safe): the key lookup reads the requested
  189. # profile's .env, matching the profile the settings UI writes to.
  190. with _config_profile_scope(profile):
  191. api_key = (load_env().get("ELEVENLABS_API_KEY") or "").strip()
  192. if not api_key:
  193. # Fallback for env-only deployments — scope-aware: under multiplex
  194. # os.environ may hold another profile's key, so honor the installed
  195. # scope's verdict. Only the unscoped default-profile path
  196. # (UnscopedSecretError) reads the env; any other failure stays empty.
  197. try:
  198. from agent.secret_scope import UnscopedSecretError, get_secret
  199. try:
  200. api_key = (get_secret("ELEVENLABS_API_KEY") or "").strip()
  201. except UnscopedSecretError:
  202. api_key = (os.environ.get("ELEVENLABS_API_KEY") or "").strip()
  203. except Exception:
  204. pass
  205. if not api_key:
  206. return {"available": False, "voices": []}
  207. request = urllib.request.Request(
  208. "https://api.elevenlabs.io/v1/voices",
  209. headers={"Accept": "application/json", "xi-api-key": api_key},
  210. )
  211. try:
  212. def _fetch() -> dict[str, Any]:
  213. with urllib.request.urlopen(request, timeout=10) as response:
  214. return _read_dashboard_json_response(response)
  215. payload = await asyncio.to_thread(_fetch)
  216. except urllib.error.HTTPError as exc:
  217. # An auth failure (bad/expired/scoped key) is a persistent, user-fixable
  218. # state and the desktop polls this on every settings open/focus, so
  219. # treat 401/403 as "integration unavailable": 200 to the UI and log at
  220. # most once until the error signature changes.
  221. if exc.code in (401, 403):
  222. if _voice_list_error_logged_once(f"http-{exc.code}"):
  223. _log.info("ElevenLabs voices unavailable: %s — check ELEVENLABS_API_KEY", exc)
  224. return {"available": False, "voices": [], "error": "unauthorized"}
  225. if _voice_list_error_logged_once(f"http-{exc.code}"):
  226. _log.warning("ElevenLabs voice list failed: %s", exc)
  227. raise HTTPException(status_code=502, detail="Could not load ElevenLabs voices")
  228. except Exception as exc:
  229. if _voice_list_error_logged_once(str(exc)):
  230. _log.warning("ElevenLabs voice list failed: %s", exc)
  231. raise HTTPException(status_code=502, detail="Could not load ElevenLabs voices")
  232. _voice_list_error_logged_once(None) # success — re-arm logging for next failure
  233. voices = []
  234. for voice in payload.get("voices") or []:
  235. if not isinstance(voice, dict):
  236. continue
  237. voice_id = str(voice.get("voice_id") or "").strip()
  238. if not voice_id:
  239. continue
  240. voices.append({
  241. "voice_id": voice_id, "name": str(voice.get("name") or voice_id),
  242. "label": _elevenlabs_voice_label(voice),
  243. })
  244. voices.sort(key=lambda item: str(item.get("label") or "").lower())
  245. return {"available": True, "voices": voices}
  246. @router.post("/api/audio/speak")
  247. async def speak_text(payload: TTSSpeakRequest, profile: Optional[str] = None):
  248. """Synthesize speech and return audio as base64 data URL.
  249. Used by the desktop voice-conversation mode to play back assistant
  250. responses without exposing the on-disk file path; reuses the TTS provider
  251. chain configured under ``tts.`` in config.yaml.
  252. """
  253. text = (payload.text or "").strip()
  254. if not text:
  255. raise HTTPException(status_code=400, detail="Text is required")
  256. # _config_profile_scope raises 400/404 for a bad profile — pass it
  257. # through instead of masking it as a 500 synthesis failure.
  258. with http_failure("Desktop voice TTS failed", 500, "Speech synthesis failed"):
  259. from tools.tts_tool import text_to_speech_tool
  260. result_json = await _run_config_scoped(profile, lambda: text_to_speech_tool(text))
  261. try:
  262. result = json.loads(result_json) if isinstance(result_json, str) else result_json
  263. except Exception:
  264. raise HTTPException(status_code=500, detail="Invalid TTS response")
  265. if not result.get("success"):
  266. raise HTTPException(
  267. status_code=400, detail=result.get("error") or "Speech synthesis failed",
  268. )
  269. file_path = result.get("file_path")
  270. if not file_path or not os.path.isfile(file_path):
  271. raise HTTPException(status_code=500, detail="Audio file missing")
  272. mime_type = _SPEAK_MIME_BY_EXT.get(os.path.splitext(file_path)[1].lower(), "audio/mpeg")
  273. def _read_and_unlink() -> bytes:
  274. # Off-loop: synthesized audio can be several MB; reading it inline
  275. # blocks the uvicorn event loop. Unlink rides the same thread hop so
  276. # the temp file cannot outlive an early return.
  277. try:
  278. with open(file_path, "rb") as fh:
  279. return fh.read()
  280. finally:
  281. _unlink_quietly(file_path)
  282. try:
  283. audio_bytes = await asyncio.to_thread(_read_and_unlink)
  284. except OSError as exc:
  285. raise HTTPException(status_code=500, detail=f"Could not read audio: {exc}")
  286. encoded = base64.b64encode(audio_bytes).decode("ascii")
  287. return {
  288. "ok": True, "data_url": f"data:{mime_type};base64,{encoded}", "mime_type": mime_type,
  289. "provider": result.get("provider"),
  290. }
  291. @router.post("/api/audio/tts-lease")
  292. async def tts_lease(payload: TTSLeaseRequest, profile: Optional[str] = None):
  293. """Desktop TTS-output toggles as warm-up / release signals.
  294. ``active: true`` registers a lease on the TTS engine and pre-loads the
  295. configured provider (local model, lazily-installed SDK) so the first spoken
  296. reply doesn't pay the load as dead air; ``active: false`` drops the lease
  297. and, once no surface holds one, unloads resident local models. Blocking
  298. work runs off the event loop. Warm-up failures are reported in the body,
  299. never as an HTTP error — the toggle must succeed even when preload fails.
  300. """
  301. lease = (payload.lease or "").strip()
  302. if not lease:
  303. raise HTTPException(status_code=400, detail="lease is required")
  304. def _apply():
  305. from tools.tts_tool_lifecycle import acquire_tts_lease, release_tts_lease
  306. if payload.active:
  307. with _config_profile_scope(profile):
  308. return acquire_tts_lease(lease)
  309. # Release reads the requester's keep_warm_seconds, but must drop the lease even when
  310. # that profile is gone — a stuck lease pins the local model in memory.
  311. try:
  312. with _config_profile_scope(profile):
  313. return release_tts_lease(lease)
  314. except HTTPException:
  315. return release_tts_lease(lease)
  316. try:
  317. result = await asyncio.to_thread(_apply)
  318. except HTTPException:
  319. raise
  320. except Exception as exc:
  321. _log.warning("TTS lease %s (%s) failed: %s", lease, payload.active, exc)
  322. result = {"leases": None, "action": "error", "error": str(exc)}
  323. return {"ok": True, "lease": lease, "active": payload.active, **result}
  324. class _SyncSentencePCMStreamer:
  325. """Speak one sentence through the sync TTS tool and yield int16 mono PCM.
  326. Not a second provider: ``text_to_speech_tool`` is the same stack the CLI
  327. speaker uses for edge and every other non-chunked provider. The desktop
  328. socket only plays PCM, so the written file is decoded before it is sent.
  329. ``sample_rate`` is updated before the first yield, matching the chunked
  330. streamers whose rate is only final once synthesis has answered.
  331. """
  332. sample_rate = 24000
  333. channels = 1
  334. def stream(self, text: str):
  335. pcm, rate = _sync_sentence_to_pcm(text)
  336. if rate:
  337. self.sample_rate = rate
  338. if pcm:
  339. yield pcm
  340. def _sync_sentence_to_pcm(text: str) -> tuple:
  341. """Synthesize *text* with the configured sync provider and return PCM."""
  342. from tools.tts_tool import text_to_speech_tool
  343. fd, tmp_path = tempfile.mkstemp(suffix=".mp3")
  344. os.close(fd)
  345. extra = None
  346. try:
  347. raw = text_to_speech_tool(text=text, output_path=tmp_path)
  348. try:
  349. payload = json.loads(raw) if isinstance(raw, str) else raw
  350. except (TypeError, ValueError) as exc:
  351. raise RuntimeError("Invalid TTS response") from exc
  352. if not isinstance(payload, dict) or not payload.get("success"):
  353. detail = str(payload.get("error") or "") if isinstance(payload, dict) else ""
  354. raise RuntimeError(detail or "Speech synthesis failed")
  355. written = payload.get("file_path") or tmp_path
  356. if not isinstance(written, str) or not os.path.isfile(written) or os.path.getsize(written) <= 0:
  357. raise RuntimeError("Audio file missing")
  358. if os.path.abspath(written) != os.path.abspath(tmp_path):
  359. extra = written
  360. pcm, rate = _audio_file_to_pcm(written)
  361. if not pcm:
  362. raise RuntimeError("TTS audio decoded to silence")
  363. return pcm, rate
  364. finally:
  365. _unlink_quietly(tmp_path)
  366. if extra:
  367. _unlink_quietly(extra)
  368. def _audio_file_to_pcm(path: str) -> tuple:
  369. """Decode *path* to int16 mono PCM. WAV via the stdlib; anything else via ffmpeg."""
  370. with open(path, "rb") as fh:
  371. head = fh.read(12)
  372. if len(head) >= 12 and head.startswith(b"RIFF") and head[8:12] == b"WAVE":
  373. pcm, rate = _wav_s16le_mono(path)
  374. if pcm:
  375. return pcm, rate
  376. return _ffmpeg_s16le_mono(path)
  377. def _wav_s16le_mono(path: str) -> tuple:
  378. import wave
  379. try:
  380. with wave.open(path, "rb") as wf:
  381. if wf.getsampwidth() != 2 or wf.getnchannels() != 1 or wf.getframerate() <= 0:
  382. return b"", 0
  383. return wf.readframes(wf.getnframes()), int(wf.getframerate())
  384. except (wave.Error, EOFError, OSError):
  385. return b"", 0
  386. def _ffmpeg_s16le_mono(path: str) -> tuple:
  387. import shutil
  388. from tools.tts_tool_delivery import _ffmpeg_run
  389. ffmpeg = shutil.which("ffmpeg")
  390. if not ffmpeg:
  391. raise RuntimeError("ffmpeg is required to decode TTS audio for speak-stream")
  392. result = _ffmpeg_run(
  393. ffmpeg,
  394. ["-i", path, "-f", "s16le", "-ac", "1", "-ar", "24000", "-loglevel", "error", "pipe:1"],
  395. timeout=60,
  396. )
  397. if result.returncode != 0 or not result.stdout:
  398. stderr = (result.stderr or b"").decode("utf-8", "replace")[:200]
  399. raise RuntimeError(f"TTS audio decode failed: {stderr}")
  400. return result.stdout, 24000
  401. @router.post("/api/audio/stt-lease")
  402. async def stt_lease(payload: STTLeaseRequest, profile: Optional[str] = None):
  403. """Desktop voice-input sessions as STT warm-up / release signals.
  404. ``active: true`` registers a lease and pre-loads the configured local STT
  405. model (first-use download + load) so the transcription request doesn't pay
  406. the cold cost inside its timeout; ``active: false`` drops the lease. The
  407. model stays resident after the last release — it is shared with the
  408. gateway/CLI surfaces in this process, and ``stt.local.unload_after_idle_seconds``
  409. still governs eviction. Blocking work runs off the event loop. Warm-up
  410. failures are reported in the body, never as an HTTP error — recording must
  411. start even when preload fails.
  412. """
  413. lease = (payload.lease or "").strip()
  414. if not lease:
  415. raise HTTPException(status_code=400, detail="lease is required")
  416. def _apply():
  417. from tools.stt_lease import acquire_stt_lease, release_stt_lease
  418. if payload.active:
  419. with _config_profile_scope(profile):
  420. return acquire_stt_lease(lease)
  421. return release_stt_lease(lease)
  422. try:
  423. result = await asyncio.to_thread(_apply)
  424. except HTTPException:
  425. raise
  426. except Exception as exc:
  427. _log.warning("STT lease %s (%s) failed: %s", lease, payload.active, exc)
  428. result = {"leases": None, "action": "error", "error": str(exc)}
  429. return {"ok": True, "lease": lease, "active": payload.active, **result}
  430. @router.websocket("/api/audio/transcribe-stream")
  431. async def transcribe_stream_ws(ws: WebSocket) -> None:
  432. """Live STT for Desktop dictation: mic PCM in while the user speaks, partial + final text out.
  433. Protocol:
  434. client → ``{"sample_rate": N}`` first (8000-48000), then binary s16le mono PCM frames,
  435. ``{"eos": true}`` when the recording ends (disconnect = cancel)
  436. server → ``{"type": "partial", "text": ...}`` as speech is recognized,
  437. ``{"type": "final", "transcript": ..., "provider": ...}``,
  438. ``{"type": "error", "message": ...}`` — also when no live STT is available, so the
  439. client falls back to ``POST /api/audio/transcribe`` with its recorded blob.
  440. """
  441. # A socket ticket opens this stream only when minted for it (``/api/webapp/socket-ticket``).
  442. if not _ws_auth_ok(ws, socket=TRANSCRIBE_STREAM_SOCKET):
  443. await ws.close(code=4401)
  444. return
  445. if not _ws_request_is_allowed(ws):
  446. await ws.close(code=4403)
  447. return
  448. await ws.accept(subprotocol=getattr(ws, "_hermes_ws_subprotocol", None))
  449. profile = (ws.query_params.get("profile") or "").strip() or None
  450. grant = getattr(ws, "_hermes_native_grant", None)
  451. loop = asyncio.get_running_loop()
  452. partials: asyncio.Queue = asyncio.Queue()
  453. def _open():
  454. from tools.transcription_streaming import open_streaming_session
  455. with grant_home_scope(grant), _config_profile_scope(profile):
  456. return open_streaming_session(on_partial=lambda text: loop.call_soon_threadsafe(partials.put_nowait, text))
  457. session = None
  458. with contextlib.suppress(Exception):
  459. session = await loop.run_in_executor(None, _open)
  460. if session is None:
  461. with contextlib.suppress(Exception):
  462. await ws.send_json({"type": "error", "message": "no live STT for the configured provider"})
  463. await ws.close()
  464. return
  465. async def _forward_partials():
  466. while True:
  467. await ws.send_json({"type": "partial", "text": await partials.get()})
  468. forwarder = asyncio.ensure_future(_forward_partials())
  469. ended = False
  470. try:
  471. first = json.loads(await ws.receive_text())
  472. rate = int(first.get("sample_rate") or 0)
  473. if not 8000 <= rate <= 48000:
  474. raise ValueError(f"unsupported sample_rate {rate}")
  475. session.set_input_rate(rate)
  476. while True:
  477. message = await ws.receive()
  478. if message.get("type") == "websocket.disconnect":
  479. break
  480. if message.get("bytes"):
  481. session.push_audio(message["bytes"])
  482. elif message.get("text") and json.loads(message["text"]).get("eos"):
  483. ended = True
  484. break
  485. except (WebSocketDisconnect, ValueError, RuntimeError) as exc:
  486. _log.debug("transcribe-stream client ended early: %s", exc)
  487. if not ended:
  488. session.cancel()
  489. forwarder.cancel()
  490. return
  491. def _finalize():
  492. with grant_home_scope(grant), _config_profile_scope(profile):
  493. return session.finalize()
  494. result = await loop.run_in_executor(None, _finalize)
  495. await asyncio.sleep(0) # let the last partial flush before the final frame
  496. forwarder.cancel()
  497. with contextlib.suppress(Exception):
  498. if result.get("success"):
  499. await ws.send_json({"type": "final", "transcript": result.get("transcript", ""),
  500. "provider": result.get("provider", "")})
  501. else:
  502. await ws.send_json({"type": "error", "message": result.get("error") or "transcription failed"})
  503. await ws.close()
  504. @router.websocket("/api/audio/speak-stream")
  505. async def speak_stream_ws(ws: WebSocket) -> None:
  506. """Streaming TTS for the desktop: text in, raw int16 PCM frames out.
  507. The socket is a per-reply speech *session*: the client feeds text
  508. incrementally as LLM deltas arrive, the server cuts sentences
  509. (``SentenceChunker`` — same cutter as the CLI/TUI speaker pipeline) and
  510. streams each one's PCM the moment it's ready, so speech overlaps generation.
  511. Protocol:
  512. client → ``{"text": "..."}`` frames (incremental; may combine with done),
  513. ``{"done": true}`` when the reply is complete,
  514. ``{"stop": true}`` or disconnect = barge-in
  515. server → ``{"type": "start", "sample_rate": N, "channels": 1}`` (sent
  516. with the first PCM frame, once the provider's rate is final),
  517. binary PCM frames, then ``{"type": "end"}``
  518. server → ``{"type": "fallback"}`` only when sentence synthesis produced
  519. no audio. Providers with no chunked API (edge, the default)
  520. still speak per sentence via ``text_to_speech_tool`` and stream
  521. that PCM — the client POST is the last resort, not the Edge path.
  522. """
  523. # A socket ticket opens this stream only when minted for it (``/api/webapp/socket-ticket``).
  524. if not _ws_auth_ok(ws, socket=SPEAK_STREAM_SOCKET):
  525. await ws.close(code=4401)
  526. return
  527. if not _ws_request_is_allowed(ws):
  528. await ws.close(code=4403)
  529. return
  530. await ws.accept(subprotocol=getattr(ws, "_hermes_ws_subprotocol", None))
  531. # Profile via query param, like /api/pty and /api/console: the provider
  532. # chain + API keys must resolve from the requesting profile's config, not
  533. # the dashboard's own — at resolve time AND in the synthesis thread. A
  534. # native grant's home owns the implicit selector (its explicit ones match it).
  535. profile = (ws.query_params.get("profile") or "").strip() or None
  536. grant = getattr(ws, "_hermes_native_grant", None)
  537. loop = asyncio.get_running_loop()
  538. def _resolve():
  539. from tools.tts_streaming import resolve_streaming_provider
  540. from tools.tts_tool import _get_provider, _load_tts_config, _resolve_max_text_length
  541. with grant_home_scope(grant), _config_profile_scope(profile):
  542. cfg = _load_tts_config()
  543. streamer = resolve_streaming_provider(cfg)
  544. cap = _resolve_max_text_length(_get_provider(cfg), cfg)
  545. return streamer, cap, cfg
  546. try:
  547. streamer, cap, cfg = await asyncio.to_thread(_resolve)
  548. except Exception:
  549. _log.exception("speak-stream provider resolution failed")
  550. streamer, cap, cfg = None, 0, {}
  551. if streamer is None:
  552. # Edge (the default) and every other non-chunked provider still have a
  553. # documented per-sentence path. type=fallback here is what makes Desktop
  554. # wait for the whole reply and POST it to /api/audio/speak.
  555. streamer = _SyncSentencePCMStreamer()
  556. # The start frame is deferred until the first PCM chunk (or end-of-speech):
  557. # the OpenAI-compatible streamer only learns the endpoint's real rate from
  558. # the response headers inside stream(), and the client opens its
  559. # AudioContext at whatever rate the start frame carries.
  560. start_sent = False
  561. async def _send_start():
  562. nonlocal start_sent
  563. if start_sent:
  564. return
  565. start_sent = True
  566. await ws.send_json(
  567. {"type": "start", "sample_rate": streamer.sample_rate, "channels": streamer.channels}
  568. )
  569. stop = threading.Event()
  570. produced_audio = False
  571. synthesis_failed = False
  572. text_q: queue.Queue = queue.Queue() # str deltas; None = end-of-text
  573. chunks: asyncio.Queue = asyncio.Queue() # PCM out; None = synthesis done
  574. def _produce():
  575. # Every streamer re-resolves its API key on each stream() call (tts_streaming ->
  576. # resolve_provider_secret), so the whole synthesis body runs under the requesting
  577. # profile's scope, not only the resolve step above (else the launch profile's key).
  578. with grant_home_scope(grant), _config_profile_scope(profile):
  579. _synthesize()
  580. def _synthesize():
  581. nonlocal produced_audio, synthesis_failed
  582. from tools.tts_streaming import SentenceChunker
  583. from tools.tts_text_normalize import _strip_markdown_for_tts
  584. chunker = SentenceChunker.from_config(cfg) # the requesting profile's tts.streaming.min_len
  585. # The session stays open for a whole agent turn and no text arrives
  586. # during tool execution, so without an idle flush a narration line with
  587. # no trailing whitespace ("Let me check.") sits in the chunker until
  588. # end-of-turn. Mirror the CLI speaker pipeline: poll with a timeout and
  589. # flush when the producer goes idle — immediately when the buffer ends
  590. # on sentence punctuation, after a longer quiet spell otherwise.
  591. idle_poll_seconds = 0.5
  592. idle_polls_before_force_flush = 4 # ~2s of silence
  593. def _sentences():
  594. idle_polls = 0
  595. while not stop.is_set():
  596. try:
  597. delta = text_q.get(timeout=idle_poll_seconds)
  598. except queue.Empty:
  599. idle_polls += 1
  600. buffered = chunker.buf.strip()
  601. if not buffered or ("<think" in chunker.buf and "</think>" not in chunker.buf):
  602. continue
  603. if buffered.endswith((".", "!", "?", "…", ":")) or idle_polls >= idle_polls_before_force_flush:
  604. yield from chunker.flush()
  605. continue
  606. idle_polls = 0
  607. if delta is None:
  608. yield from chunker.flush()
  609. return
  610. yield from chunker.feed(delta)
  611. try:
  612. for sentence in _sentences():
  613. cleaned = _strip_markdown_for_tts(sentence)
  614. if not cleaned:
  615. continue
  616. for piece in _split_text_for_speak_stream(cleaned, cap):
  617. for chunk in streamer.stream(piece):
  618. if stop.is_set():
  619. return
  620. produced_audio = True
  621. loop.call_soon_threadsafe(chunks.put_nowait, chunk)
  622. except Exception as exc:
  623. _log.warning("speak-stream synthesis failed: %s", exc)
  624. synthesis_failed = True
  625. finally:
  626. loop.call_soon_threadsafe(chunks.put_nowait, None)
  627. threading.Thread(target=_produce, daemon=True).start()
  628. async def _pump_client():
  629. # Text frames feed synthesis; done ends the text; stop/disconnect
  630. # (or any unparseable frame) is barge-in.
  631. try:
  632. while True:
  633. frame = json.loads(await ws.receive_text())
  634. if frame.get("text"):
  635. text_q.put(str(frame["text"]))
  636. if frame.get("stop"):
  637. break
  638. if frame.get("done"):
  639. text_q.put(None)
  640. except Exception:
  641. pass
  642. stop.set()
  643. text_q.put(None) # unblock the producer
  644. pump = asyncio.ensure_future(_pump_client())
  645. try:
  646. while True:
  647. chunk = await chunks.get()
  648. if chunk is None:
  649. break
  650. await _send_start()
  651. await ws.send_bytes(chunk)
  652. if not stop.is_set():
  653. # Fallback is the last resort: sentence synthesis was asked for and
  654. # produced nothing. A normal edge reply has already streamed PCM.
  655. if synthesis_failed and not produced_audio:
  656. await ws.send_json({"type": "fallback"})
  657. else:
  658. await _send_start()
  659. await ws.send_json({"type": "end"})
  660. except (WebSocketDisconnect, RuntimeError):
  661. pass
  662. finally:
  663. stop.set()
  664. text_q.put(None)
  665. pump.cancel()
  666. with contextlib.suppress(Exception):
  667. await ws.close()