transcription_streaming.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543
  1. """Live speech-to-text: feed microphone PCM while the user speaks, get partial and final text.
  2. Every caller hands 16 kHz mono s16le PCM to :meth:`StreamingSession.push_audio` in any chunk
  3. size, receives non-final text through ``on_partial`` as the provider recognizes it, and calls
  4. :meth:`StreamingSession.finalize` at end of speech for the standard transcription envelope
  5. (``success`` / ``transcript`` / ``provider`` / ``error``). One worker thread owns the provider
  6. socket, so callers never block on network I/O while audio is captured.
  7. Built-in wires (live-verified against each vendor except where noted):
  8. - ``openai``: Realtime ``type: transcription`` session (``stt.openai.streaming_model``, default
  9. ``gpt-live-transcribe``, the one model that emits deltas before the turn is committed). The
  10. endpoint requires >= 24 kHz input, so PCM is resampled 16 k -> 24 k here.
  11. - ``xai``: ``wss://api.x.ai/v1/stt`` with ``interim_results=true``.
  12. - ``elevenlabs``: ``wss://api.elevenlabs.io/v1/speech-to-text/realtime`` (``scribe_v2_realtime``),
  13. built from the vendor AsyncAPI schema.
  14. Plugin providers opt in through ``TranscriptionProvider.streaming_capable`` /
  15. ``open_stream_session``. Streaming is opt-in (``stt.streaming: true``); every surface falls back
  16. to the file path when no session opens or a session fails.
  17. """
  18. from __future__ import annotations
  19. import base64
  20. import contextvars
  21. import json
  22. import logging
  23. import queue
  24. import threading
  25. import time
  26. from array import array
  27. from typing import Any, Callable, Dict, Optional
  28. from urllib.parse import urlencode, urlsplit, urlunsplit
  29. from agent.memory_provider import spawn_context_thread
  30. from utils import is_truthy_value
  31. logger = logging.getLogger(__name__)
  32. INPUT_RATE = 16000 # the contract every caller feeds
  33. DEFAULT_OPENAI_STREAMING_MODEL = "gpt-live-transcribe"
  34. ELEVENLABS_REALTIME_MODEL = "scribe_v2_realtime"
  35. STREAMING_PROVIDERS = ("openai", "xai", "elevenlabs")
  36. _FINALIZE_TIMEOUT_S = 20.0
  37. _CONNECT_TIMEOUT_S = 10.0
  38. PartialCallback = Callable[[str], None]
  39. class Resampler:
  40. """Stateful s16le mono rate converter (linear; box-average for integer downsampling)."""
  41. def __init__(self, src_rate: int, dst_rate: int) -> None:
  42. self.src, self.dst = int(src_rate), int(dst_rate)
  43. self._pos = 0.0 # fractional read position into the carried + new samples
  44. self._carry = array("h")
  45. def __call__(self, pcm: bytes) -> bytes:
  46. if self.src == self.dst or not pcm:
  47. return pcm
  48. samples = array("h")
  49. samples.frombytes(pcm[: len(pcm) - len(pcm) % 2])
  50. if self.src % self.dst == 0:
  51. data = self._carry + samples
  52. step = self.src // self.dst
  53. usable = len(data) - len(data) % step
  54. out = array("h", (sum(data[i:i + step]) // step for i in range(0, usable, step)))
  55. self._carry = data[usable:]
  56. return out.tobytes()
  57. data = self._carry + samples
  58. ratio = self.src / self.dst
  59. out = array("h")
  60. pos = self._pos
  61. last = len(data) - 1
  62. while pos < last:
  63. i = int(pos)
  64. frac = pos - i
  65. out.append(int(data[i] + (data[i + 1] - data[i]) * frac))
  66. pos += ratio
  67. keep = int(pos)
  68. self._carry = data[keep:]
  69. self._pos = pos - keep
  70. return out.tobytes()
  71. class StreamingSession:
  72. """One utterance of live STT. Thread-safe ``push_audio``; single-use."""
  73. provider = "unknown"
  74. def __init__(self, on_partial: Optional[PartialCallback] = None) -> None:
  75. self._on_partial = on_partial
  76. self._audio: queue.Queue[Optional[bytes]] = queue.Queue()
  77. self._done = threading.Event()
  78. self._cancelled = threading.Event()
  79. self._result: dict[str, Any] = {}
  80. self._partial = ""
  81. self._worker: Optional[threading.Thread] = None
  82. self._to_16k: Optional[Resampler] = None
  83. def set_input_rate(self, rate: int) -> None:
  84. """Accept PCM at ``rate`` (e.g. the mic's device rate); the worker converts it to 16 kHz,
  85. so a realtime capture callback never pays for resampling."""
  86. if int(rate) != INPUT_RATE:
  87. self._to_16k = Resampler(int(rate), INPUT_RATE)
  88. def _take_audio(self, block: bool, timeout: float = 0.0) -> Optional[bytes]:
  89. """Next 16 kHz chunk (``b""`` when none is queued, None at end of audio)."""
  90. try:
  91. chunk = self._audio.get(timeout=timeout) if block else self._audio.get_nowait()
  92. except queue.Empty:
  93. return b""
  94. if chunk and self._to_16k is not None:
  95. chunk = self._to_16k(chunk)
  96. return chunk
  97. # ── caller API ──
  98. def start(self) -> StreamingSession:
  99. # copy_context: a plugin session reads secrets from the opener's profile scope.
  100. ctx = contextvars.copy_context()
  101. self._worker = threading.Thread(target=ctx.run, args=(self._run,), name=f"stt-stream-{self.provider}",
  102. daemon=True)
  103. self._worker.start()
  104. return self
  105. def push_audio(self, pcm16k: bytes) -> None:
  106. if pcm16k and not self._done.is_set():
  107. self._audio.put(bytes(pcm16k))
  108. def end_audio(self) -> None:
  109. """Signal end of speech without waiting (the worker flushes the provider)."""
  110. self._audio.put(None)
  111. def finalize(self, timeout: float = _FINALIZE_TIMEOUT_S) -> dict[str, Any]:
  112. self.end_audio()
  113. if not self._done.wait(timeout):
  114. self.cancel()
  115. return self._error("live transcription timed out")
  116. return dict(self._result)
  117. def cancel(self) -> None:
  118. self._cancelled.set()
  119. self._audio.put(None)
  120. self._finish(self._error("cancelled"))
  121. def partial_transcript(self) -> str:
  122. return self._partial
  123. # ── worker side ──
  124. def _set_partial(self, text: str) -> None:
  125. text = text.strip()
  126. if text and text != self._partial:
  127. self._partial = text
  128. if self._on_partial is not None:
  129. try:
  130. self._on_partial(text)
  131. except Exception:
  132. logger.debug("on_partial callback raised", exc_info=True)
  133. def _error(self, message: str) -> dict[str, Any]:
  134. return {"success": False, "transcript": "", "provider": self.provider, "error": message}
  135. def _finish(self, result: dict[str, Any]) -> None:
  136. if not self._done.is_set():
  137. self._result = result
  138. self._done.set()
  139. def _run(self) -> None:
  140. try:
  141. self._finish(self._session())
  142. except Exception as exc: # health: allow BLE001 -- worker boundary: every failure becomes the envelope
  143. logger.warning("Live STT (%s) failed: %s", self.provider, exc, exc_info=True)
  144. self._finish(self._error(f"live transcription failed: {exc}"))
  145. def _session(self) -> dict[str, Any]: # pragma: no cover - overridden
  146. raise NotImplementedError
  147. class _WebSocketSession(StreamingSession):
  148. """A provider wire over one websocket: subclasses map audio out and events in."""
  149. url = ""
  150. headers: dict[str, str] = {}
  151. def _session(self) -> dict[str, Any]:
  152. from websockets.sync.client import connect
  153. ws = connect(self.url, additional_headers=self.headers, open_timeout=_CONNECT_TIMEOUT_S,
  154. close_timeout=1, max_size=None)
  155. try:
  156. return self._loop(ws)
  157. finally:
  158. # Publish first, close after: the sync close handshake waits up to close_timeout on a
  159. # server that already hung up, and the caller is blocked on finalize() meanwhile.
  160. spawn_context_thread(ws.close, name="stt-stream-close").start()
  161. def _loop(self, ws: Any) -> dict[str, Any]:
  162. self._open(ws)
  163. ending_since: Optional[float] = None
  164. while True:
  165. if self._cancelled.is_set():
  166. return self._error("cancelled")
  167. if ending_since is None:
  168. ending_since = self._drain_audio(ws)
  169. elif time.monotonic() - ending_since > _FINALIZE_TIMEOUT_S:
  170. return self._error("provider never finalized the transcript")
  171. try:
  172. raw = ws.recv(timeout=0.02)
  173. except TimeoutError:
  174. continue
  175. if isinstance(raw, (bytes, bytearray)):
  176. continue
  177. result = self._event(json.loads(raw))
  178. if result is not None:
  179. self._finish(result)
  180. return result
  181. if self._backlog and self._ready():
  182. self._flush_backlog(ws)
  183. def _drain_audio(self, ws: Any) -> Optional[float]:
  184. """Send queued audio; on end-of-audio send the provider's flush and return its time."""
  185. while True:
  186. chunk = self._take_audio(block=False)
  187. if chunk is None:
  188. self._end(ws)
  189. return time.monotonic()
  190. if not chunk:
  191. if self._audio.empty():
  192. return None
  193. continue
  194. if self._ready():
  195. self._send_audio(ws, chunk)
  196. else:
  197. self._backlog.append(chunk)
  198. _backlog: list[bytes]
  199. def _ready(self) -> bool:
  200. return True
  201. def _flush_backlog(self, ws: Any) -> None:
  202. backlog, self._backlog = self._backlog, []
  203. for chunk in backlog:
  204. self._send_audio(ws, chunk)
  205. def _open(self, ws: Any) -> None:
  206. self._backlog = []
  207. def _send_audio(self, ws: Any, chunk: bytes) -> None: # pragma: no cover - overridden
  208. raise NotImplementedError
  209. def _end(self, ws: Any) -> None: # pragma: no cover - overridden
  210. raise NotImplementedError
  211. def _event(self, event: dict[str, Any]) -> Optional[dict[str, Any]]: # pragma: no cover
  212. raise NotImplementedError
  213. class OpenAIRealtimeSession(_WebSocketSession):
  214. provider = "openai"
  215. def __init__(self, api_key: str, base_url: str, model: str, language: Optional[str],
  216. prompt: Optional[str], on_partial: Optional[PartialCallback] = None) -> None:
  217. super().__init__(on_partial)
  218. parts = urlsplit(base_url.rstrip("/"))
  219. scheme = "wss" if parts.scheme == "https" else "ws"
  220. self.url = urlunsplit((scheme, parts.netloc, f"{parts.path}/realtime", "intent=transcription", ""))
  221. self.headers = {"Authorization": f"Bearer {api_key}"}
  222. self.model, self.language, self.prompt = model, language, prompt
  223. self._resample = Resampler(INPUT_RATE, 24000)
  224. self._text = ""
  225. self._sent_audio = False
  226. def _open(self, ws: Any) -> None:
  227. super()._open(ws)
  228. transcription: dict[str, Any] = {"model": self.model}
  229. if self.language:
  230. # gpt-live-transcribe takes a ``languages`` list and rejects the singular field.
  231. if self.model == DEFAULT_OPENAI_STREAMING_MODEL:
  232. transcription["languages"] = [self.language]
  233. else:
  234. transcription["language"] = self.language
  235. if self.prompt:
  236. transcription["prompt"] = self.prompt
  237. ws.send(json.dumps({"type": "session.update", "session": {"type": "transcription", "audio": {"input": {
  238. "format": {"type": "audio/pcm", "rate": 24000}, "transcription": transcription,
  239. "turn_detection": None}}}}))
  240. def _send_audio(self, ws: Any, chunk: bytes) -> None:
  241. pcm = self._resample(chunk)
  242. if pcm:
  243. self._sent_audio = True
  244. ws.send(json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(pcm).decode()}))
  245. def _end(self, ws: Any) -> None:
  246. if not self._sent_audio:
  247. self._finish({"success": True, "transcript": "", "provider": self.provider})
  248. self._cancelled.set()
  249. return
  250. ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
  251. def _event(self, event: dict[str, Any]) -> Optional[dict[str, Any]]:
  252. kind = str(event.get("type") or "")
  253. if kind.endswith("input_audio_transcription.delta"):
  254. self._text += str(event.get("delta") or "")
  255. self._set_partial(self._text)
  256. elif kind.endswith("input_audio_transcription.completed"):
  257. return {"success": True, "transcript": str(event.get("transcript") or self._text).strip(),
  258. "provider": self.provider}
  259. elif kind == "error":
  260. message = str((event.get("error") or {}).get("message") or "provider error")
  261. # Committing < 100 ms of audio is "buffer too small": the user said nothing.
  262. if "buffer too small" in message.lower():
  263. return {"success": True, "transcript": "", "provider": self.provider}
  264. return self._error(message)
  265. return None
  266. class XAIStreamingSession(_WebSocketSession):
  267. provider = "xai"
  268. def __init__(self, api_key: str, base_url: str, model: str, language: Optional[str],
  269. on_partial: Optional[PartialCallback] = None) -> None:
  270. super().__init__(on_partial)
  271. parts = urlsplit(base_url.rstrip("/"))
  272. scheme = "wss" if parts.scheme == "https" else "ws"
  273. query = {"sample_rate": INPUT_RATE, "encoding": "pcm", "interim_results": "true", "model": model}
  274. if language:
  275. query["language"] = language
  276. self.url = urlunsplit((scheme, parts.netloc, f"{parts.path}/stt", urlencode(query), ""))
  277. self.headers = {"Authorization": f"Bearer {api_key}"}
  278. self._created = False
  279. self._utterances: list[str] = [] # speech_final stitched utterances
  280. self._chunks: list[str] = [] # is_final chunks of the current utterance
  281. self._interim = ""
  282. def _ready(self) -> bool:
  283. return self._created # the server drops audio sent before transcript.created
  284. def _send_audio(self, ws: Any, chunk: bytes) -> None:
  285. ws.send(chunk)
  286. def _end(self, ws: Any) -> None:
  287. if not self._created:
  288. self._backlog.clear()
  289. ws.send(json.dumps({"type": "audio.done"}))
  290. def _text(self) -> str:
  291. return " ".join(p for p in (*self._utterances, *self._chunks, self._interim) if p).strip()
  292. def _event(self, event: dict[str, Any]) -> Optional[dict[str, Any]]:
  293. kind = event.get("type")
  294. if kind == "transcript.created":
  295. self._created = True
  296. return None
  297. if kind == "transcript.partial":
  298. text = str(event.get("text") or "").strip()
  299. if event.get("speech_final"):
  300. self._utterances.append(text)
  301. self._chunks, self._interim = [], ""
  302. elif event.get("is_final"):
  303. self._chunks.append(text)
  304. self._interim = ""
  305. else:
  306. self._interim = text
  307. self._set_partial(self._text())
  308. return None
  309. if kind == "transcript.done":
  310. return {"success": True, "transcript": self._text(), "provider": self.provider}
  311. if kind == "error":
  312. return self._error(str(event.get("message") or "provider error"))
  313. return None
  314. class ElevenLabsRealtimeSession(_WebSocketSession):
  315. provider = "elevenlabs"
  316. def __init__(self, api_key: str, base_url: str, language: Optional[str],
  317. on_partial: Optional[PartialCallback] = None) -> None:
  318. super().__init__(on_partial)
  319. parts = urlsplit(base_url.rstrip("/"))
  320. scheme = "wss" if parts.scheme == "https" else "ws"
  321. query = {"model_id": ELEVENLABS_REALTIME_MODEL, "audio_format": f"pcm_{INPUT_RATE}",
  322. "commit_strategy": "vad"}
  323. if language:
  324. query["language_code"] = language
  325. self.url = urlunsplit((scheme, parts.netloc, f"{parts.path}/speech-to-text/realtime", urlencode(query), ""))
  326. self.headers = {"xi-api-key": api_key}
  327. self._committed: list[str] = []
  328. self._interim = ""
  329. self._ending = False
  330. def _chunk(self, pcm: bytes, commit: bool) -> str:
  331. return json.dumps({"message_type": "input_audio_chunk", "audio_base_64": base64.b64encode(pcm).decode(),
  332. "commit": commit, "sample_rate": INPUT_RATE})
  333. def _send_audio(self, ws: Any, chunk: bytes) -> None:
  334. ws.send(self._chunk(chunk, False))
  335. def _end(self, ws: Any) -> None:
  336. self._ending = True
  337. ws.send(self._chunk(b"", True))
  338. def _text(self) -> str:
  339. return " ".join(p for p in (*self._committed, self._interim) if p).strip()
  340. def _event(self, event: dict[str, Any]) -> Optional[dict[str, Any]]:
  341. kind = event.get("message_type")
  342. if kind == "partial_transcript":
  343. self._interim = str(event.get("text") or "").strip()
  344. self._set_partial(self._text())
  345. elif kind == "committed_transcript":
  346. text = str(event.get("text") or "").strip()
  347. if text:
  348. self._committed.append(text)
  349. self._interim = ""
  350. self._set_partial(self._text())
  351. if self._ending:
  352. return {"success": True, "transcript": self._text(), "provider": self.provider}
  353. elif kind in ("insufficient_audio_activity", "commit_throttled") and self._ending:
  354. return {"success": True, "transcript": self._text(), "provider": self.provider}
  355. elif kind and kind not in ("session_started", "committed_transcript_with_timestamps", "warning"):
  356. if "error" in event:
  357. return self._error(f"{kind}: {event.get('error')}")
  358. return None
  359. class _PluginSession(StreamingSession):
  360. """Adapts a plugin ``TranscriptionStreamSession`` to the caller API (partials polled)."""
  361. def __init__(self, provider_name: str, inner: Any, on_partial: Optional[PartialCallback]) -> None:
  362. super().__init__(on_partial)
  363. self.provider, self._inner = provider_name, inner
  364. def _session(self) -> dict[str, Any]:
  365. while True:
  366. chunk = self._take_audio(block=True, timeout=0.25)
  367. if self._cancelled.is_set():
  368. return self._error("cancelled")
  369. if chunk is None:
  370. return dict(self._inner.finalize())
  371. if chunk:
  372. self._inner.push_audio(chunk)
  373. self._set_partial(str(self._inner.partial_transcript() or ""))
  374. # ── resolution ──
  375. def streaming_enabled(stt_config: dict[str, Any]) -> bool:
  376. return is_truthy_value(stt_config.get("streaming", False), default=False)
  377. def _openai_credentials() -> Optional[tuple[str, str]]:
  378. """Direct OpenAI credentials only: the managed Nous gateway proxies the REST audio API, not
  379. the Realtime socket."""
  380. from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_selection
  381. from tools.transcription_tools import _resolve_openai_audio_client_config
  382. if read_selection("stt") == NOUS_MANAGED_PROVIDER:
  383. return None
  384. try:
  385. api_key, base_url = _resolve_openai_audio_client_config()
  386. except ValueError:
  387. return None
  388. return api_key, base_url
  389. def _builtin_session(provider: str, stt_config: dict[str, Any], language: Optional[str],
  390. prompt: Optional[str], on_partial: Optional[PartialCallback]) -> Optional[StreamingSession]:
  391. from tools import transcription_common as tc
  392. from tools import transcription_tools as tt
  393. raw_section = stt_config.get(provider)
  394. section: dict[str, Any] = raw_section if isinstance(raw_section, dict) else {}
  395. if provider == "openai":
  396. creds = _openai_credentials()
  397. if creds is None:
  398. return None
  399. model = str(section.get("streaming_model") or DEFAULT_OPENAI_STREAMING_MODEL)
  400. return OpenAIRealtimeSession(creds[0], creds[1], model, language, prompt, on_partial)
  401. if provider == "xai":
  402. from hermes_cli.config import get_env_value
  403. api_key = str(get_env_value("XAI_API_KEY") or "").strip() # API key only, like the Desktop wire
  404. if not api_key:
  405. return None
  406. base = str(section.get("base_url") or get_env_value("XAI_STT_BASE_URL") or tc.XAI_STT_BASE_URL)
  407. return XAIStreamingSession(api_key, base, tc.normalize_xai_stt_model(section.get("model")),
  408. language, on_partial)
  409. if provider == "elevenlabs":
  410. api_key = tt._resolve_provider_key("ELEVENLABS_API_KEY", "elevenlabs")
  411. if not api_key:
  412. return None
  413. base = str(section.get("base_url") or tc.ELEVENLABS_STT_BASE_URL)
  414. return ElevenLabsRealtimeSession(api_key, base, language, on_partial)
  415. return None
  416. def _plugin_session(provider: str, language: Optional[str], prompt: Optional[str],
  417. on_partial: Optional[PartialCallback]) -> Optional[StreamingSession]:
  418. from agent.transcription_registry import get_provider
  419. registered = get_provider(provider)
  420. if registered is None or not registered.streaming_capable:
  421. return None
  422. return _PluginSession(provider, registered.open_stream_session(language=language, prompt=prompt), on_partial)
  423. def streaming_available(stt_config: Optional[dict[str, Any]] = None) -> bool:
  424. """Cheap capability probe (no connection): would :func:`open_streaming_session` try a wire?"""
  425. from tools import transcription_tools as tt
  426. cfg = tt._load_stt_config() if stt_config is None else stt_config
  427. if not streaming_enabled(cfg):
  428. return False
  429. provider = tt._get_provider(cfg)
  430. if provider in STREAMING_PROVIDERS:
  431. return _builtin_session(provider, cfg, None, None, None) is not None
  432. from agent.transcription_registry import get_provider
  433. registered = get_provider(provider)
  434. return bool(registered is not None and registered.streaming_capable)
  435. def open_streaming_session(on_partial: Optional[PartialCallback] = None,
  436. stt_config: Optional[dict[str, Any]] = None) -> Optional[StreamingSession]:
  437. """Start a live STT session for the active profile's provider, or None (use the file path).
  438. None when ``stt.streaming`` is off, the provider has no live wire, or it lacks credentials.
  439. """
  440. from tools import transcription_tools as tt
  441. cfg = tt._load_stt_config() if stt_config is None else stt_config
  442. if not streaming_enabled(cfg):
  443. return None
  444. provider = tt._get_provider(cfg)
  445. if provider in ("none", "local", "local_command"):
  446. return None
  447. language = tt._resolve_stt_language(provider, cfg, extra_keys=("language_code",) if provider == "elevenlabs" else ())
  448. prompt = str(cfg.get("prompt") or "").strip() or None
  449. try:
  450. if provider in STREAMING_PROVIDERS:
  451. session = _builtin_session(provider, cfg, language, prompt, on_partial)
  452. else:
  453. session = _plugin_session(provider, language, prompt, on_partial)
  454. except Exception:
  455. logger.warning("Live STT session for %s could not open", provider, exc_info=True)
  456. return None
  457. return session.start() if session is not None else None