"""Live speech-to-text: feed microphone PCM while the user speaks, get partial and final text. Every caller hands 16 kHz mono s16le PCM to :meth:`StreamingSession.push_audio` in any chunk size, receives non-final text through ``on_partial`` as the provider recognizes it, and calls :meth:`StreamingSession.finalize` at end of speech for the standard transcription envelope (``success`` / ``transcript`` / ``provider`` / ``error``). One worker thread owns the provider socket, so callers never block on network I/O while audio is captured. Built-in wires (live-verified against each vendor except where noted): - ``openai``: Realtime ``type: transcription`` session (``stt.openai.streaming_model``, default ``gpt-live-transcribe``, the one model that emits deltas before the turn is committed). The endpoint requires >= 24 kHz input, so PCM is resampled 16 k -> 24 k here. - ``xai``: ``wss://api.x.ai/v1/stt`` with ``interim_results=true``. - ``elevenlabs``: ``wss://api.elevenlabs.io/v1/speech-to-text/realtime`` (``scribe_v2_realtime``), built from the vendor AsyncAPI schema. Plugin providers opt in through ``TranscriptionProvider.streaming_capable`` / ``open_stream_session``. Streaming is opt-in (``stt.streaming: true``); every surface falls back to the file path when no session opens or a session fails. """ from __future__ import annotations import base64 import contextvars import json import logging import queue import threading import time from array import array from typing import Any, Callable, Dict, Optional from urllib.parse import urlencode, urlsplit, urlunsplit from agent.memory_provider import spawn_context_thread from utils import is_truthy_value logger = logging.getLogger(__name__) INPUT_RATE = 16000 # the contract every caller feeds DEFAULT_OPENAI_STREAMING_MODEL = "gpt-live-transcribe" ELEVENLABS_REALTIME_MODEL = "scribe_v2_realtime" STREAMING_PROVIDERS = ("openai", "xai", "elevenlabs") _FINALIZE_TIMEOUT_S = 20.0 _CONNECT_TIMEOUT_S = 10.0 PartialCallback = Callable[[str], None] class Resampler: """Stateful s16le mono rate converter (linear; box-average for integer downsampling).""" def __init__(self, src_rate: int, dst_rate: int) -> None: self.src, self.dst = int(src_rate), int(dst_rate) self._pos = 0.0 # fractional read position into the carried + new samples self._carry = array("h") def __call__(self, pcm: bytes) -> bytes: if self.src == self.dst or not pcm: return pcm samples = array("h") samples.frombytes(pcm[: len(pcm) - len(pcm) % 2]) if self.src % self.dst == 0: data = self._carry + samples step = self.src // self.dst usable = len(data) - len(data) % step out = array("h", (sum(data[i:i + step]) // step for i in range(0, usable, step))) self._carry = data[usable:] return out.tobytes() data = self._carry + samples ratio = self.src / self.dst out = array("h") pos = self._pos last = len(data) - 1 while pos < last: i = int(pos) frac = pos - i out.append(int(data[i] + (data[i + 1] - data[i]) * frac)) pos += ratio keep = int(pos) self._carry = data[keep:] self._pos = pos - keep return out.tobytes() class StreamingSession: """One utterance of live STT. Thread-safe ``push_audio``; single-use.""" provider = "unknown" def __init__(self, on_partial: Optional[PartialCallback] = None) -> None: self._on_partial = on_partial self._audio: queue.Queue[Optional[bytes]] = queue.Queue() self._done = threading.Event() self._cancelled = threading.Event() self._result: dict[str, Any] = {} self._partial = "" self._worker: Optional[threading.Thread] = None self._to_16k: Optional[Resampler] = None def set_input_rate(self, rate: int) -> None: """Accept PCM at ``rate`` (e.g. the mic's device rate); the worker converts it to 16 kHz, so a realtime capture callback never pays for resampling.""" if int(rate) != INPUT_RATE: self._to_16k = Resampler(int(rate), INPUT_RATE) def _take_audio(self, block: bool, timeout: float = 0.0) -> Optional[bytes]: """Next 16 kHz chunk (``b""`` when none is queued, None at end of audio).""" try: chunk = self._audio.get(timeout=timeout) if block else self._audio.get_nowait() except queue.Empty: return b"" if chunk and self._to_16k is not None: chunk = self._to_16k(chunk) return chunk # ── caller API ── def start(self) -> StreamingSession: # copy_context: a plugin session reads secrets from the opener's profile scope. ctx = contextvars.copy_context() self._worker = threading.Thread(target=ctx.run, args=(self._run,), name=f"stt-stream-{self.provider}", daemon=True) self._worker.start() return self def push_audio(self, pcm16k: bytes) -> None: if pcm16k and not self._done.is_set(): self._audio.put(bytes(pcm16k)) def end_audio(self) -> None: """Signal end of speech without waiting (the worker flushes the provider).""" self._audio.put(None) def finalize(self, timeout: float = _FINALIZE_TIMEOUT_S) -> dict[str, Any]: self.end_audio() if not self._done.wait(timeout): self.cancel() return self._error("live transcription timed out") return dict(self._result) def cancel(self) -> None: self._cancelled.set() self._audio.put(None) self._finish(self._error("cancelled")) def partial_transcript(self) -> str: return self._partial # ── worker side ── def _set_partial(self, text: str) -> None: text = text.strip() if text and text != self._partial: self._partial = text if self._on_partial is not None: try: self._on_partial(text) except Exception: logger.debug("on_partial callback raised", exc_info=True) def _error(self, message: str) -> dict[str, Any]: return {"success": False, "transcript": "", "provider": self.provider, "error": message} def _finish(self, result: dict[str, Any]) -> None: if not self._done.is_set(): self._result = result self._done.set() def _run(self) -> None: try: self._finish(self._session()) except Exception as exc: # health: allow BLE001 -- worker boundary: every failure becomes the envelope logger.warning("Live STT (%s) failed: %s", self.provider, exc, exc_info=True) self._finish(self._error(f"live transcription failed: {exc}")) def _session(self) -> dict[str, Any]: # pragma: no cover - overridden raise NotImplementedError class _WebSocketSession(StreamingSession): """A provider wire over one websocket: subclasses map audio out and events in.""" url = "" headers: dict[str, str] = {} def _session(self) -> dict[str, Any]: from websockets.sync.client import connect ws = connect(self.url, additional_headers=self.headers, open_timeout=_CONNECT_TIMEOUT_S, close_timeout=1, max_size=None) try: return self._loop(ws) finally: # Publish first, close after: the sync close handshake waits up to close_timeout on a # server that already hung up, and the caller is blocked on finalize() meanwhile. spawn_context_thread(ws.close, name="stt-stream-close").start() def _loop(self, ws: Any) -> dict[str, Any]: self._open(ws) ending_since: Optional[float] = None while True: if self._cancelled.is_set(): return self._error("cancelled") if ending_since is None: ending_since = self._drain_audio(ws) elif time.monotonic() - ending_since > _FINALIZE_TIMEOUT_S: return self._error("provider never finalized the transcript") try: raw = ws.recv(timeout=0.02) except TimeoutError: continue if isinstance(raw, (bytes, bytearray)): continue result = self._event(json.loads(raw)) if result is not None: self._finish(result) return result if self._backlog and self._ready(): self._flush_backlog(ws) def _drain_audio(self, ws: Any) -> Optional[float]: """Send queued audio; on end-of-audio send the provider's flush and return its time.""" while True: chunk = self._take_audio(block=False) if chunk is None: self._end(ws) return time.monotonic() if not chunk: if self._audio.empty(): return None continue if self._ready(): self._send_audio(ws, chunk) else: self._backlog.append(chunk) _backlog: list[bytes] def _ready(self) -> bool: return True def _flush_backlog(self, ws: Any) -> None: backlog, self._backlog = self._backlog, [] for chunk in backlog: self._send_audio(ws, chunk) def _open(self, ws: Any) -> None: self._backlog = [] def _send_audio(self, ws: Any, chunk: bytes) -> None: # pragma: no cover - overridden raise NotImplementedError def _end(self, ws: Any) -> None: # pragma: no cover - overridden raise NotImplementedError def _event(self, event: dict[str, Any]) -> Optional[dict[str, Any]]: # pragma: no cover raise NotImplementedError class OpenAIRealtimeSession(_WebSocketSession): provider = "openai" def __init__(self, api_key: str, base_url: str, model: str, language: Optional[str], prompt: Optional[str], on_partial: Optional[PartialCallback] = None) -> None: super().__init__(on_partial) parts = urlsplit(base_url.rstrip("/")) scheme = "wss" if parts.scheme == "https" else "ws" self.url = urlunsplit((scheme, parts.netloc, f"{parts.path}/realtime", "intent=transcription", "")) self.headers = {"Authorization": f"Bearer {api_key}"} self.model, self.language, self.prompt = model, language, prompt self._resample = Resampler(INPUT_RATE, 24000) self._text = "" self._sent_audio = False def _open(self, ws: Any) -> None: super()._open(ws) transcription: dict[str, Any] = {"model": self.model} if self.language: # gpt-live-transcribe takes a ``languages`` list and rejects the singular field. if self.model == DEFAULT_OPENAI_STREAMING_MODEL: transcription["languages"] = [self.language] else: transcription["language"] = self.language if self.prompt: transcription["prompt"] = self.prompt ws.send(json.dumps({"type": "session.update", "session": {"type": "transcription", "audio": {"input": { "format": {"type": "audio/pcm", "rate": 24000}, "transcription": transcription, "turn_detection": None}}}})) def _send_audio(self, ws: Any, chunk: bytes) -> None: pcm = self._resample(chunk) if pcm: self._sent_audio = True ws.send(json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(pcm).decode()})) def _end(self, ws: Any) -> None: if not self._sent_audio: self._finish({"success": True, "transcript": "", "provider": self.provider}) self._cancelled.set() return ws.send(json.dumps({"type": "input_audio_buffer.commit"})) def _event(self, event: dict[str, Any]) -> Optional[dict[str, Any]]: kind = str(event.get("type") or "") if kind.endswith("input_audio_transcription.delta"): self._text += str(event.get("delta") or "") self._set_partial(self._text) elif kind.endswith("input_audio_transcription.completed"): return {"success": True, "transcript": str(event.get("transcript") or self._text).strip(), "provider": self.provider} elif kind == "error": message = str((event.get("error") or {}).get("message") or "provider error") # Committing < 100 ms of audio is "buffer too small": the user said nothing. if "buffer too small" in message.lower(): return {"success": True, "transcript": "", "provider": self.provider} return self._error(message) return None class XAIStreamingSession(_WebSocketSession): provider = "xai" def __init__(self, api_key: str, base_url: str, model: str, language: Optional[str], on_partial: Optional[PartialCallback] = None) -> None: super().__init__(on_partial) parts = urlsplit(base_url.rstrip("/")) scheme = "wss" if parts.scheme == "https" else "ws" query = {"sample_rate": INPUT_RATE, "encoding": "pcm", "interim_results": "true", "model": model} if language: query["language"] = language self.url = urlunsplit((scheme, parts.netloc, f"{parts.path}/stt", urlencode(query), "")) self.headers = {"Authorization": f"Bearer {api_key}"} self._created = False self._utterances: list[str] = [] # speech_final stitched utterances self._chunks: list[str] = [] # is_final chunks of the current utterance self._interim = "" def _ready(self) -> bool: return self._created # the server drops audio sent before transcript.created def _send_audio(self, ws: Any, chunk: bytes) -> None: ws.send(chunk) def _end(self, ws: Any) -> None: if not self._created: self._backlog.clear() ws.send(json.dumps({"type": "audio.done"})) def _text(self) -> str: return " ".join(p for p in (*self._utterances, *self._chunks, self._interim) if p).strip() def _event(self, event: dict[str, Any]) -> Optional[dict[str, Any]]: kind = event.get("type") if kind == "transcript.created": self._created = True return None if kind == "transcript.partial": text = str(event.get("text") or "").strip() if event.get("speech_final"): self._utterances.append(text) self._chunks, self._interim = [], "" elif event.get("is_final"): self._chunks.append(text) self._interim = "" else: self._interim = text self._set_partial(self._text()) return None if kind == "transcript.done": return {"success": True, "transcript": self._text(), "provider": self.provider} if kind == "error": return self._error(str(event.get("message") or "provider error")) return None class ElevenLabsRealtimeSession(_WebSocketSession): provider = "elevenlabs" def __init__(self, api_key: str, base_url: str, language: Optional[str], on_partial: Optional[PartialCallback] = None) -> None: super().__init__(on_partial) parts = urlsplit(base_url.rstrip("/")) scheme = "wss" if parts.scheme == "https" else "ws" query = {"model_id": ELEVENLABS_REALTIME_MODEL, "audio_format": f"pcm_{INPUT_RATE}", "commit_strategy": "vad"} if language: query["language_code"] = language self.url = urlunsplit((scheme, parts.netloc, f"{parts.path}/speech-to-text/realtime", urlencode(query), "")) self.headers = {"xi-api-key": api_key} self._committed: list[str] = [] self._interim = "" self._ending = False def _chunk(self, pcm: bytes, commit: bool) -> str: return json.dumps({"message_type": "input_audio_chunk", "audio_base_64": base64.b64encode(pcm).decode(), "commit": commit, "sample_rate": INPUT_RATE}) def _send_audio(self, ws: Any, chunk: bytes) -> None: ws.send(self._chunk(chunk, False)) def _end(self, ws: Any) -> None: self._ending = True ws.send(self._chunk(b"", True)) def _text(self) -> str: return " ".join(p for p in (*self._committed, self._interim) if p).strip() def _event(self, event: dict[str, Any]) -> Optional[dict[str, Any]]: kind = event.get("message_type") if kind == "partial_transcript": self._interim = str(event.get("text") or "").strip() self._set_partial(self._text()) elif kind == "committed_transcript": text = str(event.get("text") or "").strip() if text: self._committed.append(text) self._interim = "" self._set_partial(self._text()) if self._ending: return {"success": True, "transcript": self._text(), "provider": self.provider} elif kind in ("insufficient_audio_activity", "commit_throttled") and self._ending: return {"success": True, "transcript": self._text(), "provider": self.provider} elif kind and kind not in ("session_started", "committed_transcript_with_timestamps", "warning"): if "error" in event: return self._error(f"{kind}: {event.get('error')}") return None class _PluginSession(StreamingSession): """Adapts a plugin ``TranscriptionStreamSession`` to the caller API (partials polled).""" def __init__(self, provider_name: str, inner: Any, on_partial: Optional[PartialCallback]) -> None: super().__init__(on_partial) self.provider, self._inner = provider_name, inner def _session(self) -> dict[str, Any]: while True: chunk = self._take_audio(block=True, timeout=0.25) if self._cancelled.is_set(): return self._error("cancelled") if chunk is None: return dict(self._inner.finalize()) if chunk: self._inner.push_audio(chunk) self._set_partial(str(self._inner.partial_transcript() or "")) # ── resolution ── def streaming_enabled(stt_config: dict[str, Any]) -> bool: return is_truthy_value(stt_config.get("streaming", False), default=False) def _openai_credentials() -> Optional[tuple[str, str]]: """Direct OpenAI credentials only: the managed Nous gateway proxies the REST audio API, not the Realtime socket.""" from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_selection from tools.transcription_tools import _resolve_openai_audio_client_config if read_selection("stt") == NOUS_MANAGED_PROVIDER: return None try: api_key, base_url = _resolve_openai_audio_client_config() except ValueError: return None return api_key, base_url def _builtin_session(provider: str, stt_config: dict[str, Any], language: Optional[str], prompt: Optional[str], on_partial: Optional[PartialCallback]) -> Optional[StreamingSession]: from tools import transcription_common as tc from tools import transcription_tools as tt raw_section = stt_config.get(provider) section: dict[str, Any] = raw_section if isinstance(raw_section, dict) else {} if provider == "openai": creds = _openai_credentials() if creds is None: return None model = str(section.get("streaming_model") or DEFAULT_OPENAI_STREAMING_MODEL) return OpenAIRealtimeSession(creds[0], creds[1], model, language, prompt, on_partial) if provider == "xai": from hermes_cli.config import get_env_value api_key = str(get_env_value("XAI_API_KEY") or "").strip() # API key only, like the Desktop wire if not api_key: return None base = str(section.get("base_url") or get_env_value("XAI_STT_BASE_URL") or tc.XAI_STT_BASE_URL) return XAIStreamingSession(api_key, base, tc.normalize_xai_stt_model(section.get("model")), language, on_partial) if provider == "elevenlabs": api_key = tt._resolve_provider_key("ELEVENLABS_API_KEY", "elevenlabs") if not api_key: return None base = str(section.get("base_url") or tc.ELEVENLABS_STT_BASE_URL) return ElevenLabsRealtimeSession(api_key, base, language, on_partial) return None def _plugin_session(provider: str, language: Optional[str], prompt: Optional[str], on_partial: Optional[PartialCallback]) -> Optional[StreamingSession]: from agent.transcription_registry import get_provider registered = get_provider(provider) if registered is None or not registered.streaming_capable: return None return _PluginSession(provider, registered.open_stream_session(language=language, prompt=prompt), on_partial) def streaming_available(stt_config: Optional[dict[str, Any]] = None) -> bool: """Cheap capability probe (no connection): would :func:`open_streaming_session` try a wire?""" from tools import transcription_tools as tt cfg = tt._load_stt_config() if stt_config is None else stt_config if not streaming_enabled(cfg): return False provider = tt._get_provider(cfg) if provider in STREAMING_PROVIDERS: return _builtin_session(provider, cfg, None, None, None) is not None from agent.transcription_registry import get_provider registered = get_provider(provider) return bool(registered is not None and registered.streaming_capable) def open_streaming_session(on_partial: Optional[PartialCallback] = None, stt_config: Optional[dict[str, Any]] = None) -> Optional[StreamingSession]: """Start a live STT session for the active profile's provider, or None (use the file path). None when ``stt.streaming`` is off, the provider has no live wire, or it lacks credentials. """ from tools import transcription_tools as tt cfg = tt._load_stt_config() if stt_config is None else stt_config if not streaming_enabled(cfg): return None provider = tt._get_provider(cfg) if provider in ("none", "local", "local_command"): return None language = tt._resolve_stt_language(provider, cfg, extra_keys=("language_code",) if provider == "elevenlabs" else ()) prompt = str(cfg.get("prompt") or "").strip() or None try: if provider in STREAMING_PROVIDERS: session = _builtin_session(provider, cfg, language, prompt, on_partial) else: session = _plugin_session(provider, language, prompt, on_partial) except Exception: logger.warning("Live STT session for %s could not open", provider, exc_info=True) return None return session.start() if session is not None else None