tts_streaming.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556
  1. """Provider-agnostic streaming TTS: sentence text → int16 mono PCM chunk iterator.
  2. ``stream_tts_to_speaker`` (``tools.tts_tool``) owns the sentence buffer, sounddevice
  3. output and stop/queue protocol; this module owns the *provider* half so playback
  4. starts on sentence one. True streamers (``StreamingTTSProvider.stream``) wrap chunked
  5. APIs; providers with no chunked API (edge, the default) get per-sentence playback via
  6. the sync ``text_to_speech_tool`` path. Adding a streamer is ``@register("name")`` on
  7. a subclass; the dispatcher, config gate (``tts.<name>.streaming``) and resolver come free.
  8. Plugin ``TTSProvider``s join by declaring ``streams_pcm`` + ``stream_sample_rate``.
  9. """
  10. from __future__ import annotations
  11. import logging
  12. import re
  13. import time
  14. from abc import ABC, abstractmethod
  15. from typing import Any, Callable, Dict, Iterator, List, Optional
  16. from agent.think_scrubber import THINK_TAG_NAMES
  17. from tools.tool_backend_helpers import resolve_openai_audio_api_key
  18. from tools.tts_tool import _get_provider, _load_tts_config
  19. from tools.tts_tool_plugins import _plugin_pcm_streaming_provider, _plugin_voice_kwargs
  20. from tools.tts_tool_providers import DEFAULT_XAI_SAMPLE_RATE
  21. logger = logging.getLogger(__name__)
  22. # Per-sentence PCM byte cap, mirroring the sync providers' 16 MiB bounded-body invariant.
  23. _STREAM_SENTENCE_BYTE_CAP = 16 * 1024 * 1024
  24. def _resolve_key(env_var: str, provider_id: str) -> str:
  25. """Provider secret lookup (config > env/.env > credential pool); seam over ``tts_tool._resolve_provider_key``.
  26. ALL streaming-provider key lookups go through here — never bare ``get_env_value``."""
  27. try:
  28. from tools.tts_tool import _resolve_provider_key
  29. return _resolve_provider_key(env_var, provider_id) or ""
  30. except Exception:
  31. from hermes_cli.config import get_env_value
  32. return get_env_value(env_var) or ""
  33. def _gemini_key() -> str:
  34. return _resolve_key("GEMINI_API_KEY", "gemini") or _resolve_key("GOOGLE_API_KEY", "gemini")
  35. # Interruption latch: a barge-in on a spoken reply marks it; the next turn's submit path takes it
  36. # and prepends SPEECH_INTERRUPTED_NOTE to the model-bound message (API-call local, never
  37. # persisted). The TTL keeps a stale barge from annotating an unrelated message minutes later.
  38. SPEECH_INTERRUPTED_NOTE = "[Note: the user interrupted your previous spoken reply before it finished.]"
  39. _INTERRUPT_TTL_S = 120.0
  40. _interrupted_at: Optional[float] = None
  41. def mark_speech_interrupted() -> None:
  42. global _interrupted_at
  43. _interrupted_at = time.monotonic()
  44. def take_speech_interrupted() -> bool:
  45. """Pop the latch; True when a barge happened within the TTL."""
  46. global _interrupted_at
  47. at, _interrupted_at = _interrupted_at, None
  48. return at is not None and time.monotonic() - at < _INTERRUPT_TTL_S
  49. # Sentence boundary: after .!? followed by whitespace, or a blank line.
  50. SENTENCE_BOUNDARY_RE = re.compile(r"(?<=[.!?])(?:\s|\n)|(?:\n\n)")
  51. # Reasoning tags come from the one canonical list (agent.think_scrubber), matched case-insensitively,
  52. # so feed() and flush() strip/cut exactly the tags every other reasoning-hiding surface does.
  53. _THINK_NAMES = "|".join(re.escape(name) for name in THINK_TAG_NAMES)
  54. _THINK_BLOCK_RE = re.compile(rf"<({_THINK_NAMES})[\s>].*?</\1>", flags=re.DOTALL | re.IGNORECASE)
  55. _THINK_OPEN_RE = re.compile(rf"<(?:{_THINK_NAMES})(?=[\s>]|$)", flags=re.IGNORECASE)
  56. class SentenceChunker:
  57. """Incremental sentence cutter for LLM token deltas, shared by the speaker pipeline and the
  58. speak-stream WebSocket so every surface cuts speech identically. Strips ``<think>`` blocks (even
  59. split across deltas) and merges fragments shorter than *min_len* into the following sentence."""
  60. def __init__(self, min_len: int = 20):
  61. self.min_len = min_len
  62. self.buf = ""
  63. @classmethod
  64. def from_config(cls, tts_config: dict) -> SentenceChunker:
  65. """Chunker honouring ``tts.streaming.min_len``. 20 suits English; a CJK opener of 5–7
  66. characters is a whole clause, so voice setups lower it to speak the first sentence
  67. alone instead of buffering it behind the second. Floor 1: 0 would emit every boundary."""
  68. try:
  69. return cls(min_len=max(1, int((tts_config.get("streaming") or {}).get("min_len", 20))))
  70. except (AttributeError, TypeError, ValueError): # non-mapping / non-numeric → default
  71. return cls()
  72. def feed(self, delta: str) -> list[str]:
  73. """Absorb *delta*; return every complete sentence now ready to speak."""
  74. self.buf = _THINK_BLOCK_RE.sub("", self.buf + delta)
  75. if _THINK_OPEN_RE.search(self.buf):
  76. return [] # open think tag — the closing tag may arrive next delta
  77. out: list[str] = []
  78. start = 0 # skip boundaries that would leave the head too short
  79. while m := SENTENCE_BOUNDARY_RE.search(self.buf, start):
  80. head = self.buf[: m.end()]
  81. if len(head.strip()) < self.min_len:
  82. start = m.end()
  83. continue
  84. out.append(head)
  85. self.buf = self.buf[m.end():]
  86. start = 0
  87. return out
  88. def flush(self) -> list[str]:
  89. """Drain the tail (end-of-text or long-idle flush)."""
  90. tail, self.buf = _THINK_BLOCK_RE.sub("", self.buf), ""
  91. if m := _THINK_OPEN_RE.search(tail):
  92. tail = tail[: m.start()] # unterminated reasoning block: never speak it
  93. tail = tail.strip()
  94. return [tail] if tail else []
  95. class StreamingTTSProvider(ABC):
  96. """Yields raw int16, little-endian, mono PCM chunks at ``sample_rate`` (built-ins: 24 kHz).
  97. ``sample_rate`` is provisional until ``stream()`` has yielded its first chunk: a provider may
  98. update the instance attribute once the endpoint's real format is known (OpenAI-compatible
  99. servers advertise it in the response headers), so consumers open their output device or WAV
  100. header after pulling the first chunk, never at construction.
  101. """
  102. sample_rate: int = 24000
  103. channels: int = 1
  104. sample_width: int = 2 # bytes/sample (int16)
  105. def __init__(self, tts_config: dict, section: dict):
  106. self.tts_config = tts_config
  107. self.section = section
  108. @staticmethod
  109. @abstractmethod
  110. def available() -> bool:
  111. """True when this provider's credentials/SDK are usable right now."""
  112. @abstractmethod
  113. def stream(self, text: str) -> Iterator[bytes]:
  114. """Yield PCM chunks for ``text``. Raise on failure (caller logs)."""
  115. _REGISTRY: dict[str, type[StreamingTTSProvider]] = {}
  116. def register(name: str) -> Callable[[type[StreamingTTSProvider]], type[StreamingTTSProvider]]:
  117. def _wrap(cls: type[StreamingTTSProvider]) -> type[StreamingTTSProvider]:
  118. _REGISTRY[name] = cls
  119. return cls
  120. return _wrap
  121. def _try_instantiate(name: str, tts_config: dict) -> Optional[StreamingTTSProvider]:
  122. """Construct the registered streamer *name* if it's usable, else None."""
  123. cls = _REGISTRY.get(name)
  124. if cls is None or not cls.available():
  125. return None
  126. try:
  127. return cls(tts_config, tts_config.get(name) or {})
  128. except Exception as exc: # pragma: no cover - defensive
  129. logger.debug("streaming provider %s init failed: %s", name, exc)
  130. return None
  131. # Fallback priority for ``tts.streaming.provider: auto`` — best chunked latency/quality
  132. # first. Deliberately hard-coded (a UX decision); edge is absent (no chunked-PCM API).
  133. _PROVIDER_PRIORITY: list[str] = ["elevenlabs", "gemini", "openai", "xai"]
  134. class _PluginPCMStreamer(StreamingTTSProvider):
  135. """A plugin ``TTSProvider`` that opted into raw PCM (``streams_pcm``; see
  136. ``tools.tts_tool_plugins._plugin_pcm_streaming_provider``) behind this ABC."""
  137. def __init__(self, provider: Any, sample_rate: int, tts_config: dict):
  138. super().__init__(tts_config, tts_config.get(provider.name) or {})
  139. self._provider, self.sample_rate = provider, sample_rate
  140. @staticmethod
  141. def available() -> bool:
  142. return True # gated at resolve time, per instance
  143. def stream(self, text: str) -> Iterator[bytes]:
  144. yield from _capped(
  145. self._provider.stream(text, format="pcm", **_plugin_voice_kwargs(self.tts_config)),
  146. f"plugin streamer {self._provider.name}")
  147. def _plugin_streamer(name: str, tts_config: dict) -> Optional[StreamingTTSProvider]:
  148. found = _plugin_pcm_streaming_provider(name, tts_config)
  149. return _PluginPCMStreamer(*found, tts_config) if found is not None else None
  150. def resolve_streaming_provider(
  151. tts_config: dict, preferred: Optional[str] = None) -> Optional[StreamingTTSProvider]:
  152. """Return a ready streamer for the *configured* provider, else ``None``.
  153. ``tts.streaming.provider`` when set: a name pins that exact streamer (``None`` if unusable);
  154. ``auto`` returns the first usable in ``_PROVIDER_PRIORITY``. Otherwise the configured TTS
  155. provider (or ``preferred``): ``None`` means "no chunked API" — the dispatcher speaks
  156. per-sentence via the sync path, preserving the user's chosen voice. We never silently swap
  157. providers just to get streaming.
  158. A plugin ``TTSProvider`` that opted into raw PCM (``streams_pcm``) streams here too, after the
  159. built-ins and under the same rules (a pinned name never substitutes another voice)."""
  160. pinned = str((tts_config.get("streaming") or {}).get("provider") or "").lower().strip()
  161. if pinned == "auto":
  162. for name in _PROVIDER_PRIORITY:
  163. if (inst := _try_instantiate(name, tts_config)) is not None:
  164. return inst
  165. return _plugin_streamer(_get_provider(tts_config), tts_config)
  166. name = pinned or (preferred or _get_provider(tts_config)).lower().strip()
  167. return _try_instantiate(name, tts_config) or _plugin_streamer(name, tts_config)
  168. def _capped(chunks: Iterator[bytes], label: str) -> Iterator[bytes]:
  169. """Pass chunks through, aborting past the per-sentence byte cap (runaway/hostile upstream)."""
  170. total = 0
  171. for chunk in chunks:
  172. total += len(chunk)
  173. if total > _STREAM_SENTENCE_BYTE_CAP:
  174. logger.warning("%s exceeded %d bytes for one sentence; truncating", label, _STREAM_SENTENCE_BYTE_CAP)
  175. return
  176. yield chunk
  177. @register("elevenlabs")
  178. class ElevenLabsStreamer(StreamingTTSProvider):
  179. """ElevenLabs chunked HTTP → pcm_24000 (the original reference path)."""
  180. @staticmethod
  181. def available() -> bool:
  182. return bool(_resolve_key("ELEVENLABS_API_KEY", "elevenlabs"))
  183. def stream(self, text: str) -> Iterator[bytes]:
  184. from tools.tts_tool import _import_elevenlabs
  185. from tools.tts_tool_providers import (
  186. DEFAULT_ELEVENLABS_STREAMING_MODEL_ID, DEFAULT_ELEVENLABS_VOICE_ID, _elevenlabs_environment_kwargs,
  187. )
  188. client = _import_elevenlabs()(
  189. api_key=_resolve_key("ELEVENLABS_API_KEY", "elevenlabs"), **_elevenlabs_environment_kwargs(self.section),
  190. )
  191. yield from client.text_to_speech.convert(
  192. text=text, voice_id=self.section.get("voice_id", DEFAULT_ELEVENLABS_VOICE_ID),
  193. model_id=self.section.get("streaming_model_id",
  194. self.section.get("model_id", DEFAULT_ELEVENLABS_STREAMING_MODEL_ID)),
  195. output_format="pcm_24000")
  196. def _openai_config_api_key() -> str:
  197. """Return ``tts.openai.api_key`` from config.yaml, or empty string."""
  198. try:
  199. return (_load_tts_config().get("openai") or {}).get("api_key") or ""
  200. except Exception:
  201. return ""
  202. def _sample_rate_from_headers(headers) -> Optional[int]:
  203. """Rate an OpenAI-compatible TTS endpoint advertises: ``X-Audio-Sample-Rate`` (the convention
  204. local servers use) or ``rate=`` in ``Content-Type`` (``audio/pcm; rate=44100``); None if absent."""
  205. if not headers:
  206. return None
  207. raw = headers.get("x-audio-sample-rate")
  208. if raw is None:
  209. m = re.search(r"(?:^|[;\s])rate\s*=\s*(\d+)", str(headers.get("content-type") or ""), re.IGNORECASE)
  210. raw = m.group(1) if m else None
  211. try:
  212. rate = int(str(raw).strip())
  213. except (TypeError, ValueError):
  214. return None
  215. return rate if rate > 0 else None
  216. @register("openai")
  217. class OpenAIStreamer(StreamingTTSProvider):
  218. """OpenAI speech with ``response_format=pcm`` (OpenAI itself: 24 kHz mono int16).
  219. Compatible servers may emit another rate: ``tts.openai.pcm_sample_rate`` sets the expected
  220. rate up front and a rate reported by the response (``X-Audio-Sample-Rate`` / Content-Type
  221. ``rate=``) overrides it before the first chunk is yielded (#76466).
  222. """
  223. def __init__(self, tts_config: dict, section: dict):
  224. super().__init__(tts_config, section)
  225. configured = section.get("pcm_sample_rate", self.sample_rate)
  226. if isinstance(configured, bool) or not isinstance(configured, (int, float, str)) \
  227. or not str(configured).strip().isdigit() or int(str(configured).strip()) <= 0:
  228. logger.warning("Invalid tts.openai.pcm_sample_rate %r; using %d Hz", configured, self.sample_rate)
  229. else:
  230. self.sample_rate = int(str(configured).strip())
  231. @staticmethod
  232. def available() -> bool:
  233. return bool(_openai_config_api_key() or resolve_openai_audio_api_key())
  234. def stream(self, text: str) -> Iterator[bytes]:
  235. from openai import OpenAI
  236. from hermes_cli.config import get_env_value
  237. client = OpenAI(
  238. api_key=(self.section.get("api_key") or resolve_openai_audio_api_key()),
  239. base_url=(self.section.get("base_url") or get_env_value("OPENAI_BASE_URL") or None))
  240. from tools.tts_tool_openai import _openai_extra_body
  241. extra = {"extra_body": body} if (body := _openai_extra_body(self.section)) else {}
  242. with client.audio.speech.with_streaming_response.create(
  243. model=self.section.get("model", "gpt-4o-mini-tts"), voice=self.section.get("voice", "alloy"),
  244. input=text, response_format="pcm", **extra,
  245. ) as response:
  246. # Runs on the first next(), before any audio is yielded, so consumers reading
  247. # ``sample_rate`` after the first chunk open their device at the endpoint's rate.
  248. rate = _sample_rate_from_headers(getattr(response, "headers", None))
  249. if rate is not None and rate != self.sample_rate:
  250. logger.info("TTS endpoint reports %d Hz PCM (expected %d Hz); honoring it", rate, self.sample_rate)
  251. self.sample_rate = rate
  252. yield from _capped(response.iter_bytes(), "OpenAI streaming TTS")
  253. @register("gemini")
  254. class GeminiStreamer(StreamingTTSProvider):
  255. """Gemini ``streamGenerateContent?alt=sse`` → SSE feed of base64 PCM chunks (24 kHz), bounded streamed body.
  256. Salvaged from PR #47588 (@Cdddo) and rebased onto the post-campaign infrastructure: credentials via the
  257. provider-secret resolver, requests (not httpx) with a bounded streamed body, and main's provider ABC.
  258. """
  259. @staticmethod
  260. def available() -> bool:
  261. return bool(_gemini_key())
  262. def stream(self, text: str) -> Iterator[bytes]:
  263. import base64
  264. import json as _json
  265. import requests
  266. from tools.tts_tool_providers import (
  267. DEFAULT_GEMINI_TTS_BASE_URL, DEFAULT_GEMINI_TTS_MODEL, DEFAULT_GEMINI_TTS_VOICE)
  268. from hermes_cli.config import get_env_value
  269. api_key = _gemini_key()
  270. model = str(self.section.get("model", DEFAULT_GEMINI_TTS_MODEL)).strip() or DEFAULT_GEMINI_TTS_MODEL
  271. voice = str(self.section.get("voice", DEFAULT_GEMINI_TTS_VOICE)).strip() or DEFAULT_GEMINI_TTS_VOICE
  272. from agent.gemini_native_adapter import normalize_gemini_base_url
  273. base_url = normalize_gemini_base_url(
  274. self.section.get("base_url") or get_env_value("GEMINI_BASE_URL") or DEFAULT_GEMINI_TTS_BASE_URL,
  275. )
  276. payload = {
  277. "contents": [{"parts": [{"text": text}]}],
  278. "generationConfig": {
  279. "responseModalities": ["AUDIO"],
  280. "speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": voice}}}}}
  281. url = f"{base_url}/models/{model}:streamGenerateContent"
  282. def _sse_chunks() -> Iterator[bytes]:
  283. with requests.post(
  284. url, params={"alt": "sse"}, headers={"x-goog-api-key": api_key},
  285. json=payload, timeout=60, stream=True,
  286. ) as response:
  287. response.raise_for_status()
  288. for line in response.iter_lines(decode_unicode=True):
  289. if not line or not line.startswith("data: "):
  290. continue
  291. try:
  292. parts = _json.loads(line[len("data: "):])["candidates"][0]["content"]["parts"]
  293. except (ValueError, KeyError, IndexError, TypeError):
  294. continue
  295. for part in parts:
  296. b64 = (part.get("inlineData") or part.get("inline_data") or {}).get("data", "")
  297. if not b64:
  298. continue
  299. try:
  300. yield base64.b64decode(b64)
  301. except (ValueError, TypeError) as exc:
  302. logger.warning("Gemini SSE: bad base64 audio: %s", exc)
  303. yield from _capped(_sse_chunks(), "Gemini streaming TTS")
  304. # Bounded like SpeakerPipeline's _CHUNK_QUEUE_MAX: the pump waits for the consumer instead of
  305. # buffering up to the full byte cap.
  306. _XAI_QUEUE_MAX = 64
  307. def _put_unless_stopped(q, item, stop, poll_s: float = 0.1) -> bool:
  308. """Put *item* on bounded *q*, giving up once *stop* is set; False when dropped."""
  309. import queue
  310. while not stop.is_set():
  311. try:
  312. q.put(item, timeout=poll_s)
  313. return True
  314. except queue.Full:
  315. continue
  316. return False
  317. @register("xai")
  318. class XAIStreamer(StreamingTTSProvider):
  319. """xAI WebSocket TTS → base64 PCM ``audio.delta`` frames (24 kHz mono int16).
  320. Salvaged from PR #47588 (@Cdddo) and rewritten against the real wire
  321. protocol: voice/language/codec/sample_rate ride in the URL query string
  322. (the server 400s the bare path at handshake), the client sends
  323. ``text.delta`` + ``text.done``, and the server streams JSON
  324. ``audio.delta`` envelopes (base64 PCM in ``delta``) until ``audio.done``.
  325. Credentials route through ``resolve_xai_http_credentials`` (XAI_API_KEY
  326. first, OAuth as fallback), same as the sync ``_generate_xai_tts`` path. The async WS
  327. loop runs on a background thread feeding a queue, so ``stream()`` yields
  328. chunks as they arrive and works from sync CLI code and from gateway
  329. adapters that already run an event loop.
  330. """
  331. sample_rate = DEFAULT_XAI_SAMPLE_RATE
  332. _RECV_TIMEOUT_S = 60 # a sentence of TTS should never gap this long
  333. @staticmethod
  334. def available() -> bool:
  335. try:
  336. from tools.xai_http import resolve_xai_http_credentials
  337. # Same ordering as the sync path: the subscription OAuth bearer
  338. # authorizes but 403s on metered TTS, so an explicit key wins (#87045).
  339. return bool(str(resolve_xai_http_credentials(prefer_api_key=True).get("api_key") or "").strip())
  340. except Exception:
  341. return False
  342. def stream(self, text: str) -> Iterator[bytes]:
  343. # The per-sentence byte cap is enforced once, pump-side (``_pump``), before enqueueing.
  344. yield from self._queued_frames(text)
  345. # -- async→sync bridge -------------------------------------------------
  346. def _queued_frames(self, text: str) -> Iterator[bytes]:
  347. """Yield PCM chunks as the WS delivers them, on a pump thread.
  348. The thread owns a fresh event loop, so ``asyncio.run`` never fights
  349. a loop the caller is already running (gateway adapters are async).
  350. Exceptions from the pump are re-raised on the consumer side so the
  351. caller's "raise on failure" contract holds.
  352. The byte budget is enforced in the pump before each enqueue (see
  353. ``_pump``); the queue is bounded and every put polls ``stop``, so a
  354. consumer that stops early makes the pump close the socket instead of
  355. blocking on (or piling PCM into) a queue nobody drains.
  356. """
  357. import asyncio
  358. import queue
  359. import threading
  360. from contextvars import copy_context
  361. q: queue.Queue[object] = queue.Queue(maxsize=_XAI_QUEUE_MAX)
  362. done = object()
  363. stop = threading.Event()
  364. def _pump_thread() -> None:
  365. try:
  366. asyncio.run(self._pump(text, q, stop))
  367. except BaseException as exc: # hand failures to the consumer
  368. _put_unless_stopped(q, exc, stop)
  369. finally:
  370. _put_unless_stopped(q, done, stop)
  371. threading.Thread(
  372. target=copy_context().run, args=(_pump_thread,), name="xai-tts-pump", daemon=True
  373. ).start()
  374. try:
  375. while True:
  376. item = q.get()
  377. if item is done:
  378. return
  379. if isinstance(item, BaseException):
  380. raise item
  381. yield item
  382. finally:
  383. # Generator closed early (cap, playback failure, caller dropped
  384. # it): stop the pump at its next frame.
  385. stop.set()
  386. async def _pump(self, text: str, q, stop) -> None:
  387. import asyncio
  388. import base64
  389. import json as _json
  390. from urllib.parse import urlencode
  391. import websockets
  392. from tools.tts_tool_providers import DEFAULT_XAI_LANGUAGE, DEFAULT_XAI_VOICE_ID
  393. from tools.xai_http import resolve_xai_http_credentials
  394. api_key = str(resolve_xai_http_credentials(prefer_api_key=True).get("api_key") or "").strip()
  395. if not api_key:
  396. raise RuntimeError("No xAI credentials for streaming TTS")
  397. voice = str(self.section.get("voice_id", DEFAULT_XAI_VOICE_ID)).strip() or DEFAULT_XAI_VOICE_ID
  398. language = str(self.section.get("language", DEFAULT_XAI_LANGUAGE)).strip() or DEFAULT_XAI_LANGUAGE
  399. base = str(
  400. self.section.get("streaming_url") or "wss://api.x.ai/v1/tts"
  401. ).strip()
  402. params = urlencode({
  403. "voice": voice,
  404. "language": language,
  405. "codec": "pcm",
  406. "sample_rate": self.sample_rate,
  407. })
  408. sep = "&" if "?" in base else "?"
  409. ws_url = f"{base}{sep}{params}"
  410. async with websockets.connect(
  411. ws_url, additional_headers={"Authorization": f"Bearer {api_key}"}
  412. ) as ws:
  413. await ws.send(_json.dumps({"type": "text.delta", "delta": text}))
  414. await ws.send(_json.dumps({"type": "text.done"}))
  415. enqueued = 0
  416. while True:
  417. try:
  418. message = await asyncio.wait_for(
  419. ws.recv(), timeout=self._RECV_TIMEOUT_S
  420. )
  421. except TimeoutError:
  422. raise RuntimeError(f"xAI streaming TTS: no audio for {self._RECV_TIMEOUT_S}s")
  423. except websockets.exceptions.ConnectionClosedOK:
  424. return # clean close, with or without audio.done
  425. except websockets.exceptions.ConnectionClosedError as exc:
  426. raise RuntimeError(f"xAI WS closed mid-stream: {exc}") from exc
  427. if isinstance(message, (bytes, bytearray, memoryview)):
  428. continue # the server speaks JSON; binary is not expected
  429. try:
  430. envelope = _json.loads(message)
  431. except (ValueError, TypeError):
  432. continue
  433. msg_type = envelope.get("type")
  434. if msg_type == "audio.delta":
  435. b64 = envelope.get("delta") or ""
  436. if b64:
  437. pcm = base64.b64decode(b64)
  438. enqueued += len(pcm)
  439. if enqueued > _STREAM_SENTENCE_BYTE_CAP:
  440. # The single per-sentence byte-budget check,
  441. # BEFORE enqueueing, so a runaway upstream
  442. # never piles decoded PCM into the queue.
  443. # Returning exits the context manager, which
  444. # closes the socket — we stop reading upstream.
  445. logger.warning(
  446. "xAI streaming TTS exceeded %d bytes for one "
  447. "sentence; closing upstream",
  448. _STREAM_SENTENCE_BYTE_CAP,
  449. )
  450. return
  451. if not _put_unless_stopped(q, pcm, stop):
  452. # Consumer went away (playback failure, caller
  453. # dropped it): close and stop reading rather
  454. # than blocking on a queue nobody drains.
  455. return
  456. elif msg_type == "audio.done":
  457. return
  458. elif msg_type == "error":
  459. detail = (
  460. envelope.get("message") or envelope.get("error") or envelope
  461. )
  462. raise RuntimeError(f"xAI streaming TTS error: {detail}")