tts_tool_speaker.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. """Speaker-side streaming pipeline: ``stream_tts_to_speaker``.
  2. Turns a queue of LLM text deltas into audio the moment each sentence is complete. Two paths
  3. share the sentence cutter (``tools.tts_streaming``): :class:`_StreamerPlayback` for a registered
  4. chunked streamer (prefetch thread per sentence, one FIFO playback worker through a sounddevice
  5. OutputStream or temp WAV + system player) and :class:`_SyncSentencePipeline` for every other
  6. provider (per-sentence ``text_to_speech_tool`` on a single-thread executor, overlapped with
  7. playback). Origin seams are resolved through :func:`_origin` at call time.
  8. """
  9. from __future__ import annotations
  10. import contextlib
  11. import itertools
  12. import json
  13. import logging
  14. import os
  15. import platform
  16. import queue
  17. import tempfile
  18. import threading
  19. from concurrent.futures import Future, ThreadPoolExecutor
  20. from typing import Callable, Iterable, Iterator, List, Optional
  21. from tools.tts_text_normalize import _strip_markdown_for_tts
  22. from tools.tts_tool_delivery import _origin, _remove_quietly as _unlink_quietly
  23. logger = logging.getLogger("tools.tts_tool")
  24. def _align_int16_chunks(chunks: Iterable[bytes], stop_evt: threading.Event, *, pad_tail: bool = True) -> Iterator[bytes]:
  25. """Yield int16-aligned byte chunks; a dangling odd byte is padded at the end (or dropped)."""
  26. leftover = b""
  27. for chunk in chunks:
  28. if stop_evt.is_set():
  29. break
  30. buf = leftover + chunk
  31. aligned_len = len(buf) - (len(buf) % 2)
  32. if aligned_len >= 2:
  33. yield buf[:aligned_len]
  34. leftover = buf[aligned_len:]
  35. if leftover and pad_tail:
  36. yield b"\x00"
  37. def _play_via_tempfile(audio_iter: Iterable[bytes], stop_evt: threading.Event, sample_rate: int = 24000) -> None:
  38. """Write PCM chunks to a temp WAV file and play it with the system player."""
  39. tmp = tmp_path = None
  40. try:
  41. import wave
  42. tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
  43. tmp_path = tmp.name
  44. with wave.open(tmp, "wb") as wf:
  45. wf.setnchannels(1)
  46. wf.setsampwidth(2) # 16-bit
  47. wf.setframerate(sample_rate)
  48. for aligned in _align_int16_chunks(audio_iter, stop_evt):
  49. wf.writeframes(aligned)
  50. # wave.open() on a file object does NOT close it; on Windows the open write
  51. # handle blocks the player and the unlink below (WinError 32).
  52. tmp.close()
  53. from tools.voice_mode import play_audio_file
  54. play_audio_file(tmp_path)
  55. except Exception as exc:
  56. logger.warning("Temp-file TTS fallback failed: %s", exc)
  57. finally:
  58. if tmp is not None:
  59. with contextlib.suppress(Exception):
  60. tmp.close() # idempotent; ensures close on early error
  61. _unlink_quietly(tmp_path)
  62. def _drain_chunks(chunk_queue: queue.Queue[Optional[bytes]]) -> list[bytes]:
  63. """Collect one sentence's PCM chunks up to the ``None`` sentinel."""
  64. return list(iter(chunk_queue.get, None))
  65. def _first_written_artifact(raw: object, requested: str) -> str:
  66. """The audio path the TTS tool actually wrote, falling back to the requested one.
  67. ``text_to_speech_tool`` reports its artifacts (``file_path``/``file_paths``) in a JSON
  68. envelope, and they often land beside — not at — the requested path: a command provider's
  69. declared ``format`` rewrites the suffix, and voice-compatible delivery ffmpeg-converts to
  70. ``.ogg``. Gating playback on the requested path alone drops every such sentence silently,
  71. so prefer the first reported artifact that actually exists and is non-empty."""
  72. candidates: list[str] = []
  73. try:
  74. payload = json.loads(raw) if isinstance(raw, str) else (raw or {})
  75. if isinstance(payload, dict):
  76. reported = payload.get("file_paths") or ([payload["file_path"]] if payload.get("file_path") else [])
  77. if isinstance(reported, list):
  78. candidates = [p for p in reported if isinstance(p, str)]
  79. except (ValueError, TypeError):
  80. pass
  81. for path in candidates:
  82. if os.path.isfile(path) and os.path.getsize(path) > 0:
  83. return path
  84. return requested
  85. class _SyncSentencePipeline:
  86. """Overlap per-sentence synthesis with playback for non-streaming providers.
  87. One single-thread synthesis executor (FIFO; providers never see concurrent calls) feeds one
  88. playback worker through a small bounded queue, so sentence n+1 synthesizes while n plays;
  89. the bound keeps lookahead/temp files small and gives the caller backpressure.
  90. ``text_to_speech_tool`` / ``play_audio_file`` are resolved late so test patches apply."""
  91. def __init__(self, stop_event: threading.Event, *, lookahead: int = 2):
  92. self._stop = stop_event
  93. self._queue: queue.Queue[Optional[tuple[str, Future]]] = queue.Queue(maxsize=max(1, lookahead))
  94. self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="tts-sync-synth")
  95. self._player = threading.Thread(target=self._drain, name="tts-sync-play", daemon=True)
  96. self._player.start()
  97. def speak(self, cleaned: str) -> None:
  98. """Queue one sentence. Blocks only when the lookahead bound is full."""
  99. if not self._stop.is_set():
  100. self._queue.put((cleaned, self._executor.submit(self._synthesize_to_tmp, cleaned)))
  101. def close(self) -> None:
  102. """Flush queued sentences in order (skipped if stopped), then join."""
  103. self._queue.put(None)
  104. self._player.join()
  105. self._executor.shutdown(wait=True)
  106. def _synthesize_to_tmp(self, cleaned: str) -> Optional[str]:
  107. if self._stop.is_set():
  108. return None
  109. tmp_path = None
  110. try:
  111. fd, tmp_path = tempfile.mkstemp(suffix=".mp3")
  112. os.close(fd)
  113. raw = _origin().text_to_speech_tool(text=cleaned, output_path=tmp_path)
  114. written = _first_written_artifact(raw, tmp_path)
  115. if os.path.abspath(written) != os.path.abspath(tmp_path):
  116. _unlink_quietly(tmp_path) # provider wrote elsewhere: the placeholder is empty
  117. return written
  118. except Exception as exc:
  119. logger.warning("Sync per-sentence TTS synthesis failed: %s", exc)
  120. _unlink_quietly(tmp_path)
  121. return None
  122. def _drain(self) -> None:
  123. for _sentence, future in iter(self._queue.get, None):
  124. tmp_path = None
  125. try:
  126. tmp_path = future.result()
  127. if tmp_path and not self._stop.is_set() and os.path.isfile(tmp_path) and os.path.getsize(tmp_path) > 0:
  128. from tools.voice_mode import play_audio_file
  129. play_audio_file(tmp_path)
  130. except Exception as exc:
  131. logger.warning("Sync per-sentence TTS failed: %s", exc)
  132. finally:
  133. _unlink_quietly(tmp_path)
  134. class _StreamerPlayback:
  135. """Prefetch + FIFO playback for a chunked :class:`StreamingTTSProvider`.
  136. ``speak(text)`` starts ``streamer.stream()`` immediately on a prefetch thread (at most 3 in
  137. flight) buffering into a bounded per-sentence queue; one playback worker drains those in order,
  138. so sentence N+1 arrives while N plays. Output is a PortAudio stream when one opened, else temp
  139. WAV files; a failing write is retried on a reinitialized stream up to ``_MAX_REINIT`` times."""
  140. _MAX_REINIT = 3
  141. _CHUNK_QUEUE_MAX = 64
  142. def __init__(self, streamer, stop_event: threading.Event):
  143. self.streamer, self.stop_event = streamer, stop_event
  144. # The device is opened lazily, once the first sentence's first chunk has arrived: an
  145. # OpenAI-compatible endpoint reports its real PCM rate in the response headers, so
  146. # ``streamer.sample_rate`` is only trustworthy after the request answered (#76466).
  147. self.output_stream = None
  148. self._use_device = self._device_usable()
  149. self._audio_queue: queue.Queue[Optional[queue.Queue[Optional[bytes]]]] = queue.Queue()
  150. self._prefetch_threads: list[threading.Thread] = []
  151. self._prefetch_sem = threading.Semaphore(3)
  152. self._worker = threading.Thread(target=self._playback_worker, daemon=True)
  153. self._worker.start()
  154. def _create_output_stream(self):
  155. sd = _origin()._import_sounddevice()
  156. stream = sd.OutputStream(
  157. samplerate=self.streamer.sample_rate, channels=self.streamer.channels, dtype="int16")
  158. stream.start()
  159. return stream
  160. def _device_usable(self) -> bool:
  161. # macOS skips sounddevice entirely: PortAudio/CoreAudio init triggers a
  162. # kTCCServiceMediaLibrary prompt though output needs no media-library access.
  163. # False routes every sentence through tempfile -> afplay. See PR #62601 / #13291.
  164. if platform.system() == "Darwin":
  165. return False
  166. try:
  167. _origin()._import_sounddevice()
  168. except (ImportError, OSError) as exc:
  169. logger.debug("sounddevice not available, streamer→tempfile: %s", exc)
  170. return False
  171. return True
  172. def _ensure_output_stream(self) -> bool:
  173. """Open PortAudio at the streamer's *current* rate, reopening it when the rate changed
  174. (a different endpoint answered); False routes the sentence through a temp WAV."""
  175. rate = int(self.streamer.sample_rate)
  176. if self._current_stream is not None and self._current_rate == rate:
  177. return True
  178. if self._current_stream is None and self._reinit_count >= self._MAX_REINIT:
  179. return False
  180. self.close_output_stream()
  181. try:
  182. self.output_stream = self._create_output_stream()
  183. except Exception as exc:
  184. logger.warning("sounddevice OutputStream failed: %s", exc)
  185. self.output_stream, self._reinit_count = None, self._MAX_REINIT # don't retry per sentence
  186. self._current_stream, self._current_rate = self.output_stream, rate
  187. return self._current_stream is not None
  188. def close_output_stream(self) -> None:
  189. """Always release the device so a later stream can open it."""
  190. if self.output_stream is not None:
  191. with contextlib.suppress(Exception):
  192. self.output_stream.stop()
  193. self.output_stream.close()
  194. def speak(self, text: str) -> None:
  195. """Start ``streamer.stream(text)`` and prefetch its chunks immediately."""
  196. try:
  197. audio_iter = self.streamer.stream(text)
  198. except Exception as exc:
  199. logger.warning("Streaming TTS synthesis failed: %s", exc)
  200. return
  201. self._prefetch_sem.acquire()
  202. chunk_queue: queue.Queue[Optional[bytes]] = queue.Queue(maxsize=self._CHUNK_QUEUE_MAX)
  203. self._audio_queue.put(chunk_queue)
  204. self._prefetch_threads.append(threading.Thread(
  205. target=self._consume_to_queue, args=(audio_iter, chunk_queue), daemon=True))
  206. self._prefetch_threads[-1].start()
  207. def _consume_to_queue(self, audio_iter: Iterator[bytes], chunk_queue: queue.Queue[Optional[bytes]]) -> None:
  208. try:
  209. for chunk in audio_iter:
  210. if self.stop_event.is_set():
  211. logger.info("TTS CUT: prefetch cancelled (stop_event set mid-sentence) — partial audio only")
  212. break
  213. chunk_queue.put(chunk, timeout=30.0)
  214. except Exception as exc:
  215. logger.warning("TTS CUT: streaming TTS prefetch failed mid-sentence (partial audio only): %s", exc)
  216. finally:
  217. chunk_queue.put(None) # sentinel: no more chunks
  218. self._prefetch_sem.release()
  219. def _play_sentence_via_tempfile(self, chunk_queue) -> None:
  220. chunks = _drain_chunks(chunk_queue) # drained first: the rate is final once chunks exist
  221. _play_via_tempfile(chunks, self.stop_event, self.streamer.sample_rate)
  222. def _for_each_sentence(self, play: Callable[[queue.Queue], None]) -> None:
  223. """Feed queued sentences to *play* in order until the end sentinel; stopped sentences are skipped."""
  224. for chunk_queue in iter(self._audio_queue.get, None):
  225. if not self.stop_event.is_set():
  226. play(chunk_queue)
  227. def _write_pcm(self, buf: bytes) -> None:
  228. self._current_stream.write(self._np.frombuffer(buf, dtype="<i2").reshape(-1, 1))
  229. def _recover_stream(self) -> bool:
  230. """Close the broken PortAudio stream and open a fresh one after a failed write; False once
  231. ``_MAX_REINIT`` is exhausted (remaining sentences go through temp files)."""
  232. if self._reinit_count >= self._MAX_REINIT:
  233. logger.warning(
  234. "TTS: PortAudio reinit exhausted after %d attempts, falling back to tempfile for remaining sentences",
  235. self._MAX_REINIT)
  236. self._current_stream = None
  237. return False
  238. self._reinit_count += 1
  239. self.close_output_stream()
  240. try:
  241. self.output_stream = self._create_output_stream()
  242. logger.info("TTS: PortAudio output stream reinitialized after error")
  243. except Exception as exc:
  244. logger.warning("TTS: PortAudio stream reinit failed: %s", exc)
  245. self.output_stream = None
  246. self._current_stream = self.output_stream
  247. return self._current_stream is not None
  248. def _play_sentence_via_stream(self, chunk_queue) -> None:
  249. """Write one sentence's PCM to PortAudio; after an unrecoverable write failure the rest is dropped."""
  250. chunks = iter(chunk_queue.get, None)
  251. first = next(chunks, None) # blocks until the endpoint answered: the rate is final now
  252. if first is None:
  253. return
  254. chunks = itertools.chain([first], chunks)
  255. if not self._ensure_output_stream():
  256. _play_via_tempfile(list(chunks), self.stop_event, self.streamer.sample_rate)
  257. return
  258. for aligned in _align_int16_chunks(chunks, self.stop_event, pad_tail=False):
  259. try:
  260. self._write_pcm(aligned)
  261. except Exception as write_exc:
  262. logger.warning("PortAudio write failed, attempting stream reinit: %s", write_exc)
  263. if not self._recover_stream():
  264. return
  265. with contextlib.suppress(Exception):
  266. self._write_pcm(aligned)
  267. def _playback_worker(self) -> None:
  268. """Single consumer: play audio segments from the queue in order."""
  269. if not self._use_device:
  270. self._for_each_sentence(self._play_sentence_via_tempfile)
  271. return
  272. import numpy as _np
  273. try:
  274. from tools.voice_mode import mark_audio_output_active
  275. except Exception:
  276. mark_audio_output_active = lambda _active: None
  277. self._np, self._reinit_count, self._current_stream, self._current_rate = _np, 0, None, None
  278. mark_audio_output_active(True)
  279. try:
  280. self._for_each_sentence(self._play_sentence_via_stream)
  281. finally:
  282. mark_audio_output_active(False)
  283. def finish(self) -> None:
  284. """Send the end sentinel, then wait for playback and prefetch threads."""
  285. self._audio_queue.put(None)
  286. self._worker.join(timeout=300.0)
  287. for t in self._prefetch_threads:
  288. t.join(timeout=10.0)
  289. self.close_output_stream()
  290. def stream_tts_to_speaker(
  291. text_queue: queue.Queue, stop_event: threading.Event, tts_done_event: threading.Event,
  292. display_callback: Optional[Callable[[str], None]] = None, provider: Optional[str] = None):
  293. """Consume text deltas from *text_queue*, cut into sentences, speak each the moment it's ready.
  294. A registered streaming provider plays chunked PCM; every other provider is spoken
  295. per-sentence via ``text_to_speech_tool``, so audio still starts on sentence one. Protocol:
  296. ``str`` deltas, a ``None`` sentinel = end-of-text (flush), *stop_event* aborts (barge-in),
  297. *tts_done_event* is **set** in ``finally`` so continuous voice mode knows playback finished."""
  298. tts_done_event.clear()
  299. origin = _origin()
  300. sync_pipeline: Optional[_SyncSentencePipeline] = None
  301. playback: Optional[_StreamerPlayback] = None
  302. try:
  303. tts_config = origin._load_tts_config()
  304. # Prefer a chunked streamer for low time-to-first-audio; otherwise per-sentence sync
  305. # synthesis (universal — edge + every non-streamer).
  306. from tools.tts_streaming import SentenceChunker, resolve_streaming_provider
  307. streamer = resolve_streaming_provider(tts_config, preferred=provider)
  308. stream_max_len = 0
  309. if streamer is None:
  310. sync_pipeline = _SyncSentencePipeline(stop_event)
  311. else:
  312. with contextlib.suppress(Exception):
  313. stream_max_len = origin._resolve_max_text_length(
  314. provider or origin._get_provider(tts_config), tts_config)
  315. playback = _StreamerPlayback(streamer, stop_event)
  316. chunker = SentenceChunker.from_config(tts_config)
  317. spoken_sentences: list[str] = [] # skip duplicate/near-duplicate sentences (LLM repetition)
  318. def _speak_sentence(sentence: str) -> None:
  319. if stop_event.is_set():
  320. return
  321. cleaned = _strip_markdown_for_tts(sentence).strip()
  322. if not cleaned:
  323. return
  324. cleaned_lower = cleaned.lower().rstrip(".!,")
  325. if any(prev.lower().rstrip(".!,") == cleaned_lower for prev in spoken_sentences):
  326. return
  327. spoken_sentences.append(cleaned)
  328. if display_callback is not None:
  329. display_callback(sentence) # raw sentence on screen before TTS processing
  330. if sync_pipeline is not None:
  331. sync_pipeline.speak(cleaned)
  332. return
  333. if stream_max_len and len(cleaned) > stream_max_len:
  334. cleaned = cleaned[:stream_max_len]
  335. playback.speak(cleaned)
  336. while not stop_event.is_set():
  337. try:
  338. delta = text_queue.get(timeout=0.5)
  339. except queue.Empty:
  340. delta = "" # idle producer: flush a long buffer instead of sitting on it
  341. sentences = chunker.flush() if len(chunker.buf) > 100 else ()
  342. else:
  343. sentences = chunker.flush() if delta is None else chunker.feed(delta)
  344. for sentence in sentences:
  345. _speak_sentence(sentence)
  346. if delta is None:
  347. break
  348. with contextlib.suppress(queue.Empty):
  349. while True:
  350. text_queue.get_nowait()
  351. except Exception as exc:
  352. logger.warning("Streaming TTS pipeline error: %s", exc)
  353. finally:
  354. # Flush the sync pipeline first: queued sentences finish playing (or are skipped when
  355. # stop_event is set) BEFORE tts_done_event fires, so continuous voice mode never reopens
  356. # the mic over its own voice. The end sentinel lives in finally: so an exception in the
  357. # text pump still lets the playback worker exit.
  358. if sync_pipeline is not None:
  359. with contextlib.suppress(Exception):
  360. sync_pipeline.close()
  361. if playback is not None:
  362. playback.finish()
  363. tts_done_event.set()