streaming_tts_consumer.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. """Gateway streaming-TTS consumer — LLM deltas to adapter PCM audio sink.
  2. ``on_delta`` (agent worker thread) never blocks: SentenceChunker -> thread-safe queue. The
  3. ``_run`` task on the gateway loop drains, synthesises via a ``StreamingTTSProvider`` and writes
  4. PCM so playback starts mid-generation. Outcome: success -> ``completed``; failure before audible
  5. output -> ``suppress_whole_file=False`` (gateway falls back to whole-file TTS); failure after
  6. partial audio -> ``partial`` + suppress (never replay the response from the beginning).
  7. """
  8. from __future__ import annotations
  9. import asyncio
  10. import contextlib
  11. import logging
  12. import queue
  13. import threading
  14. from typing import Any, Dict, Optional
  15. from gateway.platforms.base import AudioFormat, StreamingTTSHandle
  16. logger = logging.getLogger("gateway.streaming_tts_consumer")
  17. _ABORT = object()
  18. _DONE = object()
  19. class _HandleDeclined(Exception):
  20. """The adapter declined ``begin_streaming_tts`` when the first PCM chunk arrived."""
  21. class StreamingTTSConsumer:
  22. """Consumes LLM text deltas and produces streaming PCM audio for an adapter."""
  23. def __init__(self, adapter: Any, chat_id: str, tts_config: dict[str, Any],
  24. loop: asyncio.AbstractEventLoop, *, metadata: Optional[dict[str, Any]] = None,
  25. audio_format: Optional[AudioFormat] = None) -> None:
  26. from tools.tts_streaming import SentenceChunker, resolve_streaming_provider
  27. self._adapter, self._chat_id, self._loop, self._metadata = adapter, chat_id, loop, metadata
  28. # Resolved once; None => inactive, gateway falls back to whole-file TTS.
  29. self._streamer = resolve_streaming_provider(tts_config)
  30. self._chunker = SentenceChunker.from_config(tts_config)
  31. # Provisional: refreshed from the streamer when the handle opens on the first PCM chunk,
  32. # since an OpenAI-compatible endpoint reports its real rate only in the response (#76466).
  33. self._audio_format = audio_format or AudioFormat() if self._streamer is None else self._streamer_format()
  34. # Thread-safe queue of completed clauses plus the _DONE/_ABORT sentinels.
  35. self._queue: queue.Queue[Any] = queue.Queue(maxsize=256)
  36. self._handle: Optional[StreamingTTSHandle] = None
  37. self._task: Optional[asyncio.Task] = None # drain task, created once by start()
  38. self._completed = self._partial = self._aborted = False
  39. self._finished = self._dropped = self._suppress_whole_file = False
  40. self._lock, self._strip_markdown = threading.Lock(), None # stripper lazily imported
  41. def _streamer_format(self) -> AudioFormat:
  42. return AudioFormat(**{f: int(getattr(self._streamer, f, getattr(AudioFormat, f)))
  43. for f in ("sample_rate", "channels", "sample_width")})
  44. active = property(lambda self: self._streamer is not None) # usable streaming provider
  45. completed = property(lambda self: self._completed) # streaming audio fully delivered
  46. partial = property(lambda self: self._partial) # some audio audible before a failure/drop
  47. audible = property(lambda self: bool(self._handle and self._handle.audible)) # PCM written
  48. dropped = property(lambda self: self._dropped) # queue saturation dropped >= 1 clause
  49. suppress_whole_file = property(lambda self: self._suppress_whole_file) # skip whole-file TTS
  50. done = property(lambda self: self._task is not None and self._task.done()) # drain task ended
  51. def _enqueue_clauses(self, clauses, full_msg: str, *, log_errors: bool) -> None:
  52. try:
  53. for clause in clauses:
  54. self._queue.put_nowait(clause)
  55. except queue.Full:
  56. self._dropped = True
  57. logger.debug(full_msg)
  58. except Exception:
  59. if log_errors:
  60. logger.debug("streaming TTS on_delta error", exc_info=True)
  61. def on_delta(self, text: Optional[str]) -> None:
  62. """Receive text, or flush a ``None`` segment boundary without ending audio. Non-blocking."""
  63. if self._aborted or not self.active or self._finished:
  64. return
  65. clauses = self._chunker.flush() if text is None else self._chunker.feed(text)
  66. self._enqueue_clauses(clauses, "streaming TTS queue full, dropping clause",
  67. log_errors=True)
  68. def finish(self) -> None:
  69. """Signal end-of-text: flush the chunker tail, then enqueue ``_DONE`` after all flushed
  70. clauses so the drain loop ends deterministically without racing a late ``on_delta``."""
  71. if self._finished:
  72. return
  73. self._finished = True
  74. if self._aborted or not self.active:
  75. return
  76. self._enqueue_clauses(self._chunker.flush(), "streaming TTS queue full while flushing tail",
  77. log_errors=False)
  78. # The load-bearing _DONE sentinel must never be lost: evict clauses until it fits.
  79. while not self._put_sentinel(_DONE, mark_dropped=True):
  80. pass
  81. def _put_sentinel(self, sentinel, *, mark_dropped: bool) -> bool:
  82. """Try to enqueue a sentinel, evicting one queued item when the queue is full. Returns True
  83. when the caller should stop retrying: enqueued, or (abort path) nothing left to evict."""
  84. with contextlib.suppress(queue.Full):
  85. self._queue.put_nowait(sentinel)
  86. return True
  87. try:
  88. self._queue.get_nowait()
  89. except queue.Empty:
  90. return not mark_dropped
  91. self._dropped = self._dropped or mark_dropped
  92. return False
  93. def start(self) -> asyncio.Task:
  94. """Create (once) and return the async drain task on the gateway loop."""
  95. if self._task is None:
  96. self._task = self._loop.create_task(self._run())
  97. return self._task
  98. def _settle(self, *, failed: bool) -> None:
  99. """Set outcome flags from what was audible: never report completion after a failure or a
  100. dropped clause; keep suppression whenever audio was audible (no replay from the start)."""
  101. audible, degraded = bool(self._handle and self._handle.audible), failed or self._dropped
  102. self._completed = audible and not degraded
  103. self._partial = self._partial or (audible and degraded)
  104. self._suppress_whole_file = audible
  105. async def _open_handle(self) -> bool:
  106. """Open the adapter's streaming-audio handle at the streamer's now-final format; False when
  107. begin failed. Called from the first PCM chunk, not before the provider answered."""
  108. if self._streamer is not None:
  109. self._audio_format = self._streamer_format()
  110. try:
  111. self._handle = await self._adapter.begin_streaming_tts(
  112. self._chat_id, self._audio_format, metadata=self._metadata
  113. )
  114. except Exception as exc:
  115. logger.debug("begin_streaming_tts failed: %s", exc)
  116. self._handle = None
  117. return self._handle is not None
  118. async def _run(self) -> None:
  119. """Drain clauses until a sentinel/abort, synthesise + write each, then finalise the stream;
  120. a clause or finalise failure settles the outcome flags and aborts the adapter stream."""
  121. if not self.active or not self._adapter.supports_streaming_tts(self._chat_id, self._audio_format):
  122. logger.debug("adapter %s does not support streaming TTS", getattr(self._adapter, "name", "?"))
  123. return
  124. self._suppress_whole_file = False
  125. try:
  126. while not self._aborted:
  127. try:
  128. item = await asyncio.to_thread(self._queue.get, True, 0.1)
  129. except queue.Empty:
  130. continue
  131. if item is _ABORT or item is _DONE or self._aborted:
  132. break
  133. if not isinstance(item, str):
  134. continue
  135. try:
  136. await self._synthesise_and_write(item)
  137. except _HandleDeclined:
  138. return # nothing audible yet: gateway falls back to whole-file TTS
  139. except Exception as exc:
  140. logger.warning("streaming TTS clause failed: %s", exc)
  141. self._settle(failed=True)
  142. await self._safe_abort(str(exc))
  143. return
  144. if not self._aborted and self._handle is not None:
  145. try:
  146. handle, interrupted = self._handle, self._aborted
  147. await self._adapter.finish_streaming_tts(handle, interrupted=interrupted)
  148. except Exception as exc:
  149. logger.debug("finish_streaming_tts error: %s", exc)
  150. self._settle(failed=True)
  151. await self._safe_abort("finish_streaming_tts failed")
  152. else:
  153. self._settle(failed=False)
  154. except Exception as exc:
  155. logger.warning("streaming TTS consumer error: %s", exc)
  156. await self._safe_abort(str(exc))
  157. finally:
  158. with contextlib.suppress(Exception):
  159. while not self._queue.empty():
  160. self._queue.get_nowait()
  161. async def _synthesise_and_write(self, clause: str) -> None:
  162. """Synthesise one clause via the streamer and write PCM chunks."""
  163. if self._streamer is None or (self._handle is not None and self._handle.aborted):
  164. return
  165. if self._strip_markdown is None: # lazy import: tools.tts_tool would cycle at module load
  166. try:
  167. from tools.tts_text_normalize import _strip_markdown_for_tts as _strip
  168. self._strip_markdown = _strip
  169. except ImportError:
  170. self._strip_markdown = lambda t: t
  171. if not (cleaned := self._strip_markdown(clause).strip()):
  172. return
  173. iterator = iter(self._streamer.stream(cleaned))
  174. while True:
  175. # next() runs in a thread so a blocking provider never stalls the loop.
  176. chunk = await asyncio.to_thread(next, iterator, _DONE)
  177. if chunk is _DONE or self._aborted or (self._handle is not None and self._handle.aborted):
  178. return
  179. if not chunk:
  180. continue
  181. if self._handle is None and not await self._open_handle():
  182. raise _HandleDeclined()
  183. was_audible = self._handle.audible
  184. await self._adapter.write_streaming_tts(self._handle, chunk)
  185. if not was_audible:
  186. self._handle.audible = self._suppress_whole_file = True
  187. async def _safe_abort(self, reason: str) -> None:
  188. """Abort the adapter stream, swallowing errors (idempotent)."""
  189. if self._handle is None:
  190. return
  191. try:
  192. with contextlib.suppress(Exception):
  193. await self._adapter.abort_streaming_tts(self._handle, error=reason)
  194. finally:
  195. if self._handle:
  196. self._handle.aborted = True
  197. def abort(self, reason: str = "cancelled") -> None:
  198. """Idempotent cancellation from any thread."""
  199. with self._lock:
  200. if self._aborted:
  201. return
  202. self._aborted = True
  203. # The load-bearing _ABORT sentinel must reach the queue even when full: evict to make room.
  204. if not any(self._put_sentinel(_ABORT, mark_dropped=False) for _ in range(3)):
  205. logger.debug("streaming TTS _ABORT sentinel could not be enqueued")
  206. if self._handle is not None and not self._handle.aborted:
  207. with contextlib.suppress(Exception):
  208. self._loop.call_soon_threadsafe(asyncio.create_task, self._safe_abort(reason))
  209. async def wait_complete(self, timeout: float = 10.0) -> bool:
  210. """Wait for the drain task to finish. Returns True only on full success."""
  211. if self._task is not None:
  212. with contextlib.suppress(asyncio.CancelledError, Exception):
  213. await asyncio.wait_for(asyncio.shield(self._task), timeout=timeout)
  214. return self._completed