run_inbound_transcription.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187
  1. """Voice-clip transcription for inbound messages (moved out of ``gateway/run_inbound.py``)."""
  2. from __future__ import annotations
  3. import asyncio
  4. import logging
  5. import os
  6. from typing import Optional
  7. from gateway.run_common import _UNSET
  8. # Log-record parity with the origin module.
  9. logger = logging.getLogger("gateway.run")
  10. class GatewayInboundTranscriptionMixin:
  11. """STT enrichment and the once-only echo of pending voice transcripts."""
  12. async def _enrich_message_with_transcription(
  13. self, user_text: str, audio_paths: list[str], clip_memo: Optional[dict] = None
  14. ) -> tuple[str, list[str]]:
  15. """Transcribe voice clips with the configured STT provider and prepend the transcripts →
  16. ``(enriched_text, successful_transcripts)``; the transcripts (input order; empty if every clip
  17. failed or STT is disabled) let callers echo them back before the agent loop. ``clip_memo``
  18. (path → ``_transcribe_one_clip`` result) skips clips an earlier pass already transcribed."""
  19. from gateway.run import _probe_audio_duration
  20. audio_paths = list(dict.fromkeys(audio_paths))
  21. if not getattr(self.config, "stt_enabled", True):
  22. notes = []
  23. for path in audio_paths:
  24. abs_path = os.path.abspath(path)
  25. duration_str = await _probe_audio_duration(abs_path)
  26. suffix = f" (duration: {duration_str})" if duration_str else ""
  27. notes.append(f"[The user sent a voice message: {abs_path}{suffix}]")
  28. return (self._prepend_media_prefix("\n\n".join(notes), user_text) if notes else user_text), []
  29. try:
  30. from tools.transcription_tools import (
  31. transcribe_audio, transcribe_audio_local_fallback
  32. )
  33. except ModuleNotFoundError as e:
  34. logger.error("Transcription module unavailable: %s", e)
  35. return self._prepend_media_prefix("[voice message could not be transcribed]", user_text), []
  36. enriched_parts = []
  37. successful_transcripts: list[str] = []
  38. for path in audio_paths:
  39. try:
  40. if clip_memo is not None and path in clip_memo:
  41. transcript, note = clip_memo[path]
  42. else:
  43. logger.debug("Transcribing user voice: %s", path)
  44. transcript, note = await self._transcribe_one_clip(
  45. path, transcribe_audio, transcribe_audio_local_fallback,
  46. )
  47. if clip_memo is not None:
  48. clip_memo[path] = (transcript, note)
  49. if transcript is not None:
  50. successful_transcripts.append(transcript)
  51. enriched_parts.append(note)
  52. except Exception as e:
  53. logger.error("Transcription error: %s", e)
  54. enriched_parts.append(self._untranscribed_audio_note(path))
  55. if enriched_parts:
  56. user_text = self._prepend_media_prefix("\n\n".join(enriched_parts), user_text)
  57. return user_text, successful_transcripts
  58. def _pending_event_audio_paths(self, event) -> list[str]:
  59. """Return STT-eligible paths from a pending voice message."""
  60. from gateway.run import _event_media_is_stt_input
  61. return [
  62. path for i, path in enumerate(getattr(event, "media_urls", None) or [])
  63. if _event_media_is_stt_input(event, i)
  64. ]
  65. async def _transcribe_pending_audio_event_once(
  66. self, event, user_text: Optional[str] = None
  67. ) -> tuple[str | None, list[str]]:
  68. """Transcribe a pending audio event once and cache the result on the event: the interrupt
  69. monitor and the pending-drain path both need it — one STT call and one echo per message.
  70. ``user_text`` is a deliberate override of the caption; ``None`` reads ``event.text`` live."""
  71. if hasattr(event, "_gateway_pending_stt_text"):
  72. return event._gateway_pending_stt_text, list(getattr(event, "_gateway_pending_stt_transcripts", []) or [])
  73. audio_paths = self._pending_event_audio_paths(event)
  74. if not audio_paths:
  75. return user_text if user_text is not None else (getattr(event, "text", None) or None), []
  76. text = user_text if user_text is not None else (getattr(event, "text", "") or "")
  77. # Per-clip results outlive the merged-event cache below (a merge drops only that), so every
  78. # re-run — this loop's or the drain path's — transcribes just the newly merged clips.
  79. clip_memo = getattr(event, "_gateway_pending_stt_clips", None)
  80. if clip_memo is None:
  81. clip_memo = {}
  82. event._gateway_pending_stt_clips = clip_memo
  83. enriched_text, successful_transcripts = await self._enrich_message_with_transcription(text, audio_paths, clip_memo)
  84. # A voice note merged in during the await dropped a cache that did not exist yet; re-run on
  85. # the merged media rather than publish a transcript that silently omits the new note.
  86. while self._pending_event_audio_paths(event) != audio_paths:
  87. audio_paths = self._pending_event_audio_paths(event)
  88. text = user_text if user_text is not None else (getattr(event, "text", "") or "")
  89. enriched_text, successful_transcripts = await self._enrich_message_with_transcription(
  90. text, audio_paths, clip_memo,
  91. )
  92. event._gateway_pending_stt_text = enriched_text
  93. event._gateway_pending_stt_transcripts = list(successful_transcripts)
  94. return enriched_text, successful_transcripts
  95. async def _echo_pending_stt_transcripts_once(
  96. self, event, adapter, source, transcripts: list[str], *, metadata=None,
  97. log_context: str = "Transcript",
  98. ) -> None:
  99. """Echo pending-event STT transcripts to the chat at most once. Tracked as a COUNT (not a
  100. set — identical transcripts are distinct deliveries): ``merge_pending_message_event`` can
  101. append a second voice note and invalidate the cache; the re-run returns earlier transcripts
  102. as a prefix, so only the unsent tail is echoed."""
  103. if not transcripts or not self._should_echo_stt_transcripts() or adapter is None:
  104. return
  105. already_echoed = int(getattr(event, "_gateway_pending_stt_echoed", 0) or 0)
  106. event._gateway_pending_stt_echoed = max(already_echoed, len(transcripts))
  107. await self._echo_stt_transcripts(
  108. adapter, source, transcripts[already_echoed:], metadata=metadata, log_context=log_context,
  109. )
  110. async def _transcribe_and_echo_pending_voice(
  111. self, event, adapter, source, text: str, *, log_context: str, metadata=_UNSET
  112. ) -> tuple[str, list[str]]:
  113. """Transcribe a pending voice event and echo transcripts once → ``(enriched_text,
  114. transcripts)`` for ``agent.interrupt()`` or the pending-drain flow; ``(text, [])`` when there
  115. is no STT-eligible media (caller owns the ``_build_media_placeholder`` fallback). ``text`` is
  116. only that fallback: the caption is read live from ``event.text`` so one merged in during
  117. the STT await is kept."""
  118. if not self._pending_event_audio_paths(event):
  119. return text, []
  120. try:
  121. enriched_text, transcripts = await self._transcribe_pending_audio_event_once(event)
  122. if metadata is _UNSET:
  123. metadata = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event))
  124. await self._echo_pending_stt_transcripts_once(
  125. event, adapter, source, transcripts, metadata=metadata, log_context=log_context
  126. )
  127. return enriched_text or text, transcripts
  128. except Exception as trans_exc:
  129. logger.warning("%s transcription failed: %s", log_context, trans_exc)
  130. return text, []
  131. _EMPTY_TEXT_PLACEHOLDER = "(The user sent a message with no text content)"
  132. @classmethod
  133. def _prepend_media_prefix(cls, prefix: str, user_text: str) -> str:
  134. """``prefix`` + the user's text; the Discord empty-content placeholder is dropped as redundant."""
  135. if user_text and user_text.strip() != cls._EMPTY_TEXT_PLACEHOLDER:
  136. return f"{prefix}\n\n{user_text}"
  137. return prefix
  138. @staticmethod
  139. def _untranscribed_audio_note(path: str) -> str:
  140. """One minimal neutral marker for every STT failure. Never mention "no STT provider" or setup
  141. steps — persisted in history they make the model keep volunteering STT-setup advice."""
  142. from tools.credential_files import to_agent_visible_cache_path
  143. agent_path = to_agent_visible_cache_path(os.path.abspath(path))
  144. return f"[voice message could not be transcribed automatically; the audio is available at: {agent_path}]"
  145. async def _transcribe_one_clip(self, path: str, transcribe_audio, transcribe_audio_local_fallback) -> tuple[Optional[str], str]:
  146. """``(transcript_or_None, note)`` for one clip via configured STT with local fallback."""
  147. result = await asyncio.to_thread(transcribe_audio, path, None, "gateway")
  148. if not result.get("success"):
  149. fallback = await asyncio.to_thread(transcribe_audio_local_fallback, path)
  150. if fallback.get("success"):
  151. logger.info("Configured STT failed for %s; recovered with local STT", path)
  152. result = fallback
  153. if not result["success"]:
  154. logger.info("Voice transcription failed for %s: %s", path, result.get("error", "unknown error"))
  155. return None, self._untranscribed_audio_note(path)
  156. transcript = result["transcript"]
  157. # STT may return success=True with an empty/whitespace transcript (silence, cut-off);
  158. # empty quotes make the agent reply to nothing and can loop, so emit a sentinel note.
  159. # See #41603.
  160. if not (transcript or "").strip():
  161. return None, (
  162. "[The user sent a voice message but it came through "
  163. "empty or inaudible — speech-to-text returned no "
  164. "words. Do not guess at the content; ask the user "
  165. "to resend or type it out.]"
  166. )
  167. # Plain quoted line: a "The user sent a voice message..." wrapper read as a meta-instruction
  168. # and made the LLM comment on voice mode instead.
  169. return transcript, f'"{transcript}"'