openai_client.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197
  1. """OpenAI Realtime API WebSocket client + file-queue speaker.
  2. text → OpenAI Realtime → audio deltas appended as PCM to a file the audio bridge streams into
  3. Chrome's fake mic. One sync WebSocket per session; ``websockets`` is imported lazily.
  4. """
  5. from __future__ import annotations
  6. import base64
  7. import contextlib
  8. import json
  9. import threading
  10. import time
  11. import uuid
  12. from pathlib import Path
  13. from typing import Any, Callable, Optional
  14. from plugins.google_meet._jsonfile import say_queue_lock
  15. REALTIME_URL = "wss://api.openai.com/v1/realtime"
  16. _TERMINAL_FRAMES = {"response.done", "response.completed", "response.cancelled"}
  17. def _decode_audio(b64: str) -> bytes:
  18. try:
  19. return base64.b64decode(b64) if b64 else b""
  20. except (ValueError, TypeError):
  21. return b""
  22. class RealtimeSession:
  23. """Minimal sync client for the OpenAI Realtime WebSocket API; ``speak`` and ``cancel_response``
  24. may run on different threads — a lock serializes WebSocket writes."""
  25. def __init__(self, api_key: str, model: str = "gpt-realtime", voice: str = "alloy",
  26. instructions: str = "", audio_sink_path: Optional[Path] = None, sample_rate: int = 24000) -> None:
  27. self.api_key = api_key
  28. self.model = model
  29. self.voice = voice
  30. self.instructions = instructions
  31. self.audio_sink_path = Path(audio_sink_path) if audio_sink_path else None
  32. self.sample_rate = sample_rate
  33. self._ws: Any = None
  34. self._send_lock = threading.Lock()
  35. self.audio_bytes_out: int = 0 # public counters for status reporting
  36. self.last_audio_out_at: Optional[float] = None
  37. def connect(self) -> None:
  38. """Open the WS and send ``session.update`` with voice + instructions."""
  39. try:
  40. from websockets.sync.client import connect # type: ignore
  41. except ImportError as exc: # pragma: no cover - exercised via test
  42. raise RuntimeError("websockets package is required for OpenAI Realtime; "
  43. "install with: pip install websockets") from exc
  44. url = f"{REALTIME_URL}?model={self.model}"
  45. headers = [("Authorization", f"Bearer {self.api_key}"), ("OpenAI-Beta", "realtime=v1")]
  46. # Newer websockets takes additional_headers=, older extra_headers=.
  47. try:
  48. self._ws = connect(url, additional_headers=headers)
  49. except TypeError:
  50. self._ws = connect(url, extra_headers=headers)
  51. self._send_json({"type": "session.update", "session": {
  52. "voice": self.voice, "instructions": self.instructions, "modalities": ["audio", "text"],
  53. "output_audio_format": "pcm16", "input_audio_format": "pcm16"}})
  54. def close(self) -> None:
  55. if self._ws is not None:
  56. with contextlib.suppress(Exception):
  57. self._ws.close()
  58. self._ws = None
  59. def speak(self, text: str, timeout: float = 30.0) -> dict:
  60. """Send ``text`` and append the audio response to ``audio_sink_path`` (opened 'ab' per call
  61. so a streaming reader can consume it). Frames other than audio deltas/terminal/error are ignored."""
  62. if self._ws is None:
  63. raise RuntimeError("RealtimeSession.connect() must be called first")
  64. start = time.monotonic()
  65. self._send_json({"type": "conversation.item.create", "item": {
  66. "type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]}})
  67. self._send_json({"type": "response.create", "response": {"modalities": ["audio"]}})
  68. bytes_written = 0
  69. with contextlib.ExitStack() as stack:
  70. sink_fp = None
  71. if self.audio_sink_path is not None:
  72. self.audio_sink_path.parent.mkdir(parents=True, exist_ok=True)
  73. sink_fp = stack.enter_context(open(self.audio_sink_path, "ab"))
  74. while True:
  75. frame = self._recv_frame(start + timeout, timeout)
  76. if frame is None or frame.get("type") in _TERMINAL_FRAMES: # peer closed / response done
  77. break
  78. ftype = frame.get("type")
  79. if ftype == "error":
  80. raise RuntimeError(f"realtime error: {frame.get('error') or frame}")
  81. chunk = _decode_audio(frame.get("delta") or frame.get("audio") or "") if (
  82. ftype == "response.audio.delta" and sink_fp is not None) else b""
  83. if chunk:
  84. sink_fp.write(chunk)
  85. sink_fp.flush()
  86. bytes_written += len(chunk)
  87. self.audio_bytes_out += len(chunk)
  88. self.last_audio_out_at = time.time()
  89. return {"ok": True, "bytes_written": bytes_written, "duration_ms": (time.monotonic() - start) * 1000.0}
  90. def cancel_response(self) -> bool:
  91. """Barge-in: send ``response.cancel``. True if sent, False if nothing to cancel / socket closed."""
  92. if self._ws is None:
  93. return False
  94. try:
  95. self._send_json({"type": "response.cancel"})
  96. return True
  97. except Exception:
  98. return False
  99. def _send_json(self, payload: dict) -> None:
  100. assert self._ws is not None
  101. with self._send_lock:
  102. self._ws.send(json.dumps(payload))
  103. def _recv_frame(self, deadline: float, timeout: float) -> Optional[dict]:
  104. """Next dict frame before *deadline* (monotonic), ``None`` once the peer closes.
  105. Non-dict / unparseable frames are skipped; TimeoutError past the deadline."""
  106. assert self._ws is not None
  107. while True:
  108. remaining = deadline - time.monotonic()
  109. if remaining <= 0:
  110. raise TimeoutError(f"realtime response did not complete within {timeout}s")
  111. try:
  112. raw = self._ws.recv(timeout=remaining)
  113. except TypeError: # older websockets: no timeout kwarg
  114. raw = self._ws.recv()
  115. if raw is None:
  116. return None
  117. with contextlib.suppress(TypeError, ValueError):
  118. frame = json.loads(raw) if isinstance(raw, (str, bytes, bytearray)) else raw
  119. if isinstance(frame, dict):
  120. return frame
  121. class RealtimeSpeaker:
  122. """JSONL queue (``{"id", "text"}`` per line) wrapper around :class:`RealtimeSession`; processed
  123. lines are appended to ``processed_path`` (if set) and removed from the queue."""
  124. def __init__(self, session: RealtimeSession, queue_path: Path, processed_path: Optional[Path] = None) -> None:
  125. self.session = session
  126. self.queue_path = Path(queue_path)
  127. self.processed_path = Path(processed_path) if processed_path else None
  128. def _read_queue(self) -> list[dict]:
  129. """Parse the JSONL queue, skipping blank/malformed lines; entries lacking an ``id`` get one."""
  130. if not self.queue_path.exists():
  131. return []
  132. out: list[dict] = []
  133. for line in self.queue_path.read_text(encoding="utf-8-sig").splitlines():
  134. with contextlib.suppress(ValueError):
  135. entry = json.loads(line) if line.strip() else None
  136. if isinstance(entry, dict):
  137. entry.setdefault("id", str(uuid.uuid4()))
  138. out.append(entry)
  139. return out
  140. def _rewrite_queue(self, remaining: list[dict]) -> None:
  141. # Always keep the file (empty when drained): consumers may watch its mtime.
  142. body = "".join(json.dumps(e) + "\n" for e in remaining)
  143. self.queue_path.write_text(body, encoding="utf-8")
  144. def _append_processed(self, entry: dict, result: dict) -> None:
  145. if self.processed_path is None:
  146. return
  147. self.processed_path.parent.mkdir(parents=True, exist_ok=True)
  148. record = {"id": entry.get("id"), "text": entry.get("text", ""), "result": result}
  149. with open(self.processed_path, "a", encoding="utf-8") as fp:
  150. fp.write(json.dumps(record) + "\n")
  151. def run_until_stopped(self, stop_fn: Callable[[], bool], poll_interval: float = 0.5) -> None:
  152. while not stop_fn():
  153. entries = self._read_queue()
  154. if not entries:
  155. time.sleep(poll_interval)
  156. continue
  157. # One entry per iteration: the queue may grow while we speak.
  158. head = entries[0]
  159. text = (head.get("text") or "").strip()
  160. result = {"ok": True, "bytes_written": 0, "duration_ms": 0.0}
  161. if text:
  162. try:
  163. result = self.session.speak(text)
  164. except Exception as exc:
  165. result = {"ok": False, "error": str(exc)}
  166. self._append_processed(head, result)
  167. # Re-read (new entries may have arrived), then drop the head by position or id — under the
  168. # queue lock, so a meet_say append cannot land between the re-read and the rewrite.
  169. with say_queue_lock(self.queue_path):
  170. latest = self._read_queue()
  171. self._rewrite_queue(latest[1:] if latest and latest[0].get("id") == head.get("id")
  172. else [e for e in latest if e.get("id") != head.get("id")])