voice_live.py 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186
  1. """GPT-Live voice chat mode: the full-duplex voice frontend that delegates to Hermes.
  2. ``voice.voice_chat_mode: gpt-live`` replaces the chained STT → turn → TTS loop with ONE
  3. full-duplex voice model (OpenAI ``gpt-live-1``) that owns the microphone and the speaker and
  4. delegates every real request to Hermes as its *client-delegation* backend. Hermes stays the
  5. agent: whatever model/provider the session has selected answers, with the full toolset.
  6. Division of labour (the Live API has no tools of its own in client mode):
  7. * the desktop renderer holds the WebRTC media session (mic in, speech out) and the data channel;
  8. * this module resolves WHICH credentials/voice/persona to use and performs the one server-side
  9. step the API requires — exchanging the browser's SDP offer for an answer with the project key
  10. (``POST /v1/live/sessions``), so the key never reaches the client;
  11. * the renderer turns each ``session.delegation.created`` into a normal ``prompt.submit`` on the
  12. active session (surface ``voice-live``) and streams the reply back as
  13. ``session.commentary.append`` — Hermes' answer is what the voice speaks.
  14. Vendor contract: https://developers.openai.com/api/docs/guides/live (+ live-delegation,
  15. voice-webrtc). Billing is $0.05/min of session time on the OpenAI key, separate from the
  16. Hermes turn.
  17. """
  18. from __future__ import annotations
  19. import json
  20. import logging
  21. import urllib.error
  22. import urllib.request
  23. from typing import Any, Dict, Optional
  24. logger = logging.getLogger(__name__)
  25. GPT_LIVE_MODE = "gpt-live"
  26. CHAINED_MODE = "chained"
  27. DEFAULT_LIVE_MODEL = "gpt-live-1"
  28. DEFAULT_LIVE_VOICE = "marin"
  29. DEFAULT_LIVE_BASE_URL = "https://api.openai.com/v1"
  30. # Voices the vendor lists for gpt-live-1 (live-conversations guide) plus the realtime defaults it
  31. # accepts; free text stays allowed for custom voices.
  32. GPT_LIVE_VOICES = (
  33. "marin", "cedar", "quartz", "ripple", "vesper", "willow", "stone", "gleam", "meridian",
  34. "bossa", "tempo", "beacon", "delta", "cinder",
  35. )
  36. # Persona for the voice layer. Short on purpose: the live model has a small context window and
  37. # the vendor guide asks for role + style + a labelled delegation policy, nothing more. The
  38. # backend (Hermes) carries the real instructions, tools and memory.
  39. LIVE_PERSONA = (
  40. "You are Hermes, a calm and friendly voice assistant. Speak naturally at an unhurried pace. "
  41. "Be clear and direct, not overly cheerful. If the user is frustrated, acknowledge it briefly "
  42. "and focus on the next helpful step.\n\n"
  43. "Backchannel policy: Use moderate backchannels. Acknowledge naturally without competing with "
  44. "the main response.\n\n"
  45. "Interruption policy: Stop speaking when the user interrupts. Listen to what they say.\n\n"
  46. "Delegation policy:\n"
  47. "Backend tools:\n"
  48. "- Hermes agent: a full AI agent with tools — it can run commands, read and edit files, "
  49. "browse the web, search, remember things across sessions, schedule tasks, and reason "
  50. "carefully about anything. It is the one who actually does work and knows facts.\n\n"
  51. "Delegate to the backend when:\n"
  52. "- The user asks a question that needs facts, current information, or careful reasoning.\n"
  53. "- The user asks you to do, check, find, make, fix, run or remember anything.\n"
  54. "- A correction changes work already requested.\n\n"
  55. "Do not delegate to the backend when:\n"
  56. "- The user greets you, makes small talk, or asks you to repeat a result already provided.\n"
  57. "- You need a brief clarification to understand the request.\n\n"
  58. "Delegate before giving an answer that depends on backend work. Do not guess the result "
  59. "while waiting; say briefly that you are checking, then wait for the result."
  60. )
  61. # Per-turn note prepended to the MODEL INPUT (never the byte-stable system prompt) when a turn
  62. # arrives from the live voice layer. Same seam as the HUD note.
  63. VOICE_LIVE_TURN_NOTE = (
  64. "[Note: this message is a delegation from a live spoken conversation. The text is a voice "
  65. "transcript (it may contain mis-hearings, hesitations and later corrections; use the latest "
  66. "intent). Your reply will be spoken aloud by a voice model that paraphrases it: answer in plain "
  67. "conversational sentences, keep it short (a few sentences unless the user asked for detail), no "
  68. "markdown, no lists, no code blocks, no URLs read out character by character. Do the work with "
  69. "your tools as usual; only the final facts need to be spoken. Do not claim an action succeeded "
  70. "before it actually did.]"
  71. )
  72. def voice_live_turn_note(context: str = "") -> str:
  73. """The per-turn note plus, when the client sent one, the recent spoken exchange the delegation
  74. refers to (the user's last words alone are often "yes" or "Thursday, not Friday")."""
  75. context = context.strip()
  76. if not context:
  77. return VOICE_LIVE_TURN_NOTE
  78. return f"{VOICE_LIVE_TURN_NOTE}\n[Recent spoken conversation, newest last:\n{context}]"
  79. def _voice_section() -> dict[str, Any]:
  80. try:
  81. from hermes_cli.config import load_config
  82. voice = load_config().get("voice")
  83. except Exception:
  84. return {}
  85. return voice if isinstance(voice, dict) else {}
  86. def _live_section(voice: Optional[dict[str, Any]] = None) -> dict[str, Any]:
  87. section = (voice if voice is not None else _voice_section()).get("gpt_live")
  88. return section if isinstance(section, dict) else {}
  89. def voice_chat_mode(voice: Optional[dict[str, Any]] = None) -> str:
  90. """``chained`` (default) or ``gpt-live``. Accepts the underscore spelling too."""
  91. raw = (voice if voice is not None else _voice_section()).get("voice_chat_mode")
  92. mode = str(raw or CHAINED_MODE).strip().lower().replace("_", "-")
  93. return GPT_LIVE_MODE if mode in {GPT_LIVE_MODE, "gptlive", "live"} else CHAINED_MODE
  94. def _resolve_credentials(live: dict[str, Any]) -> tuple[str, str]:
  95. """``(api_key, base_url)`` — ``voice.gpt_live.api_key`` first, else the same OpenAI audio
  96. chain the STT/TTS providers use (``VOICE_TOOLS_OPENAI_KEY`` → ``OPENAI_API_KEY`` → pool).
  97. The Nous-managed audio proxy does not carry ``/live/sessions``; this mode is direct-key only.
  98. """
  99. from tools.tool_backend_helpers import resolve_openai_audio_api_key
  100. api_key = str(live.get("api_key") or "").strip() or resolve_openai_audio_api_key()
  101. base_url = str(live.get("base_url") or DEFAULT_LIVE_BASE_URL).strip().rstrip("/")
  102. return api_key, base_url
  103. def live_instructions(live: Optional[dict[str, Any]] = None) -> str:
  104. extra = str((live if live is not None else _live_section()).get("instructions") or "").strip()
  105. return f"{LIVE_PERSONA}\n\n{extra}" if extra else LIVE_PERSONA
  106. def resolve_gpt_live_status() -> dict[str, Any]:
  107. """Non-secret readiness verdict for the client: which mode is selected and whether GPT-Live
  108. can start (a key resolves). Never returns the key."""
  109. voice = _voice_section()
  110. mode = voice_chat_mode(voice)
  111. live = _live_section(voice)
  112. api_key, _base = _resolve_credentials(live)
  113. return {
  114. "mode": mode,
  115. "available": bool(api_key),
  116. "reason": None if api_key else "no OpenAI API key (set OPENAI_API_KEY or voice.gpt_live.api_key)",
  117. "model": str(live.get("model") or DEFAULT_LIVE_MODEL),
  118. "voice": str(live.get("voice") or DEFAULT_LIVE_VOICE),
  119. }
  120. def build_session_config(history: Optional[list] = None) -> dict[str, Any]:
  121. """The ``session`` object for ``POST /v1/live/sessions`` (client delegation, WebRTC — the
  122. transport negotiates the audio format, so none is set)."""
  123. live = _live_section()
  124. config: dict[str, Any] = {
  125. "model": str(live.get("model") or DEFAULT_LIVE_MODEL),
  126. "instructions": live_instructions(live),
  127. "audio": {"output": {"voice": str(live.get("voice") or DEFAULT_LIVE_VOICE)}},
  128. "delegation": {"type": "client"},
  129. }
  130. if history:
  131. config["input"] = history
  132. return config
  133. def create_webrtc_session(sdp_offer: str, history: Optional[list] = None) -> dict[str, Any]:
  134. """Exchange the renderer's SDP offer for the Live session answer.
  135. Returns the vendor response ``{"session": {"id": ...}, "transport": {"type": "webrtc",
  136. "sdp": ...}}``. Raises ``ValueError`` for a missing key and ``RuntimeError`` (with the vendor
  137. status/detail) for a rejected request.
  138. """
  139. live = _live_section()
  140. api_key, base_url = _resolve_credentials(live)
  141. if not api_key:
  142. raise ValueError("GPT-Live needs an OpenAI API key (OPENAI_API_KEY or voice.gpt_live.api_key)")
  143. body = json.dumps({
  144. "session": build_session_config(history),
  145. "transport": {"type": "webrtc", "sdp": sdp_offer},
  146. }).encode("utf-8")
  147. req = urllib.request.Request(
  148. f"{base_url}/live/sessions", data=body, method="POST",
  149. headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"})
  150. try:
  151. with urllib.request.urlopen(req, timeout=30) as resp:
  152. return json.loads(resp.read().decode("utf-8"))
  153. except urllib.error.HTTPError as exc:
  154. detail = exc.read().decode("utf-8", "replace")[:600]
  155. logger.warning("GPT-Live session creation failed: %s %s", exc.code, detail)
  156. raise RuntimeError(f"GPT-Live session creation failed ({exc.code}): {detail}") from exc