voice_client_config.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259
  1. """Resolve the active profile's STT/TTS config for CLIENT-DIRECT voice.
  2. Single resolver behind ``GET /api/audio/voice-config``: the desktop skips the
  3. audio relay hop by calling providers directly with the profile's own
  4. credentials, resolved through the exact chains of ``tools.transcription_tools``
  5. and ``tools.tts_tool`` so the client gets byte-for-byte what the gateway uses.
  6. Rules: same trust boundary as every REST route (keys never logged, never hit
  7. client disk); relay is the floor, not an error (server-host-only providers and
  8. resolution failures return ``{"mode": "relay"}``); no new key stores.
  9. Config gate: ``voice.client_direct`` (default ``true``).
  10. """
  11. from __future__ import annotations
  12. import logging
  13. from typing import Any, Dict, Optional
  14. logger = logging.getLogger(__name__)
  15. # Wire shapes the desktop knows how to speak. Anything else → relay.
  16. # openai-multipart : POST {base_url}/audio/transcriptions (multipart, Bearer)
  17. # xai-stt : POST {base_url}/stt (multipart, Bearer, model, format=true)
  18. # elevenlabs-stt : POST {base_url}/speech-to-text (multipart, xi-api-key)
  19. # openai-speech : POST {base_url}/audio/speech (JSON, Bearer) → audio bytes
  20. # elevenlabs-tts : POST {base_url}/text-to-speech/{voice_id} (JSON, xi-api-key)
  21. STT_WIRE_OPENAI = "openai-multipart"
  22. STT_WIRE_XAI = "xai-stt"
  23. STT_WIRE_ELEVENLABS = "elevenlabs-stt"
  24. TTS_WIRE_OPENAI = "openai-speech"
  25. TTS_WIRE_ELEVENLABS = "elevenlabs-tts"
  26. def _client_direct_enabled() -> bool:
  27. try:
  28. from hermes_cli.config import load_config
  29. voice_cfg = load_config().get("voice") or {}
  30. if not isinstance(voice_cfg, dict):
  31. return True
  32. value = voice_cfg.get("client_direct", True)
  33. except Exception:
  34. return True
  35. if isinstance(value, bool):
  36. return value
  37. if isinstance(value, str):
  38. return value.strip().lower() not in {"0", "false", "no", "off"}
  39. return True
  40. def _relay(reason: str) -> dict[str, Any]:
  41. """A relay verdict that tells the client WHY, without secrets."""
  42. return {"mode": "relay", "reason": reason}
  43. def _section(config: Any, provider: str) -> dict[str, Any]:
  44. """The provider's own sub-dict of an STT/TTS config, shape-guarded."""
  45. section = config.get(provider) if isinstance(config, dict) else None
  46. return section if isinstance(section, dict) else {}
  47. def _direct(wire: str, provider: str, base_url: Any, api_key: str, model: Any, **extra: Any) -> dict[str, Any]:
  48. return {"mode": "direct", "wire": wire, "provider": provider, "base_url": base_url,
  49. "api_key": api_key, "model": model, **extra}
  50. def stt_hallucination_filter() -> dict[str, Any]:
  51. """The Whisper-silence hallucination contract the relay path applies
  52. (``transcribe_recording`` → ``is_whisper_hallucination``), shipped to the
  53. client so a client-direct transcription agrees with a relayed one instead
  54. of submitting "thank you" on silence as a real turn."""
  55. from tools.voice_mode_transcript import WHISPER_HALLUCINATIONS
  56. return {
  57. "phrases": sorted(WHISPER_HALLUCINATIONS),
  58. # Python's _HALLUCINATION_REPEAT_RE (IGNORECASE) for repetitive filler
  59. # like "OK. OK. OK." — a JS regex source, so a single backslash.
  60. "repeat_regex": "^(?:thank you|thanks|bye|you|ok|okay|the end|[.,!\\s])+$",
  61. }
  62. def _deepinfra_model(section: dict[str, Any], kind: str) -> Optional[str]:
  63. """Configured model, else the first catalog model of ``kind`` (stt/tts)."""
  64. from hermes_cli.models import deepinfra_model_ids
  65. return section.get("model") or next(iter(deepinfra_model_ids(kind)), None)
  66. # ── STT ──
  67. # provider -> (env var, default-model attr on transcription_common, base_url).
  68. # ``base_url`` is a transcription_common attr name or a literal URL.
  69. _STT_KEYED: dict[str, tuple[str, str, str]] = {
  70. "groq": ("GROQ_API_KEY", "DEFAULT_GROQ_STT_MODEL", "GROQ_BASE_URL"),
  71. "mistral": ("MISTRAL_API_KEY", "DEFAULT_MISTRAL_STT_MODEL", "https://api.mistral.ai/v1"),
  72. }
  73. def _resolve_stt_client_config() -> dict[str, Any]:
  74. from tools import transcription_common as tc
  75. from tools import transcription_tools as tt
  76. stt_config = tt._load_stt_config()
  77. if not tt.is_stt_enabled(stt_config):
  78. return _relay("stt disabled")
  79. provider = tt._get_provider(stt_config)
  80. # Server-host-only: local whisper, the env-var command escape hatch,
  81. # declared command providers, and anything plugin-registered.
  82. if tt._is_local_stt_provider(provider, stt_config):
  83. return _relay("local provider")
  84. if provider not in tc.BUILTIN_STT_PROVIDERS:
  85. return _relay("command/plugin provider")
  86. language = tt._resolve_stt_language(
  87. provider, stt_config, extra_keys=("language_code",) if provider == "elevenlabs" else ())
  88. section = _section(stt_config, provider)
  89. # Same deadline the gateway's own transcription client applies
  90. # (``stt.openai.timeout``; riders such as groq/deepinfra inherit it), so a
  91. # slow endpoint fails the Desktop's direct request instead of hanging it.
  92. timeout_s = tc._config_number(_section(stt_config, "openai"), "timeout", 60.0)
  93. def direct(wire: str, base_url: Any, api_key: str, model: Any) -> dict[str, Any]:
  94. return _direct(wire, provider, base_url, api_key, model, language=language, timeout_s=timeout_s,
  95. hallucination_filter=stt_hallucination_filter())
  96. def env_base_url(env_var: str, default: str) -> str:
  97. from hermes_cli.config import get_env_value
  98. return str(section.get("base_url") or get_env_value(env_var) or default).strip().rstrip("/")
  99. if provider in _STT_KEYED:
  100. env_var, default_model, base = _STT_KEYED[provider]
  101. api_key = tt._resolve_provider_key(env_var, provider)
  102. if not api_key:
  103. return _relay("no credentials")
  104. return direct(STT_WIRE_OPENAI, getattr(tc, base, base), api_key,
  105. section.get("model") or getattr(tc, default_model))
  106. if provider == "openai":
  107. # Covers the Nous-managed selection too: the resolver returns the user's
  108. # own gateway token + managed base URL — exactly what the client should use.
  109. try:
  110. api_key, base_url = tt._resolve_openai_audio_client_config()
  111. except ValueError as exc:
  112. return _relay(f"openai resolution failed: {exc}")
  113. return direct(STT_WIRE_OPENAI, base_url, api_key, section.get("model") or tc.DEFAULT_STT_MODEL)
  114. if provider == "xai":
  115. # API key only: an xAI OAuth bearer refreshes server-side mid-session and
  116. # would strand the client on the first 401.
  117. from hermes_cli.config import get_env_value
  118. api_key = str(get_env_value("XAI_API_KEY") or "").strip()
  119. if not api_key:
  120. return _relay("xai oauth (server-managed) or no credentials")
  121. return direct(STT_WIRE_XAI, env_base_url("XAI_STT_BASE_URL", tc.XAI_STT_BASE_URL), api_key,
  122. tc.normalize_xai_stt_model(section.get("model")))
  123. if provider == "elevenlabs":
  124. api_key = tt._resolve_provider_key("ELEVENLABS_API_KEY", "elevenlabs")
  125. if not api_key:
  126. return _relay("no credentials")
  127. return direct(STT_WIRE_ELEVENLABS, env_base_url("ELEVENLABS_STT_BASE_URL", tc.ELEVENLABS_STT_BASE_URL),
  128. api_key, section.get("model") or tc.DEFAULT_ELEVENLABS_STT_MODEL)
  129. if provider == "deepinfra":
  130. api_key = tt._resolve_provider_key("DEEPINFRA_API_KEY", "deepinfra")
  131. if not api_key:
  132. return _relay("no credentials")
  133. from hermes_cli.models import deepinfra_base_url
  134. model = _deepinfra_model(section, "stt")
  135. if not model:
  136. return _relay("no deepinfra stt model")
  137. return direct(STT_WIRE_OPENAI, deepinfra_base_url(section), api_key, model)
  138. return _relay(f"provider {provider!r} has no client wire")
  139. # ── TTS ──
  140. def _resolve_tts_client_config() -> dict[str, Any]:
  141. from tools import tts_tool as tts
  142. from tools import tts_tool_openai, tts_tool_providers
  143. tts_config = tts._load_tts_config()
  144. provider = tts._get_provider(tts_config)
  145. if provider not in tts.BUILTIN_TTS_PROVIDERS:
  146. return _relay("command/plugin provider")
  147. # The desktop's client-direct sentence cutter honours the same tts.streaming.min_len as
  148. # the gateway/CLI chunkers, so a short CJK opener is spoken alone on every surface.
  149. from tools.tts_streaming import SentenceChunker
  150. min_len = SentenceChunker.from_config(tts_config).min_len
  151. if provider == "openai":
  152. # Covers the direct-key, custom-base_url, and Nous-managed selections.
  153. try:
  154. api_key, base_url, is_managed = tts_tool_openai._resolve_openai_audio_client_config()
  155. except ValueError as exc:
  156. return _relay(f"openai resolution failed: {exc}")
  157. oai = _section(tts_config, "openai")
  158. model = oai.get("model") or tts_tool_openai.DEFAULT_OPENAI_MODEL
  159. config_base = oai.get("base_url")
  160. base_url = config_base or base_url
  161. # The managed gateway only proxies MANAGED_OPENAI_TTS_MODELS — same
  162. # coercion text_to_speech applies server-side.
  163. if is_managed and not config_base and model not in tts_tool_openai.MANAGED_OPENAI_TTS_MODELS:
  164. model = tts_tool_openai.DEFAULT_OPENAI_MODEL
  165. speed_default = tts_config.get("speed", 1.0) if isinstance(tts_config, dict) else 1.0
  166. try:
  167. speed = float(oai.get("speed", speed_default))
  168. except (TypeError, ValueError):
  169. speed = 1.0
  170. return _direct(TTS_WIRE_OPENAI, "openai", base_url, api_key, model,
  171. voice=oai.get("voice") or tts_tool_openai.DEFAULT_OPENAI_VOICE, speed=speed,
  172. extra_body=tts_tool_openai._openai_extra_body(oai),
  173. min_len=min_len)
  174. if provider == "elevenlabs":
  175. api_key = tts._resolve_provider_key("ELEVENLABS_API_KEY", "elevenlabs")
  176. if not api_key:
  177. return _relay("no credentials")
  178. el = _section(tts_config, "elevenlabs")
  179. return _direct(TTS_WIRE_ELEVENLABS, "elevenlabs",
  180. str(el.get("base_url") or "https://api.elevenlabs.io/v1").rstrip("/"),
  181. api_key, el.get("model_id") or tts_tool_providers.DEFAULT_ELEVENLABS_MODEL_ID,
  182. voice=el.get("voice_id") or tts_tool_providers.DEFAULT_ELEVENLABS_VOICE_ID, speed=None,
  183. min_len=min_len)
  184. if provider == "deepinfra":
  185. api_key = tts._resolve_provider_key("DEEPINFRA_API_KEY", "deepinfra")
  186. if not api_key:
  187. return _relay("no credentials")
  188. from hermes_cli.models import deepinfra_base_url
  189. di = _section(tts_config, "deepinfra")
  190. model = _deepinfra_model(di, "tts")
  191. if not model:
  192. return _relay("no deepinfra tts model")
  193. return _direct(TTS_WIRE_OPENAI, "deepinfra", deepinfra_base_url(di), api_key, model,
  194. voice=di.get("voice") or "af_bella", speed=None, min_len=min_len)
  195. # edge / minimax / xai / mistral / gemini / neutts / kittentts / piper: server-host-only
  196. # engines or wire shapes the desktop doesn't speak yet; the relay path serves them.
  197. return _relay(f"provider {provider!r} has no client wire")
  198. def resolve_client_voice_config(client_direct: bool = True) -> dict[str, Any]:
  199. """Resolve both directions for the CURRENT profile scope.
  200. Callers scope the profile via ``hermes_constants.set_hermes_home_override``
  201. (the web server's ``_config_profile_scope``) before calling — identical to
  202. how ``/api/audio/transcribe`` scopes ``transcribe_recording``.
  203. ``client_direct=False`` is the serving surface's verdict (a browser tab):
  204. both directions relay and no key is resolved.
  205. """
  206. out: dict[str, Any] = {}
  207. if not client_direct:
  208. out = {"stt": _relay("browser surface"), "tts": _relay("browser surface")}
  209. elif not _client_direct_enabled():
  210. out = {"stt": _relay("voice.client_direct disabled"), "tts": _relay("voice.client_direct disabled")}
  211. else:
  212. for key, resolver in (("stt", _resolve_stt_client_config), ("tts", _resolve_tts_client_config)):
  213. try:
  214. out[key] = resolver()
  215. except Exception:
  216. logger.exception("client voice-config %s resolution failed", key.upper())
  217. out[key] = _relay("resolution error")
  218. # Live dictation always goes through the host's /api/audio/transcribe-stream socket (keys stay
  219. # server-side), independent of client_direct; the verdict above remains the blob fallback.
  220. from tools.transcription_streaming import streaming_available
  221. try:
  222. out["stt"]["streaming"] = streaming_available()
  223. except Exception:
  224. logger.debug("live STT capability probe failed", exc_info=True)
  225. return out