__init__.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439
  1. """FAL.ai video generation backend.
  2. The user picks a **model family** (e.g. "Pixverse v6"); the plugin routes to its text-to-video endpoint without
  3. ``image_url`` and to its image-to-video endpoint otherwise. Active-family precedence:
  4. tool ``model=`` → ``FAL_VIDEO_MODEL`` env → ``video_gen.fal.model`` → ``video_gen.model`` (family id or an endpoint
  5. path containing one) → ``DEFAULT_MODEL``. Auth via ``FAL_KEY`` or the managed Nous gateway; output is an HTTPS URL.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import threading
  10. import uuid
  11. from typing import Any, Dict, List, Optional, Tuple
  12. from agent.video_gen_provider import VideoGenProvider, error_response, success_response
  13. logger = logging.getLogger(__name__)
  14. # Family catalog. Capability flags gate which keys reach the payload — keys a family doesn't advertise are never sent (the
  15. # managed gateway forwards everything verbatim). Enums default to None (endpoint decides), flags to False. ``durations`` is always a
  16. # ``(min, max)`` range (clamp); a family whose endpoint only accepts discrete values adds ``duration_enum`` (snap to nearest; None
  17. # stays None so the endpoint default applies); ``duration_cap_by_resolution`` lowers the ceiling per resolution enum (applied after
  18. # the snap/clamp). Extras: audio_native (always on; description line only),
  19. # duration_int (JSON int, default queue-API string), duration_suffix ("4s"), image_param_key (i2v key when not `image_url`),
  20. # image_drop_keys (i2v endpoint rejects), audio_param_key (toggle key when not `generate_audio`), resolution_aliases (tool value → endpoint enum), static_payload (always required).
  21. def _family(display: str, speed: str, tier: str, strengths: str, text: Optional[str], image: str, **caps: Any) -> dict[str, Any]:
  22. return {"display": display, "speed": speed, "price": tier, "tier": tier, "strengths": strengths, "text_endpoint": text, "image_endpoint": image,
  23. "aspect_ratios": None, "resolutions": None, "durations": None, "audio": False, "negative": False, "seed": False, **caps}
  24. _SIX_ASPECTS = ("21:9", "16:9", "4:3", "1:1", "3:4", "9:16")
  25. # MiniMax H3 uses capitalized/2K-style resolution enums; aliases map the tool's usual values. Max tops out at 768P.
  26. _H3_ALIASES = {"480p": "768P", "540p": "768P", "720p": "768P", "768p": "768P", "1080p": "2K", "2k": "2K", "4k": "4K", "2160p": "4K"}
  27. _H3_MAX_ALIASES = {"480p": "480P", "540p": "480P", "720p": "768P", "768p": "768P", "1080p": "768P", "2k": "768P", "4k": "768P", "2160p": "768P"}
  28. _H3_MAX_TURBO_ALIASES = {"480p": "480P", "540p": "480P", "720p": "768P", "768p": "768P", "1080p": "1080P", "2k": "1080P", "4k": "1080P", "2160p": "1080P"}
  29. FAL_FAMILIES: dict[str, dict[str, Any]] = {
  30. # ─── Cheap / fast tier ─────────────────────────────────────────────
  31. "ltx-2.3": _family("LTX 2.3 (22B)", "~30-60s", "cheap", "22B model with native audio generation. Affordable.", # docs expose no enums
  32. "fal-ai/ltx-2.3-22b/text-to-video", "fal-ai/ltx-2.3-22b/image-to-video", audio=True, negative=True, seed=True),
  33. # Fast endpoints: duration is an integer enum (6..20 even) — snapped, sent as JSON int; i2v ladders 720p→2160p; no `seed` key.
  34. "ltx-2.5": _family("LTX 2.5", "~30-90s", "cheap", "Lightricks open-source audio-video model. Native audio, up to 20s / 4K (i2v), camera-motion presets.",
  35. "lightricks/ltx-2.5/text-to-video/fast", "lightricks/ltx-2.5/image-to-video/fast", duration_int=True, aspect_ratios=("16:9", "9:16"),
  36. resolutions=("720p", "1080p", "1440p", "2160p"), resolution_aliases={"2k": "1440p", "4k": "2160p"},
  37. durations=(6, 20), duration_enum=tuple(range(6, 21, 2)), duration_cap_by_resolution={"1440p": 10, "2160p": 10}, audio=True),
  38. "pixverse-v6": _family("Pixverse v6", "~30-90s", "cheap", "Affordable. Negative prompts. 1-15s durations.", "fal-ai/pixverse/v6/text-to-video",
  39. "fal-ai/pixverse/v6/image-to-video", resolutions=("360p", "540p", "720p", "1080p"), durations=(1, 15), audio=True, negative=True, seed=True),
  40. "seedance-2.0-mini": _family("Seedance 2.0 Mini", "~30-90s", "cheap", "ByteDance. Faster/cheaper Seedance tier, audio + lip-sync, 4-15s.",
  41. "bytedance/seedance-2.0/mini/text-to-video", "bytedance/seedance-2.0/mini/image-to-video", aspect_ratios=_SIX_ASPECTS,
  42. resolutions=("480p", "720p"), durations=(4, 15), audio=True),
  43. # ─── Expensive / premium tier ──────────────────────────────────────
  44. "veo3.1": _family("Veo 3.1", "~60-120s", "premium", "Google DeepMind. Cinematic, native audio, strong prompt adherence.", "fal-ai/veo3.1",
  45. "fal-ai/veo3.1/image-to-video", aspect_ratios=("16:9", "9:16"), resolutions=("720p", "1080p", "4k"), durations=(4, 8), duration_enum=(4, 6, 8),
  46. duration_suffix="s", audio=True, negative=True, seed=True), # wants "4s" not "4"
  47. "seedance-2.0": _family("Seedance 2.0", "~60-120s", "premium", "ByteDance. Cinematic, synchronized audio + lip-sync, 4-15s.", # no "auto" aspect, no `seed`
  48. "bytedance/seedance-2.0/text-to-video", "bytedance/seedance-2.0/image-to-video", aspect_ratios=_SIX_ASPECTS,
  49. resolutions=("480p", "720p", "1080p"), durations=(4, 15), audio=True),
  50. "seedance-2.5": _family("Seedance 2.5", "~60-180s", "premium", "ByteDance flagship. Native 30s single-pass, audio in the same latent space, lip-sync.",
  51. "bytedance/seedance-2.5/text-to-video", "bytedance/seedance-2.5/image-to-video", aspect_ratios=_SIX_ASPECTS,
  52. image_drop_keys=("aspect_ratio",), resolutions=("480p", "720p"), durations=(4, 30), audio=True), # i2v aspect is "auto" only
  53. "minimax-h3": _family("MiniMax H3", "~60-180s", "premium", "MiniMax frontier. Native 2K (up to 4K), 5-15s, seven aspect ratios.",
  54. "minimax/h3/text-to-video", "minimax/h3/image-to-video", duration_int=True, image_drop_keys=("aspect_ratio",), # i2v follows image
  55. aspect_ratios=_SIX_ASPECTS, resolutions=("768P", "2K", "4K"), resolution_aliases=_H3_ALIASES, durations=(5, 15), audio_native=True),
  56. # i2v schema doesn't declare aspect_ratio; unlike base H3, Max declares `seed` on both endpoints; static key is in the schema's required array.
  57. "minimax-h3-max": _family("MiniMax H3 Max (fal post-train)", "~5-30s", "premium", "fal's post-trained MiniMax H3. Top-ranked quality/prompt "
  58. "adherence/aesthetics, 768p in seconds, 5-15s.", "minimax/h3-max/text-to-video", "minimax/h3-max/image-to-video",
  59. duration_int=True, image_drop_keys=("aspect_ratio",), aspect_ratios=_SIX_ASPECTS, resolutions=("480P", "768P"),
  60. resolution_aliases=_H3_MAX_ALIASES, durations=(5, 15), static_payload={"prompt_expansion_mode": "balanced"}, audio_native=True, seed=True),
  61. # Same schema shape as Max (required prompt_expansion_mode, no i2v aspect_ratio) but adds a 1080P tier and an end_image_url
  62. # the tool surface doesn't expose; throughput-tuned so it's the fastest premium H3 tier ($0.025-0.08/s list).
  63. "minimax-h3-max-turbo": _family("MiniMax H3 Max Turbo (fal post-train)", "~5-20s", "premium", "fal's throughput-tuned H3 Max variant. Near-Max "
  64. "quality at a fraction of the price/latency, 480P-1080P, 5-15s.", "minimax/h3-max-turbo/text-to-video",
  65. "minimax/h3-max-turbo/image-to-video", duration_int=True, image_drop_keys=("aspect_ratio",), aspect_ratios=_SIX_ASPECTS,
  66. resolutions=("480P", "768P", "1080P"), resolution_aliases=_H3_MAX_TURBO_ALIASES, durations=(5, 15),
  67. static_payload={"prompt_expansion_mode": "balanced"}, audio_native=True, seed=True),
  68. "flux-3": _family("FLUX 3 (via FAL)", "~60-120s", "premium", "Black Forest Labs frontier video. Native audio, 5-20s, 8 aspect ratios.",
  69. "blackforestlabs/flux-3/text-to-video", "blackforestlabs/flux-3/image-to-video", duration_int=True, # enum "auto" | 5..20 ints
  70. aspect_ratios=("21:9", "2:1", "16:9", "4:3", "1:1", "3:4", "9:16"), resolutions=("720p", "1080p"), durations=(5, 20), audio=True),
  71. "grok-imagine-1.5": _family("Grok Imagine 1.5 (via FAL)", "~30-90s", "premium", "xAI. Fast stylized video with audio, 1-15s, cheap per second.",
  72. "xai/grok-imagine-video/v1.5/text-to-video", "xai/grok-imagine-video/v1.5/image-to-video", duration_int=True,
  73. image_drop_keys=("aspect_ratio",), aspect_ratios=("16:9", "4:3", "3:2", "1:1", "2:3", "3:4", "9:16"), # aspect is t2v-only
  74. resolutions=("480p", "720p", "1080p"), durations=(1, 15), audio_native=True),
  75. # v1.1 (Aug 2026) added text-to-video and a 360p-4k resolution enum; v1.0 was image-only.
  76. "gemini-omni-flash": _family("Gemini Omni Flash 1.1 (via FAL)", "~60-120s", "premium", "Google. Text & image to video with native audio, physics-grounded motion, up to 4K, 3-10s.",
  77. "google/gemini-omni-flash/v1.1/text-to-video", "google/gemini-omni-flash/v1.1/image-to-video", duration_int=True,
  78. aspect_ratios=("16:9", "9:16"), resolutions=("360p", "720p", "1080p", "4k"), durations=(3, 10), audio_native=True),
  79. # Kling 3.0 core tiers: t2v declares aspect_ratio, i2v derives it from `start_image_url`; string duration enum "3".."15";
  80. # generate_audio is a real toggle (default on, audio-on costs more); no resolution or seed keys in the v3 schemas.
  81. "kling-v3": _family("Kling 3.0 (Standard)", "~60-180s", "premium", "Kuaishou frontier core model. Cinematic motion, native audio, 3-15s.",
  82. "fal-ai/kling-video/v3/standard/text-to-video", "fal-ai/kling-video/v3/standard/image-to-video", image_param_key="start_image_url",
  83. image_drop_keys=("aspect_ratio",), aspect_ratios=("16:9", "9:16", "1:1"), durations=(3, 15), audio=True, negative=True),
  84. "kling-v3-pro": _family("Kling 3.0 Pro", "~60-180s", "premium", "Kling 3.0 top quality tier. Cinematic motion, native audio, 3-15s.",
  85. "fal-ai/kling-video/v3/pro/text-to-video", "fal-ai/kling-video/v3/pro/image-to-video", image_param_key="start_image_url",
  86. image_drop_keys=("aspect_ratio",), aspect_ratios=("16:9", "9:16", "1:1"), durations=(3, 15), audio=True, negative=True),
  87. "kling-v3-4k": _family("Kling v3 4K", "~120-300s", "premium", "4K output, native audio (Chinese/English), 3-15s.", "fal-ai/kling-video/v3/4k/text-to-video",
  88. "fal-ai/kling-video/v3/4k/image-to-video", image_param_key="start_image_url", aspect_ratios=("16:9", "9:16", "1:1"),
  89. durations=(3, 15), audio=True, negative=True, seed=True),
  90. # Kling O3: t2v declares aspect_ratio, i2v derives it from the image; string duration 3-15; generate_audio is a real toggle
  91. # (default off, audio-on costs more); no resolution or seed keys in the O3 schema.
  92. "kling-o3": _family("Kling O3 (Standard)", "~60-180s", "premium", "Kuaishou frontier. Multi-shot native storytelling, optional audio, 3-15s.",
  93. "fal-ai/kling-video/o3/standard/text-to-video", "fal-ai/kling-video/o3/standard/image-to-video",
  94. image_drop_keys=("aspect_ratio",), aspect_ratios=("16:9", "9:16", "1:1"), durations=(3, 15), audio=True),
  95. # Wan 3.0: i2v takes `start_image_url`; both modalities accept aspect_ratio (schema default "adaptive" is left to the endpoint);
  96. # integer duration 2-30 (None = smart duration); the audio toggle key is `audio`, not `generate_audio`; `seed` on both endpoints.
  97. "wan-3.0": _family("Wan 3.0", "~60-180s", "premium", "Alibaba latest gen. 2-30s clips, native audio, up to 1080p, lip-sync.",
  98. "alibaba/wan-3.0/text-to-video", "alibaba/wan-3.0/image-to-video", image_param_key="start_image_url", duration_int=True,
  99. audio_param_key="audio", aspect_ratios=("16:9", "4:3", "1:1", "3:4", "9:16"), resolutions=("480p", "720p", "1080p"),
  100. durations=(2, 30), audio=True, seed=True),
  101. "wan-3.0-prime": _family("Wan 3.0 Prime", "~60-180s", "premium", "Alibaba premium tier. Faster iteration, higher fidelity, 2-30s, native audio.",
  102. "alibaba/wan-3.0-prime/text-to-video", "alibaba/wan-3.0-prime/image-to-video", image_param_key="start_image_url",
  103. duration_int=True, audio_param_key="audio", aspect_ratios=("16:9", "4:3", "1:1", "3:4", "9:16"),
  104. resolutions=("480p", "720p", "1080p"), durations=(2, 30), audio=True, seed=True),
  105. # Wan 2.7: integer duration enum 2..15, aspect_ratio on t2v only (i2v follows the image), `seed` + `negative_prompt` on both;
  106. # audio is auto-generated when no audio_url is given (no toggle key).
  107. "wan-2.7": _family("Wan 2.7", "~60-180s", "premium", "Alibaba. 1080p, auto audio, multi-shot prompts, 2-15s.",
  108. "fal-ai/wan/v2.7/text-to-video", "fal-ai/wan/v2.7/image-to-video", duration_int=True, image_drop_keys=("aspect_ratio",),
  109. aspect_ratios=("16:9", "9:16", "1:1", "4:3", "3:4"), resolutions=("720p", "1080p"), durations=(2, 15),
  110. audio_native=True, negative=True, seed=True),
  111. # v1.1 publishes the full schema: integer duration 3-15, nine aspect ratios (t2v only — i2v follows the image), 720p/1080p,
  112. # `seed` on both endpoints, audio always on (no generate_audio key).
  113. "happy-horse": _family("Happy Horse 1.1", "~60-120s", "premium", "Alibaba flagship. 1080p, native audio + multilingual lip-sync, 3-15s, nine aspect ratios.",
  114. "alibaba/happy-horse/v1.1/text-to-video", "alibaba/happy-horse/v1.1/image-to-video", duration_int=True,
  115. image_drop_keys=("aspect_ratio",), aspect_ratios=("16:9", "9:16", "1:1", "4:3", "3:4", "21:9", "9:21", "5:4", "4:5"),
  116. resolutions=("720p", "1080p"), durations=(3, 15), audio_native=True, seed=True),
  117. }
  118. DEFAULT_MODEL = "pixverse-v6" # cheap, both modalities, sane defaults
  119. def _clamp_duration(family: dict[str, Any], duration: Optional[int], resolution: Optional[str] = None) -> Optional[int]:
  120. """Snap to the nearest ``duration_enum`` entry when the family declares one, else clamp into the ``durations``
  121. ``(min, max)`` range; None stays None (endpoint default). A ``duration_cap_by_resolution`` ceiling for the resolved
  122. *resolution* is applied last (fal rejects LTX 2.5 >10s at 1440p/2160p)."""
  123. if duration is None:
  124. return None
  125. enum = family.get("duration_enum")
  126. if enum:
  127. clamped = min(enum, key=lambda d: abs(d - duration))
  128. else:
  129. lo, hi = family["durations"]
  130. clamped = max(lo, min(hi, duration))
  131. cap = (family.get("duration_cap_by_resolution") or {}).get(resolution)
  132. return clamped if cap is None else min(clamped, cap)
  133. def _modalities(meta: dict[str, Any]) -> list[str]:
  134. return [m for m in ("text", "image") if meta[f"{m}_endpoint"]]
  135. def _normalize_family_key(c: str) -> Optional[str]:
  136. """Known family ID from a bare id, full endpoint path, truncated stem (``minimax/h3``) or provider-prefixed name."""
  137. c = c.strip()
  138. if not c or c in FAL_FAMILIES:
  139. return c or None
  140. endpoints = [(fid, ep) for fid, meta in FAL_FAMILIES.items()
  141. for ep in (meta["text_endpoint"], meta["image_endpoint"]) if isinstance(ep, str)]
  142. # Exact declared endpoint beats any segment scan (which would see "seedance-2.0" inside ".../seedance-2.0/mini/...").
  143. # Truncated stem: the segment after ``c`` must be a modality leaf so "bytedance/seedance-2.0" skips Mini's deeper path.
  144. # Last resort: longest family-id path-segment match (prefers seedance-2.0-mini over seedance-2.0 when both appear).
  145. hit = ([fid for fid, ep in endpoints if c == ep]
  146. or [fid for fid, ep in endpoints if ep.startswith(c + "/") and ep[len(c) + 1:].split("/", 1)[0] in ("text-to-video", "image-to-video")]
  147. or sorted((fid for fid in FAL_FAMILIES if fid in c.split("/")), key=len, reverse=True))
  148. return hit[0] if hit else None
  149. def _resolve_family(explicit: Optional[str]) -> tuple[str, dict[str, Any]]:
  150. """Decide which FAL family to use. Returns ``(family_id, meta)``."""
  151. import os
  152. try:
  153. from hermes_cli.config import load_config
  154. cfg = load_config()
  155. except Exception as exc:
  156. logger.debug("Could not load video_gen config: %s", exc)
  157. cfg = None
  158. cfg = cfg.get("video_gen") if isinstance(cfg, dict) else None
  159. cfg = cfg if isinstance(cfg, dict) else {}
  160. fal_cfg = cfg.get("fal") if isinstance(cfg.get("fal"), dict) else {}
  161. for c in (explicit, os.environ.get("FAL_VIDEO_MODEL"), fal_cfg.get("model"), cfg.get("model")):
  162. fid = _normalize_family_key(c) if isinstance(c, str) else None
  163. if fid:
  164. return fid, FAL_FAMILIES[fid]
  165. return DEFAULT_MODEL, FAL_FAMILIES[DEFAULT_MODEL]
  166. def _build_payload(family: dict[str, Any], *, prompt: str, image_url: Optional[str], duration: Optional[int], aspect_ratio: str,
  167. resolution: str, negative_prompt: Optional[str], audio: Optional[bool], seed: Optional[int]) -> dict[str, Any]:
  168. """Build a family-specific payload, dropping keys the family doesn't declare (unsupported enums → endpoint default)."""
  169. resolved = (family.get("resolution_aliases") or {}).get((resolution or "").lower(), resolution)
  170. clamped = _clamp_duration(family, duration, resolved) if family["durations"] else None
  171. payload: dict[str, Any] = {key: value for ok, key, value in (
  172. (prompt, "prompt", prompt),
  173. (image_url, family.get("image_param_key") or "image_url", image_url),
  174. # Newer endpoints declare no `seed` and the managed gateway forwards whatever we send — gate on the family.
  175. (seed is not None and family.get("seed", True), "seed", seed),
  176. (family["aspect_ratios"] and aspect_ratio in family["aspect_ratios"], "aspect_ratio", aspect_ratio),
  177. (family["resolutions"] and resolved in family["resolutions"], "resolution", resolved),
  178. # FAL's queue API types duration as a string ("8" not 8) unless the family says int; veo3.1 also wants a unit suffix.
  179. (clamped is not None, "duration", clamped if family.get("duration_int") else f"{clamped}{family.get('duration_suffix', '')}"),
  180. (family["audio"] and audio is not None, family.get("audio_param_key") or "generate_audio", bool(audio)), # Wan 3.0 calls it `audio`
  181. (family["negative"] and negative_prompt, "negative_prompt", negative_prompt),
  182. ) if ok}
  183. for key in family.get("image_drop_keys", ()) if image_url else (): # keys the i2v endpoint rejects outright
  184. payload.pop(key, None)
  185. for key, value in (family.get("static_payload") or {}).items(): # constants the endpoint always requires
  186. payload.setdefault(key, value)
  187. return payload
  188. def _video_url_from_result(result: Any) -> tuple[Any, Optional[str]]:
  189. """Return ``(video_field, url)`` from a FAL result dict (url None if absent)."""
  190. video = result.get("video") if isinstance(result, dict) else None
  191. url = video.get("url") if isinstance(video, dict) else video if isinstance(video, str) else None
  192. return video, url or None
  193. _fal_client: Any = None
  194. _fal_client_lock = threading.Lock()
  195. # ---------------------------------------------------------------------------
  196. # Managed FAL gateway (Nous Subscription)
  197. # ---------------------------------------------------------------------------
  198. _managed_fal_video_client: Any = None
  199. _managed_fal_video_client_config: Any = None
  200. _managed_fal_video_client_lock = threading.Lock()
  201. def _load_fal_client() -> Any:
  202. """Lazy-load ``fal_client`` once via ``tools.fal_common``."""
  203. global _fal_client
  204. with _fal_client_lock:
  205. if _fal_client is None:
  206. from tools.fal_common import import_fal_client
  207. _fal_client = import_fal_client()
  208. return _fal_client
  209. def _resolve_managed_fal_video_gateway():
  210. """Resolve the FAL video route from the stored ``video_gen`` selection: ``"nous"`` → managed only (unentitled ⇒
  211. selection-naming error); other stored provider → direct only (missing FAL_KEY ⇒ error); never-configured → autodetect."""
  212. from tools.managed_tool_gateway import resolve_managed_tool_gateway
  213. from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, fal_key_is_configured, read_selection, selection_error
  214. selected = read_selection("video_gen")
  215. if selected == NOUS_MANAGED_PROVIDER:
  216. gateway = resolve_managed_tool_gateway("fal-queue")
  217. if gateway is None:
  218. raise ValueError(selection_error("video_gen", NOUS_MANAGED_PROVIDER, "the Nous Tool Gateway is not available (not entitled or unreachable)"))
  219. return gateway
  220. if selected is not None:
  221. if not fal_key_is_configured():
  222. raise ValueError(selection_error("video_gen", selected, "FAL_KEY is not set"))
  223. return None
  224. return None if fal_key_is_configured() else resolve_managed_tool_gateway("fal-queue")
  225. def _fal_video_available() -> bool:
  226. """True if the selected (or, never-configured, any) FAL backend is reachable; raises on a stored-but-broken selection."""
  227. from tools.tool_backend_helpers import fal_key_is_configured
  228. return _resolve_managed_fal_video_gateway() is not None or fal_key_is_configured()
  229. def _get_managed_fal_video_client(managed_gateway):
  230. """Reuse the managed FAL client so its internal httpx.Client is not leaked per call."""
  231. global _managed_fal_video_client, _managed_fal_video_client_config
  232. from tools.fal_common import _ManagedFalSyncClient
  233. client_config = (managed_gateway.gateway_origin.rstrip("/"), managed_gateway.nous_user_token)
  234. with _managed_fal_video_client_lock:
  235. if _managed_fal_video_client is None or _managed_fal_video_client_config != client_config:
  236. _managed_fal_video_client = _ManagedFalSyncClient(_load_fal_client(), key=managed_gateway.nous_user_token,
  237. queue_run_origin=managed_gateway.gateway_origin)
  238. _managed_fal_video_client_config = client_config
  239. return _managed_fal_video_client
  240. def _submit_fal_video_request(endpoint: str, arguments: dict[str, Any]):
  241. """Submit via direct credentials or the managed queue gateway; ``wait_fal_queue_result`` blocks."""
  242. client = _load_fal_client()
  243. headers = {"x-idempotency-key": str(uuid.uuid4())}
  244. managed_gateway = _resolve_managed_fal_video_gateway()
  245. if managed_gateway is None:
  246. return client.submit(endpoint, arguments=arguments, headers=headers)
  247. from tools.fal_common import (
  248. _extract_http_status, _managed_fal_billing_error, submit_managed_fal_with_rate_limit_retry,
  249. )
  250. try:
  251. return submit_managed_fal_with_rate_limit_retry(
  252. lambda request_headers: _get_managed_fal_video_client(managed_gateway).submit(
  253. endpoint, arguments=arguments, headers=request_headers),
  254. what="video endpoint", name=endpoint)
  255. except Exception as exc:
  256. status = _extract_http_status(exc)
  257. if status is not None and 400 <= status < 500:
  258. billing = _managed_fal_billing_error(exc, "endpoint")
  259. if billing is not None:
  260. raise ValueError(
  261. f"Nous Subscription gateway rejected endpoint '{endpoint}' (HTTP {status}): {billing}") from exc
  262. raise ValueError(f"Nous Subscription gateway rejected endpoint '{endpoint}' (HTTP {status}). This model may not yet be enabled "
  263. f"on the Nous Portal's FAL proxy. Either:\n • Set FAL_KEY in your environment to use FAL.ai directly, or\n"
  264. f" • Pick a different model via `hermes tools` → Video Generation.") from exc
  265. raise
  266. # ByteDance SeedVR2 on FAL: $0.001/megapixel of output; a 5s 720p→1440p 2x pass is roughly $0.44.
  267. UPSCALER_ENDPOINT = "fal-ai/seedvr/upscale/video"
  268. UPSCALER_FACTOR = 2
  269. def _upscale_video(video_url: str, source_request_id: Optional[str] = None) -> Optional[str]:
  270. """Best-effort SeedVR2 upscale; returns the new URL or None (never raises)."""
  271. try:
  272. logger.info("Upscaling video with SeedVR2 (%dx)...", UPSCALER_FACTOR)
  273. arguments: dict[str, Any] = {"video_url": video_url, "upscale_mode": "factor", "upscale_factor": UPSCALER_FACTOR}
  274. if _resolve_managed_fal_video_gateway() is not None:
  275. if not source_request_id:
  276. raise RuntimeError("Managed SeedVR upscale requires the source FAL request id")
  277. arguments["source_request_id"] = source_request_id
  278. from tools.fal_common import wait_fal_queue_result
  279. result = wait_fal_queue_result(_submit_fal_video_request(UPSCALER_ENDPOINT, arguments))
  280. except Exception as exc:
  281. logger.warning("Video upscale failed: %s", exc)
  282. return None
  283. _video, url = _video_url_from_result(result)
  284. if not url:
  285. logger.warning("Video upscaler returned no URL")
  286. return url
  287. _NO_BACKEND_MSG = ("No FAL backend available. Either set FAL_KEY (run `hermes tools` → Video Generation → FAL to configure) "
  288. "or sign in to Nous (`hermes setup`) for managed gateway access.")
  289. _MODALITY_MISSING_MSG = {
  290. "image": "FAL family {fid} has no image-to-video endpoint. Pick a family with image-to-video support via `hermes tools` → Video Generation.",
  291. "text": "FAL family {fid} has no text-to-video endpoint. Pass an image_url to use its image-to-video endpoint, or pick a different family.",
  292. }
  293. def _fal_error(error: str, error_type: str, prompt: str, model: str = "", aspect_ratio: str = "") -> dict[str, Any]:
  294. return error_response(error=error, error_type=error_type, provider="fal", model=model, prompt=prompt, aspect_ratio=aspect_ratio)
  295. class FALVideoGenProvider(VideoGenProvider):
  296. """FAL.ai multi-family backend; routes t2v/i2v on ``image_url`` presence."""
  297. name = "fal"
  298. display_name = "FAL"
  299. def is_available(self) -> bool:
  300. # A stored-but-broken selection raises the selection-naming ValueError; report unavailable, never break the picker.
  301. try:
  302. return _fal_video_available()
  303. except Exception:
  304. return False
  305. def list_models(self) -> list[dict[str, Any]]:
  306. return [{"id": fid, **{k: meta[k] for k in ("display", "speed", "strengths", "price", "tier")}, "modalities": _modalities(meta),
  307. **({"min_duration": min(d), "max_duration": max(d)} if (d := meta["durations"]) else {})}
  308. for fid, meta in FAL_FAMILIES.items()]
  309. def default_model(self) -> Optional[str]:
  310. return DEFAULT_MODEL
  311. def get_setup_schema(self) -> dict[str, Any]:
  312. return {"name": "FAL", "badge": "paid", "env_vars": [{"key": "FAL_KEY", "prompt": "FAL.ai API key", "url": "https://fal.ai/dashboard/keys"}],
  313. "tag": "LTX 2.3/2.5, Pixverse, Seedance 2.0/2.5/Mini, Veo 3.1, MiniMax H3, FLUX 3, Kling 3.0/4K/O3, Wan 2.7/3.0, Happy Horse, Grok Imagine, "
  314. "Gemini Omni — text-to-video & image-to-video"}
  315. def capabilities(self) -> dict[str, Any]:
  316. # RESOLVED family's surface so the dynamic tool schema gates params on what the selected model honors; union fallback (never raises).
  317. try:
  318. # Falls back to the cross-family union if resolution fails (never raises). See #97057.
  319. _family_id, family = _resolve_family(None)
  320. except Exception:
  321. family = None
  322. if family:
  323. durations = family["durations"] or (1, 1)
  324. return {"modalities": _modalities(family) or ["text"], "aspect_ratios": list(family["aspect_ratios"] or []),
  325. "resolutions": list(family["resolutions"] or []), "max_duration": max(durations), "min_duration": min(durations),
  326. "supports_audio": bool(family["audio"]), "audio_always_on": bool(family.get("audio_native")), # no toggle: description only
  327. "supports_negative_prompt": bool(family["negative"]), "supports_seed": bool(family["seed"]),
  328. "supports_upscale": True, "max_reference_images": 0} # SeedVR chains for any family
  329. spans = [m["durations"] for m in FAL_FAMILIES.values() if m["durations"]]
  330. return {"modalities": ["text", "image"], "aspect_ratios": ["16:9", "9:16", "1:1"], "resolutions": ["360p", "540p", "720p", "1080p"],
  331. "max_duration": max([1] + [max(d) for d in spans]), "min_duration": min([min(d) for d in spans], default=1),
  332. "supports_audio": True, "supports_negative_prompt": True, "supports_seed": True, "supports_upscale": True, "max_reference_images": 0}
  333. def generate(
  334. self, prompt: str, *, model: Optional[str] = None, image_url: Optional[str] = None, reference_image_urls: Optional[list[str]] = None,
  335. duration: Optional[int] = None, aspect_ratio: str = "16:9", resolution: str = "720p", negative_prompt: Optional[str] = None,
  336. audio: Optional[bool] = None, seed: Optional[int] = None, upscale: Optional[bool] = None, **kwargs: Any,
  337. ) -> dict[str, Any]:
  338. try: # a stored selection that cannot run gets the honest selection-naming error from the strict resolver
  339. if not _fal_video_available():
  340. return _fal_error(_NO_BACKEND_MSG, "auth_required", prompt)
  341. except ValueError as exc:
  342. return _fal_error(str(exc), "auth_required", prompt)
  343. try:
  344. _load_fal_client()
  345. except ImportError:
  346. return _fal_error("fal_client Python package not installed (pip install fal-client)", "missing_dependency", prompt)
  347. prompt = (prompt or "").strip()
  348. family_id, family = _resolve_family(model)
  349. image_url_norm = (image_url or "").strip() or None
  350. modality_used = "image" if image_url_norm else "text" # routes to the i2v vs t2v endpoint
  351. endpoint = family[f"{modality_used}_endpoint"]
  352. if not endpoint:
  353. return _fal_error(_MODALITY_MISSING_MSG[modality_used].format(fid=family_id), "modality_unsupported", prompt, model=family_id)
  354. if not prompt:
  355. return _fal_error("prompt is required.", "missing_prompt", prompt, model=family_id)
  356. payload = _build_payload(family, prompt=prompt, image_url=image_url_norm, duration=duration, aspect_ratio=aspect_ratio,
  357. resolution=resolution, negative_prompt=negative_prompt, audio=audio, seed=seed)
  358. try:
  359. from tools.fal_common import wait_fal_queue_result
  360. handle = _submit_fal_video_request(endpoint, payload)
  361. source_request_id = getattr(handle, "request_id", None)
  362. video, url = _video_url_from_result(wait_fal_queue_result(handle))
  363. except Exception as exc:
  364. logger.warning("FAL video gen failed (family=%s, endpoint=%s): %s", family_id, endpoint, exc, exc_info=True)
  365. return _fal_error(f"FAL video generation failed: {exc}", "api_error", prompt, model=family_id, aspect_ratio=aspect_ratio)
  366. if not url:
  367. return _fal_error("FAL returned no video URL in response", "empty_response", prompt, model=family_id)
  368. # Optional SeedVR2 pass — explicit opt-in, best-effort: failure falls back to the native video.
  369. upscaled_url = _upscale_video(url, source_request_id) if upscale else None
  370. upscaled = bool(upscaled_url)
  371. if upscale and not upscaled:
  372. logger.warning("Video upscale pass failed — returning native-resolution video")
  373. url = upscaled_url or url
  374. extra: dict[str, Any] = {"endpoint": endpoint, "upscaled": upscaled, **({"upscale_factor": UPSCALER_FACTOR} if upscaled else {})}
  375. if isinstance(video, dict): # native-resolution file_size no longer applies after an upscale
  376. extra.update({k: video[k] for k in (("content_type",) if upscaled else ("file_size", "content_type")) if video.get(k)})
  377. return success_response(
  378. video=url, model=family_id, prompt=prompt, modality=modality_used, provider="fal", extra=extra,
  379. aspect_ratio=aspect_ratio if "aspect_ratio" in payload else "",
  380. duration=int("".join(c for c in str(payload["duration"]) if c.isdigit()) or "0") if "duration" in payload else 0,
  381. )
  382. def register(ctx) -> None:
  383. """Plugin entry point — wire ``FALVideoGenProvider`` into the registry."""
  384. ctx.register_video_gen_provider(FALVideoGenProvider())