__init__.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354
  1. """OpenRouter video generation backend.
  2. OpenRouter fronts every major video model (Veo, Sora, Kling, Seedance, Wan, Hailuo, Grok
  3. Imagine, FLUX Video, …) behind one asynchronous API: ``POST /api/v1/videos`` → poll
  4. ``GET /api/v1/videos/{id}`` → download ``GET /api/v1/videos/{id}/content`` with the
  5. bearer key. The catalog and each model's limits come live from ``GET /api/v1/videos/models``
  6. (public, no key), so new releases are selectable without a patch and a retired model drops
  7. out on its own. ``capabilities()`` reflects the *selected* model so the dynamic
  8. ``video_generate`` schema advertises only what that model honours.
  9. Docs: https://openrouter.ai/docs/guides/overview/multimodal/video-generation
  10. """
  11. from __future__ import annotations
  12. import logging
  13. import time
  14. from typing import Any, Dict, List, Optional, Tuple
  15. from agent.video_gen_provider import VideoGenProvider, error_response, save_url_video, success_response
  16. logger = logging.getLogger(__name__)
  17. DEFAULT_MODEL = "minimax/hailuo-3-max"
  18. DEFAULT_BASE_URL = "https://openrouter.ai/api/v1"
  19. _CATALOG_TTL_S = 300.0
  20. _CATALOG_TIMEOUT_S = 10.0
  21. _TERMINAL_STATUSES = frozenset({"completed", "failed", "cancelled", "canceled", "expired"})
  22. # Every value the API accepts (docs "Supported Resolutions / Aspect Ratios"); the union fallback
  23. # when the live catalog is unreachable. Heights drive nearest-match clamping.
  24. _RESOLUTION_HEIGHT = {"480p": 480, "540p": 540, "720p": 720, "768p": 768, "1080p": 1080, "1K": 1024, "2K": 1440, "4K": 2160}
  25. _ALL_ASPECT_RATIOS = ("16:9", "9:16", "1:1", "4:3", "3:4", "3:2", "2:3", "21:9", "9:21")
  26. _MAX_REFERENCE_IMAGES = 3 # image references are accepted by every provider per the API schema
  27. # Offline snapshot so the picker/default work before the first successful catalog fetch.
  28. _FALLBACK_CATALOG: list[dict[str, Any]] = [
  29. {"id": DEFAULT_MODEL, "name": "MiniMax: Hailuo 3 Max", "supported_durations": list(range(5, 16)),
  30. "supported_resolutions": ["768p", "480p"], "supported_aspect_ratios": ["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"],
  31. "supported_frame_images": ["first_frame", "last_frame"], "generate_audio": False, "seed": False,
  32. "pricing_skus": {"duration_seconds_480p": "0.05", "duration_seconds_768p": "0.08"}},
  33. ]
  34. def _price_label(skus: Any) -> str:
  35. """Compact per-second price from ``pricing_skus`` (``$0.10/s`` or ``$0.05–0.28/s``); ``""`` when the
  36. SKU shape is token- or megapixel-priced (Seedance, FLUX upscale) — a wrong number is worse than none."""
  37. if not isinstance(skus, dict):
  38. return ""
  39. per_second: list[float] = []
  40. for key, value in skus.items():
  41. try:
  42. amount = float(value)
  43. except (TypeError, ValueError):
  44. continue
  45. if key.startswith("duration_seconds"):
  46. per_second.append(amount)
  47. elif key.startswith(("cents_per_second_output", "cents_per_video_output_second")):
  48. per_second.append(amount / 100)
  49. if not per_second:
  50. return ""
  51. lo, hi = min(per_second), max(per_second)
  52. return f"${lo:.2f}/s" if lo == hi else f"${lo:.2f}–{hi:.2f}/s"
  53. def _ratio(value: Any) -> Optional[float]:
  54. try:
  55. w, h = str(value).split(":")
  56. return float(w) / float(h)
  57. except (ValueError, ZeroDivisionError):
  58. return None
  59. def _nearest(value: Any, supported: list[Any], height: Optional[dict[str, int]] = None) -> Any:
  60. """Closest supported value (durations by distance, resolutions by pixel height, aspect ratios by
  61. numeric ratio); the request is otherwise a guaranteed 400 because the tool always sends a default
  62. resolution/aspect ratio."""
  63. if not supported or value in supported:
  64. return value
  65. if height is not None:
  66. target = height.get(str(value))
  67. if target is None:
  68. return supported[0]
  69. scored = [(abs(height[str(s)] - target), i, s) for i, s in enumerate(supported) if str(s) in height]
  70. return min(scored)[2] if scored else supported[0]
  71. if ":" in str(value):
  72. target = _ratio(value)
  73. if target is None:
  74. return supported[0]
  75. scored = [(abs(r - target), i, s) for i, s in enumerate(supported) if (r := _ratio(s)) is not None]
  76. return min(scored)[2] if scored else supported[0]
  77. try:
  78. return min(supported, key=lambda s: (abs(int(s) - int(value)), s))
  79. except (TypeError, ValueError):
  80. return supported[0]
  81. def _is_generative(entry: dict[str, Any]) -> bool:
  82. """Text/image-to-video models declare durations; edit, upscale and avatar models (video/audio input)
  83. do not and are outside the unified ``video_generate`` surface."""
  84. return bool(entry.get("supported_durations"))
  85. def _entry_capabilities(entry: dict[str, Any]) -> dict[str, Any]:
  86. durations = [int(d) for d in entry.get("supported_durations") or [] if isinstance(d, (int, float))]
  87. return {
  88. "modalities": ["text", "image"] if entry.get("supported_frame_images") else ["text"],
  89. "aspect_ratios": list(entry.get("supported_aspect_ratios") or _ALL_ASPECT_RATIOS),
  90. "resolutions": list(entry.get("supported_resolutions") or _RESOLUTION_HEIGHT),
  91. "max_duration": max(durations) if durations else 30, "min_duration": min(durations) if durations else 1,
  92. "supports_audio": bool(entry.get("generate_audio")), "supports_negative_prompt": False,
  93. "supports_seed": bool(entry.get("seed")), "supports_upscale": False,
  94. "max_reference_images": _MAX_REFERENCE_IMAGES,
  95. }
  96. def _image_part(url: str) -> dict[str, Any]:
  97. return {"type": "image_url", "image_url": {"url": url}}
  98. def _acceptable_image_ref(value: str) -> bool:
  99. """OpenRouter fetches the frame itself, so it needs a public HTTPS URL or an inline ``data:image/`` URL
  100. (what the sandbox confinement chokepoint hands us for local files)."""
  101. lowered = value.lower()
  102. return lowered.startswith("https://") or lowered.startswith("data:image/")
  103. def _build_payload(
  104. entry: dict[str, Any], *, model: str, prompt: str, image_url: Optional[str], reference_image_urls: Optional[list[str]],
  105. duration: Optional[int], aspect_ratio: str, resolution: str, audio: Optional[bool], seed: Optional[int],
  106. ) -> dict[str, Any]:
  107. """Unified inputs → OpenRouter body, clamped to the model's live limits; unsupported toggles are dropped
  108. rather than sent (the API 400s on ``seed``/``generate_audio`` for models that lack them)."""
  109. payload: dict[str, Any] = {"model": model, "prompt": prompt}
  110. if aspect_ratio:
  111. payload["aspect_ratio"] = _nearest(aspect_ratio, list(entry.get("supported_aspect_ratios") or []))
  112. if resolution:
  113. payload["resolution"] = _nearest(resolution, list(entry.get("supported_resolutions") or []), _RESOLUTION_HEIGHT)
  114. if duration:
  115. payload["duration"] = int(_nearest(int(duration), list(entry.get("supported_durations") or [])))
  116. if image_url:
  117. payload["frame_images"] = [{**_image_part(image_url), "frame_type": "first_frame"}]
  118. if reference_image_urls:
  119. payload["input_references"] = [_image_part(u) for u in reference_image_urls[:_MAX_REFERENCE_IMAGES]]
  120. if audio is not None and entry.get("generate_audio"):
  121. payload["generate_audio"] = bool(audio)
  122. if seed is not None and entry.get("seed"):
  123. payload["seed"] = int(seed)
  124. return payload
  125. class OpenRouterVideoGenProvider(VideoGenProvider):
  126. """Every generative model on OpenRouter's video API, catalog and limits discovered live."""
  127. name = "openrouter"
  128. display_name = "OpenRouter"
  129. _poll_interval_s = 10.0
  130. _poll_deadline_s = 900.0
  131. _request_timeout_s = 60.0
  132. def __init__(self) -> None:
  133. self._catalog_cache: Optional[tuple[list[dict[str, Any]], float]] = None
  134. # ---- credentials / transport -------------------------------------------------------------
  135. def _credentials(self) -> tuple[str, str]:
  136. """``(api_key, base_url)`` from the runtime resolver chat uses, so a pooled or OAuth credential counts
  137. and a multiplexed profile never spends the launch profile's ``os.environ`` key; raises on failure."""
  138. from hermes_cli.runtime_provider import resolve_runtime_provider
  139. runtime = resolve_runtime_provider(requested="openrouter")
  140. return (str(runtime.get("api_key") or "").strip(),
  141. (str(runtime.get("base_url") or "").strip() or DEFAULT_BASE_URL).rstrip("/"))
  142. def _headers(self, api_key: str) -> dict[str, str]:
  143. return {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json",
  144. "HTTP-Referer": "https://github.com/NousResearch/hermes-agent", "X-Title": "Hermes Agent"}
  145. def _session(self) -> Any:
  146. import requests
  147. return requests.Session()
  148. def is_available(self) -> bool:
  149. try:
  150. return bool(self._credentials()[0])
  151. except Exception as exc:
  152. logger.debug("OpenRouter video credential resolution failed: %s", exc)
  153. return False
  154. # ---- catalog -------------------------------------------------------------------------------
  155. def _catalog(self) -> list[dict[str, Any]]:
  156. """Live ``/videos/models`` entries (public endpoint), cached per TTL; the snapshot when unreachable."""
  157. if self._catalog_cache and time.monotonic() - self._catalog_cache[1] < _CATALOG_TTL_S:
  158. return self._catalog_cache[0]
  159. entries: list[dict[str, Any]] = []
  160. try:
  161. import requests
  162. response = requests.get(f"{self._credentials()[1]}/videos/models", timeout=_CATALOG_TIMEOUT_S)
  163. response.raise_for_status()
  164. data = response.json().get("data")
  165. entries = [e for e in (data if isinstance(data, list) else []) if isinstance(e, dict) and e.get("id")]
  166. except Exception as exc:
  167. logger.debug("OpenRouter video catalog unavailable: %s", exc)
  168. if not entries:
  169. return _FALLBACK_CATALOG
  170. self._catalog_cache = (entries, time.monotonic())
  171. return entries
  172. def _entry(self, model_id: str) -> dict[str, Any]:
  173. """Catalog row for *model_id*; ``{}`` for an unknown id (request passes through unclamped so a
  174. brand-new model works before our cache refreshes — the API validates)."""
  175. return next((e for e in self._catalog() if e.get("id") == model_id), {})
  176. def _configured_model(self) -> str:
  177. try:
  178. from hermes_cli.config import cfg_get, load_config
  179. value = cfg_get(load_config(), "video_gen", "model")
  180. except Exception as exc:
  181. logger.debug("Could not read video_gen.model: %s", exc)
  182. value = None
  183. return value.strip() if isinstance(value, str) and value.strip() else DEFAULT_MODEL
  184. def list_models(self) -> list[dict[str, Any]]:
  185. rows = []
  186. for entry in self._catalog():
  187. if not _is_generative(entry):
  188. continue
  189. caps = _entry_capabilities(entry)
  190. extras = [f"{caps['min_duration']}-{caps['max_duration']}s", "/".join(caps["resolutions"])]
  191. extras += ["audio"] if caps["supports_audio"] else []
  192. extras += ["i2v"] if "image" in caps["modalities"] else []
  193. rows.append({"id": entry["id"], "display": entry.get("name") or entry["id"], "strengths": ", ".join(extras),
  194. "price": _price_label(entry.get("pricing_skus")), "modalities": caps["modalities"],
  195. "min_duration": caps["min_duration"], "max_duration": caps["max_duration"]})
  196. return rows
  197. def default_model(self) -> Optional[str]:
  198. return DEFAULT_MODEL
  199. def capabilities(self) -> dict[str, Any]:
  200. """The selected model's live surface; the API-wide union when the id is unknown or the catalog is down.
  201. ``supports_seed``/``supports_audio`` are per-model (Veo/Wan/Seedance: yes; Hailuo/Grok: no)."""
  202. entry = self._entry(self._configured_model())
  203. if entry:
  204. return _entry_capabilities(entry)
  205. return {"modalities": ["text", "image"], "aspect_ratios": list(_ALL_ASPECT_RATIOS),
  206. "resolutions": list(_RESOLUTION_HEIGHT), "max_duration": 30, "min_duration": 1,
  207. "supports_audio": True, "supports_negative_prompt": False, "supports_seed": True,
  208. "supports_upscale": False, "max_reference_images": _MAX_REFERENCE_IMAGES}
  209. def get_setup_schema(self) -> dict[str, Any]:
  210. return {"name": "OpenRouter", "badge": "paid",
  211. "tag": "Veo 3.1, Sora 2 Pro, Kling 3, Seedance 2, Wan 3, Hailuo 3, Grok Imagine & more — live catalog; "
  212. "text-to-video, image-to-video & reference-to-video; uses OPENROUTER_API_KEY",
  213. "env_vars": [{"key": "OPENROUTER_API_KEY", "prompt": "OpenRouter API key", "url": "https://openrouter.ai/settings/keys"}]}
  214. # ---- generation ----------------------------------------------------------------------------
  215. def _poll(self, session: Any, job_id: str, base_url: str, headers: dict[str, str]) -> dict[str, Any]:
  216. deadline = time.monotonic() + self._poll_deadline_s
  217. url = f"{base_url}/videos/{job_id}"
  218. last_status = "unknown"
  219. while True:
  220. remaining = deadline - time.monotonic()
  221. if remaining <= 0:
  222. raise TimeoutError(f"video job {job_id} did not finish within {int(self._poll_deadline_s)}s (last status={last_status})")
  223. response = session.get(url, headers=headers, timeout=max(0.001, min(self._request_timeout_s, remaining)))
  224. response.raise_for_status()
  225. payload = response.json()
  226. last_status = str(payload.get("status") or "").lower() or "unknown"
  227. if last_status in _TERMINAL_STATUSES:
  228. return payload
  229. time.sleep(min(self._poll_interval_s, max(0.0, deadline - time.monotonic())))
  230. def _save_completed_video(self, job_id: str, base_url: str, headers: dict[str, str]) -> str:
  231. # The content endpoint is derived from our configured origin, never from ``unsigned_urls``: the
  232. # bearer key must only ever be sent to the host the operator selected. That origin is operator
  233. # chosen (a LAN relay is legitimate), so the first hop is trusted for the private-address check.
  234. return str(save_url_video(f"{base_url}/videos/{job_id}/content", prefix="openrouter",
  235. headers=headers, require_video_content_type=True,
  236. trusted_origin=True))
  237. def generate(
  238. self, prompt: str, *, model: Optional[str] = None, image_url: Optional[str] = None,
  239. reference_image_urls: Optional[list[str]] = None, duration: Optional[int] = None,
  240. aspect_ratio: str = "16:9", resolution: str = "720p", negative_prompt: Optional[str] = None,
  241. audio: Optional[bool] = None, seed: Optional[int] = None, **kwargs: Any,
  242. ) -> dict[str, Any]:
  243. del negative_prompt, kwargs # no top-level negative_prompt on this API; unknown kwargs are ignored per the ABC
  244. prompt = (prompt or "").strip()
  245. model_id = (model or "").strip() or self._configured_model()
  246. def fail(error: str, error_type: str) -> dict[str, Any]:
  247. return error_response(error=error, error_type=error_type, provider=self.name, model=model_id, prompt=prompt,
  248. aspect_ratio=aspect_ratio)
  249. if not prompt:
  250. return fail("prompt is required", "invalid_request")
  251. try:
  252. api_key, base_url = self._credentials()
  253. except Exception as exc:
  254. return fail(f"Could not resolve OpenRouter credentials: {exc}", "missing_credentials")
  255. if not api_key:
  256. return fail("No OpenRouter credential: set OPENROUTER_API_KEY or run `hermes auth add openrouter`",
  257. "missing_credentials")
  258. # Resolved once: a rotating pool must not submit under one key and poll or download under another.
  259. headers = self._headers(api_key)
  260. image_url = (image_url or "").strip() or None
  261. refs = [r.strip() for r in (reference_image_urls or []) if isinstance(r, str) and r.strip()]
  262. for ref in ([image_url] if image_url else []) + refs:
  263. if not _acceptable_image_ref(ref):
  264. return fail("image inputs must be public HTTPS URLs or data:image/ URLs (OpenRouter fetches them itself)",
  265. "invalid_request")
  266. entry = self._entry(model_id)
  267. payload = _build_payload(entry, model=model_id, prompt=prompt, image_url=image_url, reference_image_urls=refs,
  268. duration=duration, aspect_ratio=aspect_ratio, resolution=resolution, audio=audio, seed=seed)
  269. session = self._session()
  270. try:
  271. submitted = session.post(f"{base_url}/videos", headers=headers, json=payload,
  272. timeout=self._request_timeout_s)
  273. if submitted.status_code >= 400:
  274. detail = ""
  275. try:
  276. detail = str((submitted.json().get("error") or {}).get("message") or "")
  277. except Exception:
  278. detail = ""
  279. return fail(f"OpenRouter rejected the request (HTTP {submitted.status_code}): {detail or submitted.text[:300]}",
  280. "api_error")
  281. job_id = str(submitted.json().get("id") or "").strip()
  282. if not job_id:
  283. return fail("OpenRouter submit response did not contain a job id", "api_error")
  284. job = self._poll(session, job_id, base_url, headers)
  285. status = str(job.get("status") or "").lower()
  286. if status != "completed":
  287. return fail(str(job.get("error") or f"video job ended with status={status!r}"), "job_failed")
  288. video_path = self._save_completed_video(job_id, base_url, headers)
  289. except Exception as exc:
  290. logger.debug("OpenRouter video generation failed", exc_info=True)
  291. return fail(f"OpenRouter video generation failed: {exc}", "api_error")
  292. finally:
  293. close = getattr(session, "close", None)
  294. if callable(close):
  295. close()
  296. raw_usage = job.get("usage")
  297. usage: dict[str, Any] = raw_usage if isinstance(raw_usage, dict) else {}
  298. extra: dict[str, Any] = {"job_id": job_id, **({"cost": usage["cost"]} if usage.get("cost") is not None else {})}
  299. return success_response(
  300. video=video_path, model=model_id, prompt=prompt, modality="image" if image_url else "text",
  301. aspect_ratio=str(payload.get("aspect_ratio") or ""), duration=int(payload.get("duration") or 0),
  302. provider=self.name, extra=extra)
  303. def register(ctx) -> None:
  304. """Plugin entry point — wire ``OpenRouterVideoGenProvider`` into the registry."""
  305. ctx.register_video_gen_provider(OpenRouterVideoGenProvider())