video_gen_provider.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289
  1. """Video Generation Provider ABC.
  2. Providers register via ``PluginContext.register_video_gen_provider()`` and live
  3. in ``<repo>/plugins/video_gen/<name>/`` (built-in) or
  4. ``~/.hermes/plugins/video_gen/<name>/``; mirrors ``agent/image_gen_provider.py``.
  5. One tool covers text-to-video and image-to-video: ``image_url`` present routes to
  6. the provider's image-to-video endpoint. Video edit/extend are deliberately NOT
  7. exposed — backends are too inconsistent for one unified tool.
  8. Response shape (:func:`success_response` / :func:`error_response`): ``success``,
  9. ``video`` (URL or absolute path), ``model``, ``prompt``, ``modality``
  10. ("text" | "image"), ``aspect_ratio``, ``duration`` (seconds, 0 if n/a),
  11. ``provider``; plus ``error`` / ``error_type`` only when ``success`` is False.
  12. """
  13. from __future__ import annotations
  14. import abc
  15. import logging
  16. import time
  17. from pathlib import Path
  18. from typing import Any, Dict, List, Optional, Tuple
  19. from agent import provider_media
  20. from agent.provider_base import CatalogProviderBase
  21. from agent.secret_scope import get_secret_str
  22. logger = logging.getLogger(__name__)
  23. # Advertised as an enum hint in the tool schema; providers may accept a narrower
  24. # or wider set and are responsible for clamping.
  25. COMMON_ASPECT_RATIOS: tuple[str, ...] = (
  26. "16:9", "9:16", "1:1", "4:3", "3:4", "3:2", "2:3", "21:9"
  27. )
  28. DEFAULT_ASPECT_RATIO = "16:9"
  29. COMMON_RESOLUTIONS: tuple[str, ...] = ("480p", "540p", "720p", "768p", "1080p")
  30. DEFAULT_RESOLUTION = "720p"
  31. class VideoGenProvider(CatalogProviderBase):
  32. """Abstract base class for a video generation backend: implement :attr:`name`
  33. and :meth:`generate`. ``list_models`` entries are **model families** and may
  34. add ``speed`` / ``strengths`` / ``price`` / advisory ``modalities``."""
  35. def capabilities(self) -> dict[str, Any]:
  36. """Supported features (keys below, all optional) used for soft validation,
  37. capability-gated params in the dynamic ``video_generate`` schema, and the
  38. picker. Default fails closed: text-only, no optional features."""
  39. return {
  40. "modalities": ["text"], "aspect_ratios": list(COMMON_ASPECT_RATIOS),
  41. "resolutions": list(COMMON_RESOLUTIONS), "max_duration": 10, "min_duration": 1,
  42. "supports_audio": False, "supports_negative_prompt": False, "supports_seed": False,
  43. "supports_upscale": False, "max_reference_images": 0,
  44. }
  45. @abc.abstractmethod
  46. def generate(
  47. self, prompt: str, *, model: Optional[str] = None, image_url: Optional[str] = None,
  48. reference_image_urls: Optional[list[str]] = None, duration: Optional[int] = None,
  49. aspect_ratio: str = DEFAULT_ASPECT_RATIO, resolution: str = DEFAULT_RESOLUTION,
  50. negative_prompt: Optional[str] = None, audio: Optional[bool] = None,
  51. seed: Optional[int] = None, **kwargs: Any,
  52. ) -> dict[str, Any]:
  53. """Generate a video from a prompt, or animate ``image_url`` when given; return
  54. :func:`success_response` / :func:`error_response`. Unknown ``kwargs`` MUST be
  55. ignored. Known optional kwarg ``upscale`` (bool): a post-generation high-res
  56. pass; providers that honor it report ``upscaled: True`` in ``extra``."""
  57. _GENERATED_VIDEO_KIND = f"{provider_media.GENERATED_SUBDIR}/videos"
  58. def save_b64_video(b64_data: str,*, prefix: str="video", extension: str="mp4") -> Path:
  59. """Decode base64 video data into ``$HERMES_HOME/cache/generated/videos/``; return the path."""
  60. return provider_media.save_b64(_GENERATED_VIDEO_KIND, b64_data, prefix=prefix, extension=extension)
  61. def save_bytes_video(raw: bytes,*, prefix: str="video", extension: str="mp4") -> Path:
  62. """Write raw video bytes (e.g. an HTTP download body) to the cache."""
  63. return provider_media.save_bytes(_GENERATED_VIDEO_KIND, raw, prefix=prefix, extension=extension)
  64. _URL_VIDEO_CONTENT_TYPES = {
  65. "video/mp4": "mp4", "video/webm": "webm", "video/quicktime": "mov", "video/x-matroska": "mkv"
  66. }
  67. def save_url_video(
  68. url: str,
  69. *,
  70. prefix: str = "video",
  71. timeout: float = 180.0,
  72. max_bytes: int = 200 * 1024 * 1024,
  73. headers: Optional[dict[str, str]] = None,
  74. require_video_content_type: bool = False,
  75. trusted_origin: bool = False,
  76. ) -> Path:
  77. """Download an (often ephemeral) video URL into ``$HERMES_HOME/cache/generated/videos/``;
  78. raises on network / HTTP / oversize / empty errors so callers can fall back to the URL.
  79. ``trusted_origin`` is only for URLs built from the operator's configured provider
  80. ``base_url`` (see ``provider_media.save_url``)."""
  81. return provider_media.save_url(
  82. _GENERATED_VIDEO_KIND, url, prefix=prefix, timeout=timeout, max_bytes=max_bytes,
  83. chunk_size=256 * 1024, content_types=_URL_VIDEO_CONTENT_TYPES,
  84. url_extensions=("mp4", "webm", "mov", "mkv"), default_extension="mp4",
  85. label="Video", empty_error="Video at {url} was empty (0 bytes).",
  86. headers=headers, require_known_content_type=require_video_content_type,
  87. trusted_origin=trusted_origin,
  88. )
  89. def success_response(
  90. *, video: str, model: str, prompt: str, modality: str = "text", aspect_ratio: str = "",
  91. duration: int = 0, provider: str, extra: Optional[dict[str, Any]] = None,
  92. ) -> dict[str, Any]:
  93. """Uniform success dict; ``extra`` keys are added without overriding standard ones."""
  94. payload: dict[str, Any] = {
  95. "success": True, "video": video, "model": model, "prompt": prompt, "modality": modality,
  96. "aspect_ratio": aspect_ratio, "duration": int(duration) if duration else 0, "provider": provider,
  97. }
  98. for k, v in (extra or {}).items():
  99. payload.setdefault(k, v)
  100. return payload
  101. def error_response(
  102. *, error: str, error_type: str = "provider_error", provider: str = "", model: str = "",
  103. prompt: str = "", aspect_ratio: str = "",
  104. ) -> dict[str, Any]:
  105. """Build a uniform error response dict."""
  106. return {
  107. "success": False, "video": None, "error": error, "error_type": error_type, "model": model,
  108. "prompt": prompt, "aspect_ratio": aspect_ratio, "provider": provider,
  109. }
  110. class OpenAICompatibleVideoGenProvider(VideoGenProvider):
  111. """Generic text/image-to-video over the OpenAI ``client.videos`` API.
  112. DeepInfra, OpenAI/Sora, and OpenRouter share the ``POST /videos`` async-job
  113. shape (``create`` → poll → ``download_content``); a concrete backend sets
  114. ``name``, ``_env_key``, ``_default_base_url`` and ``list_models()`` (entries
  115. with an ``id`` key; ``default_model()`` uses ``[0]``). Provider-specific
  116. fields (``image_url``/``negative_prompt``/``seed``) ride in ``extra_body``.
  117. """
  118. _env_key: str = "OPENAI_API_KEY"
  119. _default_base_url: str = "https://api.openai.com/v1"
  120. # The SDK's ``create_and_poll`` polls ~1/s forever on a non-terminal status,
  121. # pinning the tool-executor thread on a stuck job; we poll coarsely with a
  122. # hard wall-clock deadline instead.
  123. _poll_interval_s: float = 5.0
  124. _poll_deadline_s: float = 900.0
  125. def _api_key(self) -> str:
  126. # Through the profile secret scope: under multiplexing os.environ holds another profile's key.
  127. return get_secret_str(self._env_key).strip()
  128. def is_available(self) -> bool:
  129. return bool(self._api_key())
  130. def _create_and_poll(self, client: Any, call_kwargs: dict[str, Any]) -> Any:
  131. """Create the job and poll to a terminal status (any); raise
  132. :class:`TimeoutError` when ``_poll_deadline_s`` passes first."""
  133. video = client.videos.create(**call_kwargs)
  134. terminal = {"completed", "succeeded", "failed", "error", "cancelled", "canceled"}
  135. deadline = time.monotonic() + self._poll_deadline_s
  136. while getattr(video, "status", None) not in terminal:
  137. if time.monotonic() >= deadline:
  138. raise TimeoutError(
  139. f"video job {getattr(video, 'id', '?')} did not reach a terminal "
  140. f"status within {int(self._poll_deadline_s)}s "
  141. f"(last status={getattr(video, 'status', None)!r})"
  142. )
  143. time.sleep(self._poll_interval_s)
  144. video = client.videos.retrieve(video.id)
  145. return video
  146. def _base_url(self) -> str:
  147. return get_secret_str(f"{self.name.upper()}_BASE_URL").strip() or self._default_base_url
  148. def generate(
  149. self, prompt: str, *, model: Optional[str] = None, image_url: Optional[str] = None,
  150. reference_image_urls: Optional[list[str]] = None, duration: Optional[int] = None,
  151. aspect_ratio: str = DEFAULT_ASPECT_RATIO, resolution: str = DEFAULT_RESOLUTION,
  152. negative_prompt: Optional[str] = None, audio: Optional[bool] = None,
  153. seed: Optional[int] = None, **kwargs: Any,
  154. ) -> dict[str, Any]:
  155. if not prompt or not prompt.strip():
  156. return error_response(error="prompt is required", error_type="invalid_request", provider=self.name)
  157. if not self._api_key():
  158. return error_response(
  159. error=f"{self._env_key} is not set", error_type="missing_credentials", provider=self.name
  160. )
  161. try:
  162. import openai
  163. except ImportError:
  164. return error_response(
  165. error="openai Python package not installed. Run: hermes pm repair",
  166. error_type="missing_dependency", provider=self.name,
  167. )
  168. model_id = model or self.default_model()
  169. if not model_id:
  170. return error_response(
  171. error=f"no {self.name} video model available (live catalog empty?)",
  172. error_type="no_model", provider=self.name,
  173. )
  174. def fail(error: str, error_type: str) -> dict[str, Any]:
  175. return error_response(
  176. error=error, error_type=error_type, provider=self.name, model=model_id, prompt=prompt,
  177. aspect_ratio=aspect_ratio,
  178. )
  179. # Fields ``videos.create`` doesn't name natively ride in ``extra_body``.
  180. extra_body = {
  181. k: v
  182. for k, v in {
  183. "negative_prompt": negative_prompt, "aspect_ratio": aspect_ratio,
  184. "image_url": image_url, # presence ⇒ image-to-video
  185. "seed": seed,
  186. }.items()
  187. if v is not None
  188. }
  189. call_kwargs: dict[str, Any] = {"model": model_id, "prompt": prompt}
  190. if duration:
  191. call_kwargs["seconds"] = str(duration)
  192. if resolution:
  193. call_kwargs["size"] = resolution
  194. if extra_body:
  195. call_kwargs["extra_body"] = extra_body
  196. # Env-only-proxy httpx client: a macOS system proxy (ExceptionsList invisible to httpx)
  197. # must not swallow a local/custom ``<NAME>_BASE_URL`` (#64888).
  198. from agent.process_bootstrap import build_keepalive_http_client
  199. client_kwargs: dict[str, Any] = {"api_key": self._api_key(), "base_url": self._base_url()}
  200. http_client = build_keepalive_http_client(client_kwargs["base_url"])
  201. if http_client is not None:
  202. client_kwargs["http_client"] = http_client
  203. client = openai.OpenAI(**client_kwargs)
  204. try:
  205. try:
  206. video = self._create_and_poll(client, call_kwargs)
  207. except Exception as exc:
  208. logger.debug("%s video generation failed", self.name, exc_info=True)
  209. return fail(f"{self.name} video generation failed: {exc}", "api_error")
  210. # DeepInfra reports "succeeded", OpenAI/Sora "completed" — accept both.
  211. status = getattr(video, "status", None)
  212. if status not in ("completed", "succeeded"):
  213. # ``video.error`` is a pydantic object — str() keeps the dict JSON-serializable.
  214. job_error = getattr(video, "error", None)
  215. return fail(str(job_error) if job_error else f"video job ended with status={status!r}", "job_failed")
  216. # Output is a delivery URL in ``data`` (DeepInfra/FAL) or only reachable
  217. # via the SDK download endpoint (OpenAI/Sora). Save locally either way —
  218. # DeepInfra's delivery URLs are short-lived.
  219. url = None
  220. for item in getattr(video, "data", None) or []:
  221. url = (item.get("url") if isinstance(item, dict) else getattr(item, "url", None)) or None
  222. if url:
  223. break
  224. try:
  225. if url:
  226. video_ref = str(save_url_video(url, prefix=self.name))
  227. else:
  228. raw = client.videos.download_content(video.id).read()
  229. video_ref = str(save_bytes_video(raw, prefix=self.name))
  230. except Exception as exc:
  231. if not url:
  232. return fail(f"{self.name} video job succeeded but no output could be retrieved: {exc}", "empty_response")
  233. logger.debug("%s: saving video locally failed (%s); returning URL", self.name, exc)
  234. video_ref = url
  235. return success_response(
  236. video=video_ref, model=model_id, prompt=prompt,
  237. modality="image" if image_url else "text", aspect_ratio=aspect_ratio,
  238. duration=duration or 0, provider=self.name,
  239. )
  240. finally:
  241. close = getattr(client, "close", None)
  242. if callable(close):
  243. close()