video_generation_tool.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376
  1. #!/usr/bin/env python3
  2. """``video_generate``: one tool dispatching to a plugin-registered :class:`VideoGenProvider`
  3. (``agent/video_gen_provider.py`` ABC, ``agent/video_gen_registry.py``, ``plugins/video_gen/<name>/``).
  4. Ships **no in-tree provider**: enable a plugin and select it in ``hermes tools`` → Video
  5. Generation. The tool layer only does lightweight validation; each provider clamps/ignores
  6. unsupported params inside ``generate``. Video edit/extend are deliberately not exposed here.
  7. """
  8. from __future__ import annotations
  9. import json
  10. import logging
  11. from typing import Any, Dict, List, Optional
  12. from agent.video_gen_provider import (
  13. COMMON_ASPECT_RATIOS,
  14. COMMON_RESOLUTIONS,
  15. DEFAULT_ASPECT_RATIO,
  16. DEFAULT_RESOLUTION,
  17. error_response)
  18. from tools.registry import registry, tool_error
  19. logger = logging.getLogger(__name__)
  20. VIDEO_GENERATE_SCHEMA: dict[str, Any] = {
  21. "name": "video_generate",
  22. # Placeholder: description AND params are rebuilt at get_tool_definitions() time by
  23. # _build_dynamic_video_schema() from capabilities() + the model's catalog entry. Optional
  24. # args are advertised ONLY when honored; the handler accepts them regardless (replay compat).
  25. "description": "(rebuilt at get_definitions() time — see _build_dynamic_video_schema)",
  26. "parameters": {
  27. "type": "object",
  28. "properties": {
  29. "prompt": {
  30. "type": "string",
  31. "description": (
  32. "Text instruction describing the desired video, motion, "
  33. "subject, style, camera movement, etc."
  34. ),
  35. },
  36. "duration": {
  37. "type": "integer",
  38. "description": (
  39. "Desired video duration in seconds. Providers clamp to "
  40. "their supported range. Omit for the provider default."
  41. ),
  42. },
  43. "aspect_ratio": {
  44. "type": "string",
  45. "enum": list(COMMON_ASPECT_RATIOS),
  46. "description": "Output aspect ratio.",
  47. "default": DEFAULT_ASPECT_RATIO,
  48. },
  49. "resolution": {
  50. "type": "string",
  51. "enum": list(COMMON_RESOLUTIONS),
  52. "description": "Output resolution.",
  53. "default": DEFAULT_RESOLUTION,
  54. },
  55. # No ``model`` here: the backend/model is user configuration (``video_gen.model``), never an
  56. # agent choice (#83080 ruling). Capability-gated args are added by _build_dynamic_video_schema; never statically.
  57. },
  58. # NOTE (schema diet, #95681): image_url / reference_image_urls / negative_prompt / audio / seed /
  59. # upscale are added per-capability by _build_dynamic_video_schema.
  60. "required": ["prompt"],
  61. },
  62. }
  63. def _read_video_gen_key(key: str) -> Optional[str]:
  64. """Return the stripped ``video_gen.<key>`` string from config.yaml, or None."""
  65. try:
  66. from hermes_cli.config import cfg_get, load_config
  67. value = cfg_get(load_config(), "video_gen", key)
  68. except Exception as exc:
  69. logger.debug("Could not read video_gen config: %s", exc)
  70. return None
  71. return value.strip() if isinstance(value, str) and value.strip() else None
  72. def _read_configured_video_provider() -> Optional[str]:
  73. return _read_video_gen_key("provider")
  74. def _read_configured_video_model() -> Optional[str]:
  75. return _read_video_gen_key("model")
  76. def _discovered_registry():
  77. """Import the provider registry after (idempotent) plugin discovery so user-installed plugins are visible."""
  78. from agent import video_gen_registry
  79. from hermes_cli.plugins import _ensure_plugins_discovered
  80. _ensure_plugins_discovered()
  81. return video_gen_registry, _ensure_plugins_discovered
  82. def check_video_generation_requirements() -> bool:
  83. """True when at least one registered provider reports available."""
  84. registry_mod, _ = _discovered_registry()
  85. return any(_provider_call(p, "is_available", False) for p in registry_mod.list_providers())
  86. def _resolve_active_provider():
  87. """Active provider or None; a miss forces a discovery refresh (sessions older than the plugin install)."""
  88. try:
  89. registry_mod, ensure_discovered = _discovered_registry()
  90. provider = registry_mod.get_active_provider()
  91. if provider is None:
  92. ensure_discovered(force=True)
  93. provider = registry_mod.get_active_provider()
  94. return provider
  95. except Exception as exc:
  96. logger.debug("video_gen provider resolution failed: %s", exc)
  97. return None
  98. def _missing_provider_error(configured: Optional[str]) -> str:
  99. if configured:
  100. return json.dumps(error_response(
  101. error=(f"video_gen.provider='{configured}' is set but no plugin registered that name. "
  102. f"Run `hermes plugins list` to see installed video gen backends, or "
  103. f"`hermes tools` → Video Generation to pick one."),
  104. error_type="provider_not_registered", provider=configured))
  105. return json.dumps(error_response(
  106. error=("No video generation backend is configured. Run `hermes tools` → "
  107. "Video Generation to enable one (xAI, FAL, OpenRouter, or DeepInfra)."),
  108. error_type="no_provider_configured"))
  109. _BOOL_WORDS = {"true": True, "1": True, "yes": True, "on": True,
  110. "false": False, "0": False, "no": False, "off": False}
  111. def _coerce_int(value: Any) -> Optional[int]:
  112. try:
  113. return None if value is None or value == "" else int(value)
  114. except (TypeError, ValueError):
  115. return None
  116. def _coerce_bool(value: Any) -> Optional[bool]:
  117. if isinstance(value, bool):
  118. return value
  119. return _BOOL_WORDS.get(value.strip().lower()) if isinstance(value, str) else None
  120. def _normalize_reference_images(value: Any) -> Optional[list[str]]:
  121. if isinstance(value, str):
  122. value = [value]
  123. if not isinstance(value, (list, tuple)):
  124. return None
  125. return [item.strip() for item in value if isinstance(item, str) and item.strip()] or None
  126. def _handle_video_generate(args: dict[str, Any], **_kw: Any) -> str:
  127. prompt = (args.get("prompt") or "").strip()
  128. image_url = (args.get("image_url") or "").strip() or None
  129. reference_image_urls = _normalize_reference_images(args.get("reference_image_urls"))
  130. task_id = _kw.get("task_id")
  131. # Confinement chokepoint (mirrors image_generate): non-local backends hand providers data: URLs.
  132. from tools.image_generation_tool import _confine_source_images
  133. image_url, reference_image_urls, confine_error = _confine_source_images(
  134. image_url, reference_image_urls, task_id)
  135. if confine_error is not None:
  136. return confine_error
  137. # Coerced BEFORE validation (ordering parity: a bad value raises before a missing prompt).
  138. optional = {
  139. "duration": _coerce_int(args.get("duration")),
  140. "aspect_ratio": (args.get("aspect_ratio") or DEFAULT_ASPECT_RATIO).strip() or DEFAULT_ASPECT_RATIO,
  141. "resolution": (args.get("resolution") or DEFAULT_RESOLUTION).strip() or DEFAULT_RESOLUTION,
  142. "negative_prompt": (args.get("negative_prompt") or "").strip() or None,
  143. "audio": _coerce_bool(args.get("audio")),
  144. "seed": _coerce_int(args.get("seed")),
  145. "upscale": _coerce_bool(args.get("upscale"))}
  146. # Soft validation — providers do their own; our surface never accepts image-only.
  147. if not prompt:
  148. return tool_error("prompt is required for video generation")
  149. if "operation" in args or "video_url" in args:
  150. return tool_error(
  151. "video_generate only supports text-to-video, image-to-video, and "
  152. "reference-to-video; use a provider-specific tool for video edit/extend")
  153. configured = _read_configured_video_provider()
  154. provider = _resolve_active_provider()
  155. if provider is None:
  156. return _missing_provider_error(configured)
  157. # Config, then provider default; a ``model`` in args is ignored (models do not choose models).
  158. model = _read_configured_video_model() or provider.default_model()
  159. kwargs: dict[str, Any] = {
  160. "model": model, "image_url": image_url, "reference_image_urls": reference_image_urls, **optional}
  161. # Drop None entries so providers see clean defaults.
  162. kwargs = {k: v for k, v in kwargs.items() if v is not None}
  163. pname = getattr(provider, "name", "?")
  164. def _err(error: str, error_type: str) -> str:
  165. return json.dumps(error_response(
  166. error=error, error_type=error_type,
  167. provider=getattr(provider, "name", ""), model=model or "", prompt=prompt))
  168. try:
  169. result = provider.generate(prompt=prompt, **kwargs)
  170. except TypeError as exc:
  171. # An un-widened provider signature is a plugin bug, not a caller error.
  172. logger.warning("video_gen provider '%s' rejected kwargs (signature too narrow): %s", pname, exc)
  173. return _err(
  174. f"Provider '{pname}' signature is out of date with the video_generate schema. "
  175. f"Report this to the plugin author.",
  176. "provider_contract")
  177. except Exception as exc:
  178. logger.warning("video_gen provider '%s' raised: %s", pname, exc)
  179. return _err(f"Provider '{pname}' error: {exc}", "provider_exception")
  180. if not isinstance(result, dict):
  181. return _err("Provider returned a non-dict result", "provider_contract")
  182. return json.dumps(result)
  183. # Dynamic schema — reflects the active backend's actual capabilities so the model usually gets
  184. # the call right first try. model_tools.get_tool_definitions() keys its cache on config.yaml
  185. # mtime, so the schema rebuilds on provider/model change. Optional params below are advertised
  186. # only when the provider's capabilities() sets the flag (order = schema property order).
  187. _CAPABILITY_PARAMS = (
  188. ("supports_negative_prompt", "negative_prompt", {
  189. "type": "string",
  190. "description": "Content to avoid in the output.",
  191. }),
  192. ("supports_audio", "audio", {
  193. "type": "boolean",
  194. "description": "Enable native audio generation (affects pricing tier).",
  195. }),
  196. ("supports_seed", "seed", {
  197. "type": "integer",
  198. "description": "Seed for reproducible outputs.",
  199. }),
  200. ("supports_upscale", "upscale", {
  201. "type": "boolean",
  202. "description": (
  203. "High-resolution pass via the backend's video upscaler "
  204. "(~2x, extra cost/latency). Omit for native resolution."
  205. ),
  206. }),
  207. )
  208. _GENERIC_DESCRIPTION = (
  209. "Generate video from text, a still image or reference images (configured model). "
  210. "Pass `image_url` to animate an image or `reference_image_urls` for "
  211. "reference-to-video. Video edit/extend workflows are not part of this "
  212. "unified surface; use a dedicated provider-specific tool when one is "
  213. "available. The backend and model family are user-configured via "
  214. "`hermes tools` → Video Generation; the agent does not pick them. "
  215. "Long-running generations may take 30 seconds to several minutes — "
  216. "the call blocks until the video is ready. Returns the result in the "
  217. "`video` field — either an HTTP URL or an absolute file path; reference "
  218. "it in your response using the current platform's file-delivery convention."
  219. )
  220. def _schema(description: str, properties: dict[str, Any]) -> dict[str, Any]:
  221. return {
  222. "description": description,
  223. "parameters": {"type": "object", "properties": properties, "required": ["prompt"]}}
  224. def _provider_call(provider: Any, method: str, default: Any) -> Any:
  225. """``provider.<method>()`` or ``default`` when it raises or returns a falsy value."""
  226. try:
  227. return getattr(provider, method)() or default
  228. except Exception:
  229. return default
  230. def _build_dynamic_video_schema() -> dict[str, Any]:
  231. """Description AND params from capabilities() + the model's catalog entry; enums and duration
  232. bounds tighten to the active model. Unadvertised args are still accepted (replay compat)."""
  233. static_props = VIDEO_GENERATE_SCHEMA["parameters"]["properties"]
  234. parts: list[str] = [_GENERIC_DESCRIPTION]
  235. configured_model = _read_configured_video_model()
  236. provider = _resolve_active_provider()
  237. if provider is None:
  238. parts.append(
  239. "\nNo video backend is available. Calls will return an error "
  240. "until the user picks one via `hermes tools` → Video Generation.")
  241. return _schema("\n".join(parts), {"prompt": static_props["prompt"]})
  242. caps = _provider_call(provider, "capabilities", {})
  243. models = _provider_call(provider, "list_models", [])
  244. active_model = configured_model or provider.default_model()
  245. model_meta = next((m for m in models if isinstance(m, dict) and m.get("id") == active_model), {})
  246. # Model caveats surface only what differs from the backend's overall capabilities.
  247. # FAL's plugin uses the singular ``modality`` key for single-modality entries.
  248. model_modalities = set(model_meta.get("modalities") or [])
  249. modality = model_meta.get("modality")
  250. if modality:
  251. model_modalities.add(modality)
  252. if "image" in model_modalities and "text" not in model_modalities:
  253. parts.append(
  254. "- this model is image-to-video only — image_url is REQUIRED; "
  255. "text-only calls will be rejected")
  256. elif "text" in model_modalities and "image" not in model_modalities:
  257. parts.append("- this model is text-to-video only — image_url is not supported")
  258. effective_modalities = model_modalities or set(caps.get("modalities") or [])
  259. can_i2v = "image" in effective_modalities
  260. t2v = "text" in effective_modalities
  261. if can_i2v and not t2v:
  262. parts.append("- image-to-video only: image_url is REQUIRED")
  263. elif not can_i2v:
  264. parts.append("- text-to-video only (no image input)")
  265. if provider.name == "xai":
  266. parts.append(
  267. "- chaining: for edit/extend pass the public HTTPS MP4 in `video` "
  268. "or `public_url` from the prior Imagine result (files-cdn). For "
  269. "image-to-video / reference-to-video pass public image URLs the "
  270. "same way")
  271. try:
  272. from tools.xai_http import xai_storage_notice_text
  273. notice = xai_storage_notice_text("video_gen")
  274. except Exception:
  275. notice = ""
  276. if notice:
  277. parts.append(f"- storage: {notice}")
  278. properties: dict[str, Any] = {"prompt": static_props["prompt"]}
  279. if can_i2v:
  280. properties["image_url"] = {
  281. "type": "string",
  282. "description": (
  283. "Public HTTPS URL of a still image to animate "
  284. "(image-to-video). Omit for text-to-video.")}
  285. max_refs = int(caps.get("max_reference_images") or 0)
  286. if max_refs > 0:
  287. properties["reference_image_urls"] = {
  288. "type": "array",
  289. "items": {"type": "string"},
  290. "maxItems": max_refs,
  291. "description": (
  292. f"Up to {max_refs} public HTTPS reference image URLs "
  293. "(style or character refs).")}
  294. min_duration = model_meta.get("min_duration", caps.get("min_duration"))
  295. max_duration = model_meta.get("max_duration", caps.get("max_duration"))
  296. duration_param = dict(static_props["duration"])
  297. if min_duration and max_duration:
  298. duration_param["minimum"] = int(min_duration)
  299. duration_param["maximum"] = int(max_duration)
  300. duration_param["description"] = (
  301. f"Video duration in seconds ({min_duration}-{max_duration}). "
  302. "Omit for the provider default.")
  303. properties["duration"] = duration_param
  304. # Tighten enums to the active backend's actual sets when declared.
  305. for key, caps_key in (("aspect_ratio", "aspect_ratios"), ("resolution", "resolutions")):
  306. param = dict(static_props[key])
  307. if caps.get(caps_key):
  308. param["enum"] = list(caps[caps_key])
  309. properties[key] = param
  310. for flag, key, param in _CAPABILITY_PARAMS:
  311. if caps.get(flag):
  312. properties[key] = param
  313. if caps.get("audio_always_on") and not caps.get("supports_audio"):
  314. parts.append(
  315. "- audio: native stereo audio is generated with every video "
  316. "(always on; no toggle) — describe the desired sound in the "
  317. "prompt")
  318. return _schema("\n".join(parts), properties)
  319. registry.register(
  320. name="video_generate",
  321. toolset="video_gen",
  322. schema=VIDEO_GENERATE_SCHEMA,
  323. handler=_handle_video_generate,
  324. check_fn=check_video_generation_requirements,
  325. requires_env=[],
  326. is_async=False,
  327. emoji="🎬",
  328. dynamic_schema_overrides=_build_dynamic_video_schema)