image_generation_tool.py 51 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062
  1. #!/usr/bin/env python3
  2. """Image generation via FAL.ai (model picked in ``hermes tools``, persisted to ``image_gen.model``).
  3. ``_build_fal_payload()`` / ``_build_fal_edit_payload()`` translate unified inputs into the
  4. ``FAL_MODELS`` payload filtered to its ``supports`` whitelist so models never receive rejected
  5. keys. Clarity upscaling is strictly per-call opt-in: default-on degraded text/CJK/faces.
  6. """
  7. import json
  8. import logging
  9. import os
  10. import datetime
  11. import threading
  12. import uuid
  13. from typing import Any, Dict, Optional
  14. # Imported lazily by _load_fal_client() (~64 ms on every CLI cold start); a test-monkeypatched
  15. # value short-circuits the loader.
  16. fal_client: Any = None
  17. def _load_fal_client() -> Any:
  18. """Lazily import fal_client into the module global (idempotent; keeps a test-installed mock)."""
  19. global fal_client
  20. if fal_client is None:
  21. from tools.fal_common import import_fal_client
  22. fal_client = import_fal_client()
  23. return fal_client
  24. from tools.debug_helpers import DebugSession
  25. from tools.fal_common import (
  26. _ManagedFalSyncClient, _extract_http_status, _managed_fal_billing_error,
  27. _normalize_fal_queue_url_format, submit_managed_fal_with_rate_limit_retry, wait_fal_queue_result,
  28. )
  29. from tools.image_generation_catalog import (
  30. DEFAULT_ASPECT_RATIO, DEFAULT_MODEL, FAL_MODELS, UPSCALER_CREATIVITY, UPSCALER_DEFAULT_PROMPT,
  31. UPSCALER_FACTOR, UPSCALER_GUIDANCE_SCALE, UPSCALER_MODEL, UPSCALER_NEGATIVE_PROMPT,
  32. UPSCALER_NUM_INFERENCE_STEPS, UPSCALER_RESEMBLANCE, UPSCALER_SAFETY_CHECKER, VALID_ASPECT_RATIOS,
  33. )
  34. from tools.managed_tool_gateway import resolve_managed_tool_gateway
  35. from tools.tool_backend_helpers import (
  36. NOUS_MANAGED_PROVIDER, fal_key_is_configured, managed_nous_tools_enabled,
  37. nous_tool_gateway_unavailable_message, read_selection, selection_error)
  38. logger = logging.getLogger(__name__)
  39. _debug = DebugSession("image_tools", env_var="IMAGE_TOOLS_DEBUG")
  40. _managed_fal_client = None
  41. _managed_fal_client_config = None
  42. _managed_fal_client_lock = threading.Lock()
  43. # --- Managed FAL gateway (Nous Subscription) ---
  44. def _resolve_managed_fal_gateway():
  45. """Managed gateway config for the stored `hermes tools` selection, or ``None`` for direct FAL.
  46. ``"nous"`` (or legacy ``use_gateway: true``) → managed ONLY (unreachable = selection-naming
  47. error, never a silent FAL_KEY fallback). Other stored provider → direct ONLY (missing FAL_KEY
  48. = error naming the selection). Never configured → autodetect: direct if FAL_KEY, else managed.
  49. """
  50. selected = read_selection("image_gen")
  51. if selected == NOUS_MANAGED_PROVIDER:
  52. gateway = resolve_managed_tool_gateway("fal-queue")
  53. if gateway is None:
  54. raise ValueError(selection_error(
  55. "image_gen", NOUS_MANAGED_PROVIDER,
  56. "the Nous Tool Gateway is not available (not entitled or unreachable)"))
  57. return gateway
  58. if selected is not None:
  59. if fal_key_is_configured():
  60. return None
  61. raise ValueError(selection_error("image_gen", selected, "FAL_KEY is not set"))
  62. # Never-configured category: legacy credential autodetect (do NOT persist).
  63. return None if fal_key_is_configured() else resolve_managed_tool_gateway("fal-queue")
  64. def _get_managed_fal_client(managed_gateway):
  65. """Reuse the managed FAL client so its internal httpx.Client is not leaked per call."""
  66. global _managed_fal_client, _managed_fal_client_config
  67. client_config = (managed_gateway.gateway_origin.rstrip("/"), managed_gateway.nous_user_token)
  68. with _managed_fal_client_lock:
  69. if _managed_fal_client is None or _managed_fal_client_config != client_config:
  70. # Resolved on this module so monkeypatching ``image_generation_tool.fal_client`` still applies.
  71. _managed_fal_client = _ManagedFalSyncClient(
  72. _load_fal_client(), key=managed_gateway.nous_user_token,
  73. queue_run_origin=managed_gateway.gateway_origin)
  74. _managed_fal_client_config = client_config
  75. return _managed_fal_client
  76. class ImageGenerationInterrupted(Exception):
  77. """Raised when the user interrupts while a FAL job is in flight."""
  78. def _wait_fal_result(handler, *, poll_seconds: float = 0.5):
  79. """Interrupt-aware ``wait_fal_queue_result``: the wait blocks 30-60s, hiding user interrupts.
  80. The get runs on a daemon worker; the interrupt bit is polled between join slices and on
  81. interrupt the worker is abandoned (remote job keeps running).
  82. """
  83. from tools.interrupt import is_interrupted
  84. result_box: list = []
  85. error_box: list = []
  86. def _get():
  87. try:
  88. result_box.append(wait_fal_queue_result(handler))
  89. except BaseException as exc:
  90. error_box.append(exc)
  91. worker = threading.Thread(target=_get, daemon=True, name="fal-result-wait")
  92. worker.start()
  93. while worker.is_alive():
  94. if is_interrupted():
  95. raise ImageGenerationInterrupted(
  96. "Image generation interrupted by user — abandoned the in-flight FAL job.")
  97. worker.join(timeout=poll_seconds)
  98. if error_box:
  99. raise error_box[0]
  100. return result_box[0] if result_box else None
  101. def _submit_fal_request(model: str, arguments: dict[str, Any]):
  102. """Submit a FAL request using direct credentials or the managed queue gateway."""
  103. _load_fal_client()
  104. request_headers = {"x-idempotency-key": str(uuid.uuid4())}
  105. managed_gateway = _resolve_managed_fal_gateway()
  106. if managed_gateway is None:
  107. return fal_client.submit(model, arguments=arguments, headers=request_headers)
  108. try:
  109. return submit_managed_fal_with_rate_limit_retry(
  110. lambda headers: _get_managed_fal_client(managed_gateway).submit(
  111. model, arguments=arguments, headers=headers),
  112. what="image model", name=model)
  113. except Exception as exc:
  114. # A managed-gateway 4xx usually means the portal doesn't proxy this model
  115. # (allowlist miss, billing gate): give remediation instead of a raw httpx error.
  116. status = _extract_http_status(exc)
  117. if status is not None and 400 <= status < 500:
  118. billing = _managed_fal_billing_error(exc, "model")
  119. if billing is not None:
  120. raise ValueError(
  121. f"Nous Subscription gateway rejected model '{model}' (HTTP {status}): {billing}") from exc
  122. gateway_message = ""
  123. if status in {401, 402, 403}:
  124. gateway_message = "\n\n" + nous_tool_gateway_unavailable_message(
  125. "managed FAL image generation", force_fresh=True)
  126. raise ValueError(
  127. f"Nous Subscription gateway rejected model '{model}' (HTTP {status}). This model "
  128. f"may not yet be enabled on the Nous Portal's FAL proxy. Either:\n"
  129. f" • Set FAL_KEY in your environment to use FAL.ai directly, or\n"
  130. f" • Pick a different model via `hermes tools` → Image Generation."
  131. f"{gateway_message}") from exc
  132. raise
  133. # --- Config readers, model resolution + payload construction ---
  134. def _read_image_gen_key(key: str) -> Optional[str]:
  135. """Return the stripped ``image_gen.<key>`` string from config.yaml, or None."""
  136. try:
  137. from hermes_cli.config import load_config
  138. cfg = load_config()
  139. section = cfg.get("image_gen") if isinstance(cfg, dict) else None
  140. value = section.get(key) if isinstance(section, dict) else None
  141. if isinstance(value, str) and value.strip():
  142. return value.strip()
  143. except Exception as exc:
  144. logger.debug("Could not read image_gen.%s: %s", key, exc)
  145. return None
  146. def _read_configured_image_model():
  147. """``image_gen.model`` from config.yaml, or None."""
  148. return _read_image_gen_key("model")
  149. def _read_configured_image_provider():
  150. """``image_gen.provider`` from config.yaml, or None (unset keeps the in-tree FAL fallback even
  151. when other providers are registered; ``"fal"`` routes via ``plugins/image_gen/fal/``).
  152. We only consult the plugin registry when this is explicitly set — an unset value keeps users on the
  153. in-tree FAL fallback even when other providers happen to be registered (e.g. a user has OPENAI_API_KEY
  154. set for other features but never asked for OpenAI image gen). ``"fal"`` explicitly routes through
  155. ``plugins/image_gen/fal/`` (which delegates back into this module's pipeline via call-time indirection —
  156. see issue #26241).
  157. """
  158. return _read_image_gen_key("provider")
  159. def _plugin_provider_name() -> Optional[str]:
  160. """Configured provider that must go through the plugin registry; None for unset/fal/nous."""
  161. configured = _read_configured_image_provider()
  162. if not configured or configured in ("fal", NOUS_MANAGED_PROVIDER):
  163. return None
  164. return configured
  165. def _resolve_fal_model() -> tuple:
  166. """Return ``(model_id, meta)`` for the configured FAL model, falling back to DEFAULT_MODEL (warned) when unknown."""
  167. # FAL_IMAGE_MODEL is an undocumented escape hatch (backward-compat for tests/scripts).
  168. model_id = _read_image_gen_key("model") or os.getenv("FAL_IMAGE_MODEL", "").strip()
  169. if model_id and model_id not in FAL_MODELS:
  170. logger.warning("Unknown FAL model '%s' in config; falling back to %s", model_id, DEFAULT_MODEL)
  171. model_id = None
  172. model_id = model_id or DEFAULT_MODEL
  173. return model_id, FAL_MODELS[model_id]
  174. _SIZE_KEY_BY_STYLE = {"image_size_preset": "image_size", "gpt_literal": "image_size",
  175. "aspect_ratio": "aspect_ratio"}
  176. def _build_payload(model_id, prompt, aspect_ratio, seed, overrides, image_urls=None) -> dict[str, Any]:
  177. """Text-to-image / edit payload (``image_urls`` selects edit mode): defaults + native size
  178. spec + overrides, filtered to the model whitelist.
  179. Edit endpoints mostly auto-infer size, so the size key is sent only when ``edit_supports``
  180. lists it. ``prompt`` (and the source-image key on edits) survive a whitelist gap: every FAL
  181. endpoint requires them, so a catalog mistake can't send a broken request.
  182. """
  183. meta = FAL_MODELS[model_id]
  184. edit = image_urls is not None
  185. supports = (meta.get("edit_supports") or set()) if edit else meta["supports"]
  186. sizes = meta["sizes"]
  187. aspect = (aspect_ratio or DEFAULT_ASPECT_RATIO).lower().strip()
  188. if aspect not in sizes:
  189. aspect = DEFAULT_ASPECT_RATIO
  190. payload: dict[str, Any] = dict(meta.get("defaults", {}))
  191. payload["prompt"] = (prompt or "").strip()
  192. required = {"prompt"}
  193. if edit: # a few edit endpoints (Kling Image v3) take a singular `image_url` string instead of the list
  194. image_param = meta.get("edit_image_param") or "image_urls"
  195. payload[image_param] = next(iter(image_urls)) if image_param != "image_urls" else list(image_urls)
  196. required.add(image_param)
  197. size_key = _SIZE_KEY_BY_STYLE.get(meta["size_style"])
  198. if size_key is None and not edit:
  199. raise ValueError(f"Unknown size_style: {meta['size_style']!r}")
  200. if size_key is not None and (not edit or size_key in supports):
  201. payload[size_key] = sizes[aspect]
  202. if isinstance(seed, int):
  203. payload["seed"] = seed
  204. payload.update({k: v for k, v in (overrides or {}).items() if v is not None})
  205. return {k: v for k, v in payload.items() if k in supports or k in required}
  206. def _build_fal_payload(model_id, prompt, aspect_ratio=DEFAULT_ASPECT_RATIO, seed=None, overrides=None):
  207. """FAL text-to-image payload for ``model_id`` from unified inputs."""
  208. return _build_payload(model_id, prompt, aspect_ratio, seed, overrides)
  209. def _build_fal_edit_payload(model_id, prompt, image_urls, aspect_ratio=DEFAULT_ASPECT_RATIO,
  210. seed=None, overrides=None):
  211. """FAL *edit* (image-to-image) payload: ``image_urls`` + prompt, filtered to ``edit_supports``."""
  212. return _build_payload(model_id, prompt, aspect_ratio, seed, overrides, image_urls=image_urls)
  213. # --- Upscaler ---
  214. def _upscale_image(image_url: str, original_prompt: str) -> Optional[dict[str, Any]]:
  215. """Upscale via FAL's Clarity Upscaler; None on failure (caller keeps the original)."""
  216. try:
  217. logger.info("Upscaling image with Clarity Upscaler...")
  218. handler = _submit_fal_request(UPSCALER_MODEL, arguments={
  219. "image_url": image_url, "prompt": f"{UPSCALER_DEFAULT_PROMPT}, {original_prompt}",
  220. "upscale_factor": UPSCALER_FACTOR, "negative_prompt": UPSCALER_NEGATIVE_PROMPT,
  221. "creativity": UPSCALER_CREATIVITY, "resemblance": UPSCALER_RESEMBLANCE,
  222. "guidance_scale": UPSCALER_GUIDANCE_SCALE,
  223. "num_inference_steps": UPSCALER_NUM_INFERENCE_STEPS,
  224. "enable_safety_checker": UPSCALER_SAFETY_CHECKER})
  225. result = _wait_fal_result(handler)
  226. if result and "image" in result:
  227. up = result["image"]
  228. logger.info("Image upscaled successfully to %sx%s",
  229. up.get("width", "unknown"), up.get("height", "unknown"))
  230. return {
  231. "url": up["url"], "width": up.get("width", 0), "height": up.get("height", 0),
  232. "upscaled": True, "upscale_factor": UPSCALER_FACTOR}
  233. logger.error("Upscaler returned invalid response")
  234. return None
  235. except ImageGenerationInterrupted:
  236. # A user interrupt must not degrade into a silent "use original" fallback.
  237. raise
  238. except Exception as e:
  239. logger.error("Error upscaling image: %s", e, exc_info=True)
  240. return None
  241. # --- Artifact path hinting for non-local terminal backends ---
  242. _CONTAINER_HOME_ENVS = {"DockerEnvironment", "ModalEnvironment"}
  243. # No env yet: only deterministic cache roots translate side-effect free (SSH: tilde path; its
  244. # first sync uploads the cache file).
  245. _CACHE_BASE_BY_BACKEND = {"docker": "/root/.hermes", "modal": "/root/.hermes", "ssh": "~/.hermes"}
  246. def _looks_like_absolute_file_path(value: str) -> bool:
  247. if not value or not isinstance(value, str) or value.lower().startswith(("http://", "https://", "data:")):
  248. return False
  249. return os.path.isabs(value) or (len(value) >= 3 and value[1] == ":" and value[2] in {"/", "\\"})
  250. def _active_terminal_env(task_id: str | None):
  251. try:
  252. from tools.terminal_tool_lifecycle import get_active_env
  253. return get_active_env(task_id or "default")
  254. except Exception as exc:
  255. logger.debug("Could not inspect active terminal environment: %s", exc)
  256. return None
  257. def _agent_cache_base_for_env(env: Any) -> str | None:
  258. if env is not None:
  259. # Optional extension hook: an environment may expose its own agent-visible cache root.
  260. explicit = getattr(env, "agent_visible_cache_base", None)
  261. if callable(explicit):
  262. try:
  263. value = explicit()
  264. if value:
  265. return str(value).rstrip("/")
  266. except Exception as exc:
  267. logger.debug("active env agent_visible_cache_base failed: %s", exc)
  268. remote_home = getattr(env, "_remote_home", None)
  269. if remote_home:
  270. return f"{str(remote_home).rstrip('/')}/.hermes"
  271. if env.__class__.__name__ in _CONTAINER_HOME_ENVS:
  272. return "/root/.hermes"
  273. from tools.terminal_scope import terminal_env
  274. backend = (terminal_env("TERMINAL_ENV") or "local").strip().lower()
  275. return _CACHE_BASE_BY_BACKEND.get(backend)
  276. def _postprocess_image_generate_result(raw: str, task_id: str | None = None) -> str:
  277. """Annotate successful local results: ``image`` stays the host/gateway-deliverable path;
  278. ``agent_visible_image`` is the same file as seen by a non-local terminal backend."""
  279. try:
  280. payload = json.loads(raw) if isinstance(raw, str) else raw
  281. except Exception:
  282. return raw
  283. if not isinstance(payload, dict) or not payload.get("success"):
  284. return raw
  285. image = payload.get("image")
  286. if not isinstance(image, str) or not _looks_like_absolute_file_path(image):
  287. return raw
  288. env = _active_terminal_env(task_id)
  289. cache_base = _agent_cache_base_for_env(env)
  290. if not cache_base:
  291. return raw
  292. try:
  293. from tools.credential_files import map_cache_path_to_container
  294. agent_path = map_cache_path_to_container(image, container_base=cache_base)
  295. except Exception as exc:
  296. logger.debug("Could not translate image cache path for backend: %s", exc)
  297. return raw
  298. if not agent_path or agent_path == image:
  299. return raw
  300. sync_manager = getattr(env, "_sync_manager", None)
  301. if sync_manager is not None:
  302. try:
  303. sync_manager.sync(force=True)
  304. except Exception as exc:
  305. logger.warning("Could not force-sync generated image artifact: %s", exc)
  306. payload.setdefault("host_image", image)
  307. payload.setdefault("agent_visible_image", agent_path)
  308. return json.dumps(payload, ensure_ascii=False)
  309. # --- Tool entry point ---
  310. def _format_images(images: list, should_upscale: bool, prompt: str) -> list:
  311. """Normalize FAL result images, optionally chaining the upscaler (falls back to the original on failure)."""
  312. formatted = []
  313. for img in images:
  314. if not (isinstance(img, dict) and "url" in img):
  315. continue
  316. if should_upscale:
  317. upscaled = _upscale_image(img["url"], prompt.strip())
  318. if upscaled:
  319. formatted.append(upscaled)
  320. continue
  321. logger.warning("Using original image as fallback (upscale failed)")
  322. formatted.append({
  323. "url": img["url"], "width": img.get("width", 0), "height": img.get("height", 0),
  324. "upscaled": False})
  325. return formatted
  326. # The managed FAL gateway answers HTTP 413 above ~4.5 MB of request body (probed: 4.0 MB of base64
  327. # passes, 4.5 MB fails), so inlined sources share this budget and are downscaled to fit it.
  328. _FAL_INLINE_BUDGET = 3_500_000
  329. def _inline_fal_sources(sources: list) -> list:
  330. """FAL only fetches URLs, so local paths become ``data:`` URLs through the guarded resolver
  331. (credential-read guard, image sniff) on every terminal backend; then fit the gateway budget."""
  332. from model_tools import _run_async
  333. from tools.image_source import ImageResolutionError, resolve_local_source_to_data_url
  334. try:
  335. urls = [_run_async(resolve_local_source_to_data_url(s, None)) for s in sources]
  336. except ImageResolutionError as exc:
  337. raise ValueError(f"Could not read source image: {exc}") from exc
  338. inline = [u for u in urls if u.startswith("data:")]
  339. if sum(map(len, inline)) <= _FAL_INLINE_BUDGET:
  340. return urls
  341. share = _FAL_INLINE_BUDGET // len(inline)
  342. return [_shrink_data_url(u, share) if u.startswith("data:") and len(u) > share else u for u in urls]
  343. def _shrink_data_url(url: str, budget: int) -> str:
  344. """Re-encode a ``data:`` image (JPEG unless it has alpha), shrinking the long edge until it fits."""
  345. import base64
  346. import io
  347. from PIL import Image, ImageOps
  348. try:
  349. img = Image.open(io.BytesIO(base64.b64decode(url.partition(",")[2])))
  350. img.load()
  351. except (OSError, ValueError, Image.DecompressionBombError):
  352. return url # not a raster Pillow reads (e.g. SVG): send as-is and let FAL report it
  353. img = ImageOps.exif_transpose(img) # re-encoding drops EXIF, so bake the orientation into the pixels
  354. has_alpha = "A" in img.getbands() or "transparency" in img.info # palette PNGs carry alpha in info
  355. fmt, mime = ("PNG", "image/png") if has_alpha else ("JPEG", "image/jpeg")
  356. img = img.convert("RGBA") if has_alpha else img.convert("RGB")
  357. side = max(img.size)
  358. while True:
  359. frame = img.copy()
  360. frame.thumbnail((side, side))
  361. buf = io.BytesIO()
  362. frame.save(buf, fmt, quality=90)
  363. out = f"data:{mime};base64,{base64.b64encode(buf.getvalue()).decode('ascii')}"
  364. if len(out) <= budget or side <= 256:
  365. logger.info("Downscaled FAL source image to %s (%d bytes inline)", frame.size, len(out))
  366. return out
  367. side = int(side * 0.75)
  368. def _prepare_fal_request(model_id, meta, prompt, aspect_ratio, seed, overrides, source_images):
  369. """Validate inputs and return ``(endpoint, arguments)``; raises ValueError with the user-facing message."""
  370. if not isinstance(prompt, str) or not prompt.strip():
  371. raise ValueError("Prompt is required and must be a non-empty string")
  372. # A stored-but-broken selection raises the selection-naming error from
  373. # _resolve_managed_fal_gateway(); only never-configured reports "no backend at all".
  374. if not (fal_key_is_configured() or _resolve_managed_fal_gateway()):
  375. raise ValueError(_build_no_backend_setup_message())
  376. edit_endpoint, display = meta.get("edit_endpoint"), meta.get("display", model_id)
  377. # Fail clearly rather than silently dropping sources and producing an unrelated picture.
  378. if source_images and not edit_endpoint:
  379. raise ValueError(
  380. f"Model '{display}' ({model_id}) is not capable of image-to-image / editing. "
  381. f"Provide a text-only prompt (omit image_url), or switch to an edit-capable model "
  382. f"via `hermes tools` → Image Generation.")
  383. aspect_lc = (aspect_ratio or DEFAULT_ASPECT_RATIO).lower().strip()
  384. if aspect_lc not in VALID_ASPECT_RATIOS:
  385. logger.warning("Invalid aspect_ratio '%s', defaulting to '%s'", aspect_ratio, DEFAULT_ASPECT_RATIO)
  386. aspect_lc = DEFAULT_ASPECT_RATIO
  387. if source_images:
  388. # Clamp reference count to the model's declared cap.
  389. max_refs = int(meta.get("max_reference_images") or 1)
  390. clamped_sources = _inline_fal_sources(source_images[:max_refs] if max_refs > 0 else source_images)
  391. arguments = _build_fal_edit_payload(
  392. model_id, prompt, clamped_sources, aspect_lc, seed=seed, overrides=overrides)
  393. logger.info("Editing image with %s (%s) — %d source image(s), prompt: %s",
  394. display, edit_endpoint, len(clamped_sources), prompt[:80])
  395. return edit_endpoint, arguments
  396. arguments = _build_fal_payload(model_id, prompt, aspect_lc, seed=seed, overrides=overrides)
  397. logger.info("Generating image with %s (%s) — prompt: %s", display, model_id, prompt[:80])
  398. return model_id, arguments
  399. def image_generate_tool(
  400. prompt: str, aspect_ratio: str = DEFAULT_ASPECT_RATIO,
  401. num_inference_steps: Optional[int] = None, guidance_scale: Optional[float] = None,
  402. num_images: Optional[int] = None, output_format: Optional[str] = None,
  403. seed: Optional[int] = None, image_url: Optional[str] = None,
  404. reference_image_urls: Optional[list] = None, upscale: Optional[bool] = None,
  405. fal_model: Optional[str] = None) -> str:
  406. """Generate (or, with source images + an ``edit_endpoint`` model, edit) an image via FAL.
  407. Extra kwargs are overrides filtered per-model via ``supports`` / ``edit_supports`` (dropped
  408. silently so callers survive model switches). Returns JSON ``{"success", "image", "modality",
  409. "error", "error_type"}``.
  410. """
  411. if fal_model in FAL_MODELS:
  412. model_id, meta = fal_model, FAL_MODELS[fal_model]
  413. else:
  414. model_id, meta = _resolve_fal_model()
  415. refs = reference_image_urls if isinstance(reference_image_urls, (list, tuple)) else []
  416. source_images = [c.strip() for c in (image_url, *refs) if isinstance(c, str) and c.strip()]
  417. use_edit = bool(source_images) and bool(meta.get("edit_endpoint"))
  418. modality = "image" if use_edit else "text"
  419. overrides: dict[str, Any] = {
  420. "num_inference_steps": num_inference_steps, "guidance_scale": guidance_scale,
  421. "num_images": num_images, "output_format": output_format}
  422. debug_call_data = {
  423. "model": model_id,
  424. "parameters": {"prompt": prompt, "aspect_ratio": aspect_ratio, **overrides, "seed": seed,
  425. "modality": modality, "source_images": len(source_images)},
  426. "error": None, "success": False, "images_generated": 0, "generation_time": 0}
  427. start_time = datetime.datetime.now()
  428. def finish(generation_time: float, response: dict[str, Any]) -> str:
  429. debug_call_data["generation_time"] = generation_time
  430. _debug.log_call("image_generate_tool", debug_call_data)
  431. _debug.save()
  432. return json.dumps(response, indent=2, ensure_ascii=False)
  433. try:
  434. endpoint, arguments = _prepare_fal_request(
  435. model_id, meta, prompt, aspect_ratio, seed,
  436. {k: v for k, v in overrides.items() if v is not None}, source_images)
  437. result = _wait_fal_result(_submit_fal_request(endpoint, arguments=arguments))
  438. generation_time = (datetime.datetime.now() - start_time).total_seconds()
  439. if not result or "images" not in result:
  440. raise ValueError("Invalid response from FAL.ai API — no images returned")
  441. images = result.get("images", [])
  442. if not images:
  443. raise ValueError("No images were generated")
  444. # Explicit ``upscale`` wins, including for edits; the catalog default never upscales
  445. # edits (Clarity is a text-to-image pass and must not silently alter compositions).
  446. if upscale is not None:
  447. should_upscale = bool(upscale)
  448. else:
  449. should_upscale = bool(meta.get("upscale", False)) and not use_edit
  450. formatted_images = _format_images(images, should_upscale, prompt)
  451. if not formatted_images:
  452. raise ValueError("No valid image URLs returned from API")
  453. upscaled_count = sum(1 for img in formatted_images if img.get("upscaled"))
  454. logger.info("Generated %s image(s) in %.1fs (%s upscaled) via %s [%s]",
  455. len(formatted_images), generation_time, upscaled_count, endpoint, modality)
  456. debug_call_data["success"] = True
  457. debug_call_data["images_generated"] = len(formatted_images)
  458. return finish(generation_time, {
  459. "success": True,
  460. "image": formatted_images[0]["url"],
  461. "modality": modality,
  462. "upscaled": bool(formatted_images[0].get("upscaled"))})
  463. except Exception as e:
  464. error_msg = f"Error generating image: {e!s}"
  465. logger.error("%s", error_msg, exc_info=True)
  466. debug_call_data["error"] = error_msg
  467. generation_time = (datetime.datetime.now() - start_time).total_seconds()
  468. return finish(generation_time,
  469. {"success": False, "image": None, "error": str(e), "error_type": type(e).__name__})
  470. def check_fal_api_key() -> bool:
  471. """True if the selected FAL backend (never configured: any FAL backend) is available.
  472. A stored-but-broken selection reports False here (registry gating); the naming error
  473. surfaces at call time from ``_resolve_managed_fal_gateway``.
  474. """
  475. try:
  476. gateway = _resolve_managed_fal_gateway()
  477. except ValueError:
  478. return False
  479. return bool(gateway) or fal_key_is_configured()
  480. def _build_no_backend_setup_message() -> str:
  481. """Actionable no-backend error: FAL_KEY signup, managed-gateway status, plugin alternative."""
  482. managed = managed_nous_tools_enabled()
  483. lines = ["Image generation is unavailable in this environment.", "", "Missing requirements:"]
  484. if managed:
  485. lines.append(" - FAL_KEY is not set and the managed FAL gateway is unreachable")
  486. else:
  487. lines.append(" - FAL_KEY environment variable is not set")
  488. if gateway_message := nous_tool_gateway_unavailable_message("managed FAL image generation"):
  489. lines.append(f" - {gateway_message}")
  490. lines += ["", "To enable image generation, do one of:",
  491. " 1. Get a free API key at https://fal.ai and set FAL_KEY=<your-key> "
  492. "(then restart the session)"]
  493. if managed:
  494. lines.append(" 2. Sign in to a Nous account that has the managed FAL gateway enabled "
  495. "(`hermes setup`)")
  496. lines.append(" 3. Configure a different image_gen provider via `hermes tools` → Image Generation "
  497. "(run `hermes plugins list` to see installed backends)")
  498. return "\n".join(lines)
  499. def _get_plugin_provider(name: str, *, force: bool = False):
  500. """Discover plugins (local import: importing this module must not trigger discovery) and return the named provider."""
  501. from agent.image_gen_registry import get_provider
  502. from hermes_cli.plugins import _ensure_plugins_discovered
  503. if force:
  504. _ensure_plugins_discovered(force=True)
  505. else:
  506. _ensure_plugins_discovered()
  507. return get_provider(name)
  508. def check_image_generation_requirements() -> bool:
  509. """True if FAL or the explicitly configured image backend is available."""
  510. try:
  511. if check_fal_api_key():
  512. # Lazy import doubles as the SDK presence check: ImportError falls through to plugins.
  513. _load_fal_client()
  514. return True
  515. except ImportError:
  516. pass
  517. configured = _plugin_provider_name()
  518. if configured is None:
  519. return False
  520. # Probe only the selected plugin: a cloud key alone must not opt a user into a paid backend.
  521. provider = _get_plugin_provider(configured)
  522. return bool(provider and provider.is_available())
  523. # --- Registry ---
  524. from tools.registry import registry, tool_error
  525. IMAGE_GENERATE_SCHEMA = {
  526. "name": "image_generate",
  527. # Placeholder: description AND params are rebuilt at get_tool_definitions() time by
  528. # _build_dynamic_image_schema() from the active backend's capabilities. Edit-only args
  529. # and upscale are advertised ONLY when supported; the handler accepts them regardless
  530. # (replay compat + teaching errors).
  531. "description": (
  532. "Generate or edit images with the configured model. Edit/reference "
  533. "capabilities are rendered at serving time."
  534. ),
  535. "parameters": {
  536. "type": "object",
  537. "properties": {
  538. "prompt": {
  539. "type": "string",
  540. "description": (
  541. "The text prompt describing the desired image (text-to-"
  542. "image) or the edit to apply (image-to-image). Be detailed "
  543. "and descriptive."
  544. ),
  545. },
  546. "aspect_ratio": {
  547. "type": "string",
  548. "enum": list(VALID_ASPECT_RATIOS),
  549. "description": "The aspect ratio of the generated image. 'landscape' is 16:9 wide, 'portrait' is 16:9 tall, 'square' is 1:1.",
  550. "default": DEFAULT_ASPECT_RATIO,
  551. },
  552. # image_url / reference_image_urls / upscale are added per-capability; never statically.
  553. },
  554. # See #95681.
  555. "required": ["prompt"],
  556. },
  557. }
  558. # --- Plugin provider dispatch + managed-mode Krea routing ---
  559. def _provider_error(error: str, error_type: str) -> str:
  560. """JSON error envelope shared by every provider-dispatch failure path."""
  561. return json.dumps({"success": False, "image": None, "error": error, "error_type": error_type})
  562. def _provider_result(result, contract_error: str) -> str:
  563. """JSON-encode a provider's dict result; anything else is a contract violation."""
  564. if not isinstance(result, dict):
  565. return _provider_error(contract_error, "provider_contract")
  566. return json.dumps(result)
  567. def _add_provider_kwargs(kwargs, image_url, reference_image_urls, upscale, model=None, controls=None) -> dict[str, Any]:
  568. """Add the optional ``provider.generate(**kwargs)`` args in place (edit args only when supplied)."""
  569. if model:
  570. kwargs["model"] = model
  571. if controls:
  572. kwargs.update(controls)
  573. if isinstance(image_url, str) and image_url.strip():
  574. kwargs["image_url"] = image_url.strip()
  575. if reference_image_urls is not None:
  576. from agent.image_gen_provider import normalize_reference_images
  577. norm_refs = normalize_reference_images(reference_image_urls)
  578. if norm_refs:
  579. kwargs["reference_image_urls"] = norm_refs
  580. if upscale is not None:
  581. kwargs["upscale"] = bool(upscale)
  582. return kwargs
  583. def _declared_controls(provider, controls: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
  584. """The subset of ``controls`` that ``provider`` declares in ``creative_controls``.
  585. A model can still send a control the current schema no longer offers (it copies earlier turns
  586. after the backend changed), and a third-party ``generate()`` without ``**kwargs`` would raise."""
  587. if not controls:
  588. return None
  589. try:
  590. declared = (provider.capabilities() or {}).get("creative_controls") or ()
  591. except Exception:
  592. return None
  593. return {name: value for name, value in controls.items() if name in declared} or None
  594. def _dispatch_to_plugin_provider(
  595. prompt: str, aspect_ratio: str, image_url: Optional[str] = None,
  596. reference_image_urls: Optional[list] = None, upscale: Optional[bool] = None,
  597. controls: Optional[dict[str, Any]] = None):
  598. """JSON result from the selected plugin provider, or ``None`` to fall through to in-tree FAL
  599. (provider unset / ``"fal"`` / ``"nous"``). Providers without ``upscale`` ignore it via ``**kwargs``."""
  600. configured = _plugin_provider_name()
  601. if configured is None:
  602. return None
  603. try:
  604. provider = _get_plugin_provider(configured)
  605. except Exception as exc:
  606. logger.debug("image_gen plugin dispatch skipped: %s", exc)
  607. if _read_image_gen_key("allow_provider_fallback") is False:
  608. return _provider_error(f"Could not load image provider '{configured}': {exc}", "provider_not_registered")
  609. return None
  610. if provider is None:
  611. # Long-lived sessions may have discovered plugins before a bundled backend
  612. # was patched in or config changed: retry once with a forced refresh.
  613. try:
  614. provider = _get_plugin_provider(configured, force=True)
  615. except Exception as exc:
  616. logger.debug("image_gen plugin force-refresh skipped: %s", exc)
  617. if provider is None:
  618. return _provider_error(
  619. f"image_gen.provider='{configured}' is set but no plugin registered that name. "
  620. f"Run `hermes plugins list` to see available image gen backends.", "provider_not_registered")
  621. pname = getattr(provider, "name", "?")
  622. kwargs: dict[str, Any] = {"prompt": prompt, "aspect_ratio": aspect_ratio}
  623. try:
  624. _add_provider_kwargs(kwargs, image_url, reference_image_urls, upscale,
  625. model=_read_configured_image_model(), controls=_declared_controls(provider, controls))
  626. result = provider.generate(**kwargs)
  627. except Exception as exc:
  628. # A TypeError from generate() predating image_url support (third-party plugin not yet
  629. # updated): text-to-image keeps working; surface a clear note when an edit was requested.
  630. is_type_error = isinstance(exc, TypeError)
  631. if is_type_error and ("image_url" in kwargs or "reference_image_urls" in kwargs):
  632. logger.warning("image_gen provider '%s' rejected image-to-image kwargs "
  633. "(signature too narrow): %s", pname, exc)
  634. return _provider_error(
  635. f"Provider '{pname}' does not support image-to-image / editing (its generate() "
  636. f"signature is out of date with the image_generate schema). Omit image_url for "
  637. f"text-to-image, or pick a backend that supports editing via `hermes tools` → "
  638. f"Image Generation.", "modality_unsupported")
  639. logger.warning("Image gen provider '%s' raised%s: %s", pname,
  640. " TypeError" if is_type_error else "", exc)
  641. return _provider_error(f"Provider '{pname}' error: {exc}", "provider_exception")
  642. return _provider_result(result, "Provider returned a non-dict result")
  643. def _normalize_krea_model(model_id: Optional[str]) -> Optional[str]:
  644. """Return ``model_id`` when it is one of the Krea plugin's model ids, else ``None``."""
  645. from plugins.image_gen.krea import KREA_MODEL_IDS
  646. candidate = model_id.strip() if isinstance(model_id, str) else None
  647. return candidate if candidate in KREA_MODEL_IDS else None
  648. def _managed_model_plugin() -> Optional[tuple]:
  649. """``(plugin_name, model_id)`` when the stored selection routes to the Krea or Portal gateway
  650. (rule: :func:`tools.image_generation_managed.managed_route`), else ``None`` for the FAL path."""
  651. from tools.image_generation_managed import KREA, PORTAL, managed_route
  652. model_id = _read_configured_image_model()
  653. return {KREA: ("krea", model_id), PORTAL: ("nous", model_id)}.get(
  654. managed_route(_read_configured_image_provider(), model_id))
  655. def _maybe_route_managed_model(
  656. prompt: str, aspect_ratio: str, image_url: Optional[str] = None,
  657. reference_image_urls: Optional[list] = None, upscale: Optional[bool] = None,
  658. controls: Optional[dict[str, Any]] = None) -> Optional[str]:
  659. """JSON result from the Krea or Portal gateway the stored model belongs to, or ``None`` to fall
  660. through to FAL.
  661. A Krea model with no reachable Krea gateway falls through (direct/BYO users keep their
  662. pipeline); a Portal model never does — falling through would silently bill a FAL default.
  663. """
  664. target = _managed_model_plugin()
  665. if target is None:
  666. return None
  667. plugin_name, model_id = target
  668. try:
  669. if plugin_name == "krea":
  670. from plugins.image_gen.krea import _resolve_managed_krea_gateway
  671. if _resolve_managed_krea_gateway() is None:
  672. if _read_image_gen_key("allow_provider_fallback") is False:
  673. return _provider_error("The selected Krea image gateway is unavailable.", "provider_not_registered")
  674. return None
  675. provider = _get_plugin_provider(plugin_name)
  676. except Exception as exc:
  677. logger.debug("Managed %s routing unavailable: %s", plugin_name, exc)
  678. provider = None
  679. if provider is None:
  680. if plugin_name == "krea" and _read_image_gen_key("allow_provider_fallback") is not False:
  681. return None
  682. if plugin_name == "krea":
  683. return _provider_error("The selected Krea image backend is unavailable.", "provider_not_registered")
  684. return _provider_error(
  685. f"image_gen.model='{model_id}' is a Nous Portal model but the Portal image backend is not "
  686. f"available. Pick another model via `hermes tools` → Image Generation.", "provider_not_registered")
  687. kwargs: dict[str, Any] = {"prompt": prompt, "aspect_ratio": aspect_ratio, "model": model_id}
  688. try:
  689. _add_provider_kwargs(kwargs, image_url, reference_image_urls, upscale,
  690. controls=_declared_controls(provider, controls))
  691. result = provider.generate(**kwargs)
  692. except Exception as exc:
  693. logger.warning("Managed %s routing failed: %s", plugin_name, exc)
  694. return _provider_error(f"Managed {provider.display_name} generation error: {exc}", "provider_exception")
  695. return _provider_result(result, f"{provider.display_name} provider returned a non-dict result")
  696. def _confine_source_images(image_url, reference_image_urls, task_id, *, permitted: tuple = ("image",)):
  697. """Resolve path-like sources to ``data:`` URLs under a non-local terminal backend.
  698. Routes through ``tools.image_source`` (in-sandbox exec-read, media-cache host reads,
  699. credential guard) so generation obeys the same confinement as vision. URLs/data: pass
  700. through; local backend is a no-op. Returns ``(image_url, reference_image_urls, error_json_or_None)``.
  701. """
  702. from tools.terminal_scope import terminal_env
  703. if (terminal_env("TERMINAL_ENV") or "local").strip().lower() in ("", "local"):
  704. return image_url, reference_image_urls, None
  705. from model_tools import _run_async
  706. from tools.image_source import ImageResolutionError, resolve_local_source_to_data_url
  707. def resolve(ref):
  708. return _run_async(resolve_local_source_to_data_url(ref, task_id, permitted=permitted))
  709. try:
  710. if isinstance(image_url, str) and image_url.strip():
  711. image_url = resolve(image_url)
  712. if isinstance(reference_image_urls, (list, tuple)):
  713. reference_image_urls = [resolve(r) if isinstance(r, str) else r for r in reference_image_urls]
  714. except ImageResolutionError as exc:
  715. return image_url, reference_image_urls, _provider_error(
  716. f"Could not read source image: {exc}", type(exc).__name__)
  717. return image_url, reference_image_urls, None
  718. def _handle_image_generate(args, **kw):
  719. prompt = args.get("prompt", "")
  720. if not prompt:
  721. return tool_error("prompt is required for image generation")
  722. aspect_ratio = args.get("aspect_ratio", DEFAULT_ASPECT_RATIO)
  723. upscale = args.get("upscale")
  724. task_id = kw.get("task_id")
  725. # Confinement chokepoint BEFORE any dispatch: every route receives sandbox-confined bytes.
  726. image_url, reference_image_urls, confine_error = _confine_source_images(
  727. args.get("image_url"), args.get("reference_image_urls"), task_id)
  728. if confine_error is not None:
  729. return confine_error
  730. # Order matters: explicit plugin provider, then the model-driven managed gateways (Krea /
  731. # Portal — only under the "nous"/unset selection, so BYO/direct FAL stays untouched), then FAL.
  732. sources = dict(image_url=image_url, reference_image_urls=reference_image_urls,
  733. upscale=upscale if isinstance(upscale, bool) else None)
  734. controls = {name: args[name] for name in _CREATIVE_CONTROL_PARAMS if name in args}
  735. raw = _dispatch_to_plugin_provider(prompt, aspect_ratio, controls=controls or None, **sources)
  736. if raw is None:
  737. raw = _maybe_route_managed_model(prompt, aspect_ratio, controls=controls or None, **sources)
  738. if raw is not None:
  739. raw = _fall_back_to_fal(raw, prompt, aspect_ratio, sources)
  740. if raw is None:
  741. raw = image_generate_tool(prompt, aspect_ratio, **sources)
  742. return _postprocess_image_generate_result(raw, task_id=task_id)
  743. def _fall_back_to_fal(raw: str, prompt: str, aspect_ratio: str, sources: dict[str, Any]) -> str:
  744. """Rerun a managed Krea request on FAL's default model when Krea refused it before creating a job.
  745. Only the Krea plugin marks a failure ``fallback_eligible``, and only when the submit provably never
  746. created a job: a job that exists has already been authorized for billing and may still complete.
  747. Source images never fall back; FAL would route them to an edit endpoint with different semantics."""
  748. if _read_image_gen_key("allow_provider_fallback") is False:
  749. return raw
  750. try:
  751. result = json.loads(raw)
  752. except ValueError:
  753. return raw
  754. if not isinstance(result, dict) or result.get("success") or not result.get("fallback_eligible"):
  755. return raw
  756. if sources.get("image_url") or sources.get("reference_image_urls"):
  757. return raw
  758. from plugins.image_gen.krea import fallback_to_fal_enabled
  759. if not fallback_to_fal_enabled():
  760. return raw
  761. logger.warning("Managed Krea refused the request before starting a job; falling back to FAL: %s", result.get("error"))
  762. fal_result = json.loads(image_generate_tool(prompt, aspect_ratio, upscale=sources.get("upscale"), fal_model=DEFAULT_MODEL))
  763. note = (f"Krea was unavailable, so this image was generated on FAL ({DEFAULT_MODEL}) instead."
  764. if fal_result.get("success") else
  765. f"Krea was unavailable and the fallback to FAL ({DEFAULT_MODEL}) also failed.")
  766. if fal_result.get("upscaled"):
  767. note += f" The upscale ran on FAL's {UPSCALER_MODEL}, not Krea Enhance."
  768. fal_result.update(
  769. provider="fal", model=DEFAULT_MODEL, fallback_from="krea", fallback_reason=result.get("error"), note=note)
  770. return json.dumps(fal_result)
  771. # --- Dynamic schema — reflect the active backend's image-to-image capability ---
  772. # Advertising edit capability up front saves a wasted turn. Memoized by config.yaml mtime in
  773. # model_tools.get_tool_definitions(), so it rebuilds on switch.
  774. _NO_CAPABILITIES = {"modalities": ["text"], "max_reference_images": 0, "supports_upscale": False}
  775. def _active_image_capabilities() -> dict[str, Any]:
  776. """Best-effort capabilities of the active backend/model; never raises.
  777. Mirrors runtime dispatch: a Krea or Portal model id under the managed selection asks that
  778. plugin, a set ``image_gen.provider`` asks that plugin, else the FAL catalog.
  779. Fail-closed: an undeclared capability is advertised as absent.
  780. """
  781. info: dict[str, Any] = dict(_NO_CAPABILITIES)
  782. configured_provider = _read_configured_image_provider()
  783. managed = _managed_model_plugin()
  784. if managed is not None:
  785. plugin_name = managed[0]
  786. elif configured_provider and configured_provider != "fal":
  787. plugin_name = configured_provider
  788. else:
  789. plugin_name = None
  790. if plugin_name:
  791. try:
  792. provider = _get_plugin_provider(plugin_name)
  793. if provider is not None:
  794. try:
  795. caps = provider.capabilities() or {}
  796. except Exception:
  797. caps = {}
  798. info["provider"] = provider.display_name
  799. info["model"] = _read_configured_image_model() or (provider.default_model() or "")
  800. if caps.get("modalities"):
  801. info["modalities"] = list(caps["modalities"])
  802. if caps.get("max_reference_images"):
  803. info["max_reference_images"] = int(caps["max_reference_images"])
  804. # Plugins opt in explicitly; absent = no upscale param.
  805. info["supports_upscale"] = bool(caps.get("supports_upscale"))
  806. if caps.get("creative_controls"):
  807. info["creative_controls"] = list(caps["creative_controls"])
  808. if caps.get("source_image_role") == "style":
  809. info["source_image_role"] = "style"
  810. return info
  811. except Exception:
  812. pass
  813. # In-tree FAL path (provider unset or == "fal"); _resolve_fal_model() never raises.
  814. model_id, meta = _resolve_fal_model()
  815. can_edit = bool(meta.get("edit_endpoint"))
  816. info["provider"] = "FAL.ai"
  817. info["model"] = meta.get("display", model_id)
  818. info["modalities"] = ["text", "image"] if can_edit else ["text"]
  819. info["max_reference_images"] = int(meta.get("max_reference_images") or 1) if can_edit else 0
  820. # Clarity is available on request for ANY catalog model (``upscale`` is only the default).
  821. info["supports_upscale"] = True
  822. return info
  823. # Param snippets assembled per-capability by _build_dynamic_image_schema.
  824. _IMAGE_URL_PARAM = {
  825. "type": "string",
  826. "description": (
  827. "Source image to edit/transform (image-to-image). A public URL or "
  828. "an absolute local file path from the conversation. Omit for "
  829. "text-to-image."
  830. ),
  831. }
  832. # Creative-control vocabulary (Krea 2 and Krea 3 today); a provider advertises the names it honors
  833. # via ``capabilities()["creative_controls"]`` and only those reach the schema and its ``generate()``.
  834. _CREATIVE_CONTROL_PARAMS = {
  835. "creativity": {
  836. "type": "string", "enum": ["raw", "low", "medium", "high"],
  837. "description": "Prompt expansion: raw (none), low, medium or high.",
  838. },
  839. "thinking": {
  840. "type": "string", "enum": ["fast", "high"],
  841. "description": "How much the model plans before generating: fast (default) or high for complex multi-step edits.",
  842. },
  843. "resolution": {
  844. "type": "string", "enum": ["1K", "2K"],
  845. "description": "Output size: 1K (default) or 2K when small text or fine detail matters.",
  846. },
  847. "intensity": {
  848. "type": "integer", "minimum": -100, "maximum": 100,
  849. "description": "Style intensity, -100 muted to 100 highly stylized. 0 neutral.",
  850. },
  851. "complexity": {
  852. "type": "integer", "minimum": -100, "maximum": 100,
  853. "description": "Composition density, -100 minimal to 100 dense. 0 neutral.",
  854. },
  855. "movement": {
  856. "type": "integer", "minimum": -100, "maximum": 100,
  857. "description": "Motion in the scene, -100 static to 100 dynamic. 0 neutral.",
  858. },
  859. }
  860. _STYLE_PROMPT_PARAM = {
  861. "type": "string",
  862. "description": (
  863. "The subject and scene to generate. Be detailed and descriptive; "
  864. "reference images supply only the look."
  865. ),
  866. }
  867. _STYLE_IMAGE_URL_PARAM = {
  868. "type": "string",
  869. "description": (
  870. "Style reference: its look is copied, the image itself is not edited. "
  871. "A public URL or an absolute local file path. Describe the subject in "
  872. "the prompt. Omit for text-to-image."
  873. ),
  874. }
  875. _UPSCALE_PARAM = {
  876. "type": "boolean",
  877. "description": (
  878. "Post-generation high-resolution pass (~2x, extra cost/latency), "
  879. "off by default. A creative enhancer that can alter fine detail "
  880. "(rendered text, faces) — use only when resolution matters more "
  881. "than fidelity."
  882. ),
  883. }
  884. def _build_dynamic_image_schema() -> dict[str, Any]:
  885. """Render description AND params from the active model's capabilities; args it cannot
  886. honor are NOT advertised (the handler still accepts them for replay compat)."""
  887. base_desc = (
  888. "{lead} {edit_clause}Use this rather than calling image APIs or scripts directly. "
  889. "Returns the result in the `image` field — a URL or an absolute "
  890. "file path; reference it in your response using the current "
  891. "platform's file-delivery convention."
  892. )
  893. info = _active_image_capabilities()
  894. max_refs = int(info.get("max_reference_images") or 0)
  895. can_edit = "image" in set(info.get("modalities") or ["text"])
  896. static_props = IMAGE_GENERATE_SCHEMA["parameters"]["properties"]
  897. properties: dict[str, Any] = {
  898. "prompt": static_props["prompt"], "aspect_ratio": static_props["aspect_ratio"]}
  899. style_refs = info.get("source_image_role") == "style"
  900. if can_edit:
  901. if style_refs:
  902. lead = "Generate images, optionally styled on reference images, via configured model."
  903. edit_clause = "Pass style references in image_url. "
  904. properties["prompt"] = _STYLE_PROMPT_PARAM
  905. properties["image_url"] = _STYLE_IMAGE_URL_PARAM
  906. refs_desc = f"Up to {max_refs} more style references. URLs or absolute local paths."
  907. else:
  908. lead = ("Generate or edit images (with reference images) using the configured model."
  909. if max_refs > 1 else "Generate or edit images using the configured image model.")
  910. edit_clause = "Pass image_url to edit / transform an existing image. "
  911. properties["image_url"] = _IMAGE_URL_PARAM
  912. refs_desc = (
  913. f"Up to {max_refs} additional reference images (style, "
  914. "character, or composition) guiding an edit. URLs or "
  915. "absolute local paths."
  916. )
  917. if max_refs > 1:
  918. properties["reference_image_urls"] = {
  919. "type": "array",
  920. "items": {"type": "string"},
  921. "maxItems": max_refs,
  922. "description": refs_desc,
  923. }
  924. else:
  925. lead = "Generate images from text with the configured model (it cannot edit images)."
  926. edit_clause = ""
  927. if info.get("supports_upscale"):
  928. properties["upscale"] = _UPSCALE_PARAM
  929. for name in info.get("creative_controls") or []:
  930. if name in _CREATIVE_CONTROL_PARAMS:
  931. properties[name] = _CREATIVE_CONTROL_PARAMS[name]
  932. return {"description": base_desc.format(lead=lead, edit_clause=edit_clause),
  933. "parameters": {"type": "object", "properties": properties, "required": ["prompt"]}}
  934. registry.register(
  935. name="image_generate", toolset="image_gen", schema=IMAGE_GENERATE_SCHEMA,
  936. handler=_handle_image_generate, check_fn=check_image_generation_requirements, requires_env=[],
  937. is_async=False, # sync fal_client API to avoid "Event loop is closed" in gateway
  938. emoji="🎨", dynamic_schema_overrides=_build_dynamic_image_schema,
  939. )