__init__.py 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834
  1. """OpenRouter-compatible image generation backend (OpenRouter + Nous Portal).
  2. Both speak the OpenAI-style ``/chat/completions`` image protocol (``modalities:
  3. ["image","text"]``, references as ``image_url`` parts, output in
  4. ``choices[0].message.images[].image_url.url``); only ``(base_url, api_key)`` differs,
  5. via :func:`hermes_cli.runtime_provider.resolve_runtime_provider`. OpenRouter alone also
  6. has the Dedicated Image API (``POST /images/generations``, own catalog, exact ratios,
  7. up to 16 references) — see :func:`_select_surface`.
  8. """
  9. from __future__ import annotations
  10. import base64
  11. import json
  12. import logging
  13. import mimetypes
  14. import os
  15. import time
  16. from pathlib import Path
  17. from typing import Any, Dict, List, Optional, Tuple
  18. from agent.image_gen_provider import (
  19. DEFAULT_ASPECT_RATIO, ImageGenProvider, error_response, resolve_aspect_ratio, save_b64_image,
  20. save_url_image, success_response)
  21. from plugins.image_gen._common import error_factory, load_image_gen_config, post_json, record_token_usage
  22. logger = logging.getLogger(__name__)
  23. # Quality-first default chain: OpenAI first, Gemini 3 Pro Image when it is gated /
  24. # unavailable / times out. Any explicit override is exact — no auto fallback.
  25. DEFAULT_MODEL = "openai/gpt-5.4-image-2"
  26. _FALLBACK_MODEL = "google/gemini-3-pro-image"
  27. _DEFAULT_MODEL_CHAIN = (DEFAULT_MODEL, _FALLBACK_MODEL)
  28. _MODEL_PRIORITY = {DEFAULT_MODEL: 0, _FALLBACK_MODEL: 1}
  29. # Semantic aspect ratio → OpenRouter ``image_config.aspect_ratio``.
  30. _ASPECT_RATIOS = {"square": "1:1", "landscape": "16:9", "portrait": "9:16"}
  31. _MAX_REFERENCE_IMAGES = 3 # Gemini Flash Image accepts up to 3 input images per prompt.
  32. _REQUEST_TIMEOUT = 300.0 # per image call; a cold quality-first row can run past 3 minutes.
  33. # Curated metadata for well-known chat-completions image models.
  34. _KNOWN_MODEL_META = {
  35. DEFAULT_MODEL: {
  36. "display": "OpenAI GPT-5.4 Image 2",
  37. "strengths": "Highest fidelity; best prompt adherence; slower on OpenRouter",
  38. },
  39. _FALLBACK_MODEL: {
  40. "display": "Gemini 3 Pro Image",
  41. "strengths": "Fast, reliable fallback with good layout adherence",
  42. },
  43. }
  44. _EXCLUDED_MODEL_PREFIXES = ("openrouter/auto",) # router pseudo-models advertise image output
  45. _LIVE_CACHE_TTL = 300.0
  46. _LIVE_TIMEOUT = 10.0
  47. _load_image_gen_config = load_image_gen_config
  48. # ---------------------------------------------------------------------------
  49. # OpenRouter Dedicated Image API (POST /images/generations)
  50. # ---------------------------------------------------------------------------
  51. # Knob env prefix (``OPENROUTER_IMAGE_API_QUALITY`` …); ``OPENROUTER_IMAGE_MODEL`` picks the model.
  52. _IMAGE_API_ENV_PREFIX = "OPENROUTER_IMAGE_API_"
  53. # Separate connect budget: no TLS in 20s means the endpoint is down — don't wait out the read budget.
  54. _IMAGE_API_CONNECT_TIMEOUT = 20.0
  55. # ``/images/models`` probes keyed by (base URL, key fingerprint): ``(fetched_at, ids)``; empty sets
  56. # are cached too, so the key must include the credential or one profile's 401 would pin a sibling
  57. # profile (same base URL, different key) to chat-completions for the whole TTL.
  58. _CATALOG_TTL_SECONDS = 900.0
  59. _CATALOG_CACHE: dict[tuple[str, Optional[str]], tuple[float, frozenset]] = {}
  60. _GEMINI_RATIOS = (
  61. "1:1", "1:4", "1:8", "2:3", "3:2", "3:4", "4:1", "4:3", "4:5", "5:4", "8:1", "9:16", "16:9", "21:9",
  62. )
  63. _MAI_RATIOS = ("1:1", "4:3", "3:4", "16:9", "9:16", "3:2", "2:3", "auto")
  64. _KREA_RATIOS = ("1:1", "4:3", "3:2", "16:9", "4:5", "2:3", "9:16")
  65. _OPENAI_QUALITY = ("auto", "low", "medium", "high")
  66. _NO_KNOBS = {"quality": (), "background": (), "output_format": (), "compression": False, "seed": False}
  67. def _image_api_model(display: str, strengths: str, **spec: Any) -> dict[str, Any]:
  68. """Curated Image API model entry; unspecified knobs default to "not supported"."""
  69. return {"display": display, "strengths": strengths, **_NO_KNOBS, "resolutions": (), **spec}
  70. # Curated Image API models + the parameters each declares in ``GET /images/models``. An
  71. # id missing here still works (no per-model knob filtering; one cached catalog probe).
  72. # Keys mirror the payload field they gate; an empty tuple means "no such knob".
  73. _IMAGE_API_MODELS: dict[str, dict[str, Any]] = {
  74. "google/gemini-3.1-flash-lite-image": _image_api_model(
  75. "Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)",
  76. "Cheap and fast; 14 exact aspect ratios; 14 reference images",
  77. aspect_ratios=_GEMINI_RATIOS, resolutions=("1K",), max_n=1, max_refs=14),
  78. "google/gemini-3.1-flash-image": _image_api_model(
  79. "Nano Banana 2 (Gemini 3.1 Flash Image)",
  80. "Same ratios as Lite plus resolution control (512/1K/2K/4K)",
  81. aspect_ratios=_GEMINI_RATIOS, resolutions=("512", "1K", "2K", "4K"), max_n=1, max_refs=14),
  82. "openai/gpt-image-2": _image_api_model(
  83. "OpenAI GPT Image 2",
  84. "Best editing fidelity; up to 16 references; strongest prompt adherence",
  85. aspect_ratios=("1:1", "3:2", "2:3", "4:3", "3:4", "16:9", "9:16", "21:9", "auto"),
  86. quality=_OPENAI_QUALITY, background=("auto", "opaque"), compression=True, max_n=10, max_refs=16,
  87. ),
  88. "openai/gpt-image-1-mini": _image_api_model(
  89. "OpenAI GPT Image 1 Mini",
  90. "The only model here with background=transparent (cut-out PNG)",
  91. aspect_ratios=("1:1", "3:2", "2:3", "auto"),
  92. quality=_OPENAI_QUALITY, background=("auto", "transparent", "opaque"), compression=True,
  93. max_n=10, max_refs=16),
  94. "microsoft/mai-image-2.5": _image_api_model(
  95. "Microsoft MAI-Image-2.5", "Standard ratios; a good second opinion next to Gemini",
  96. aspect_ratios=_MAI_RATIOS, max_n=1, max_refs=1),
  97. "microsoft/mai-image-2.5-pro": _image_api_model(
  98. "Microsoft MAI-Image-2.5 Pro", "Reach for it when gpt-image-2 misses the brief",
  99. aspect_ratios=_MAI_RATIOS, max_n=1, max_refs=1),
  100. "x-ai/grok-imagine-image-2.0": _image_api_model(
  101. "Grok Imagine Image 2.0",
  102. "Photoreal; widest exotic-ratio set (9:19.5, 20:9, 2:1 …); 1K/2K; low/medium quality",
  103. aspect_ratios=(
  104. "1:1", "3:4", "4:3", "9:16", "16:9", "2:3", "3:2", "9:19.5", "19.5:9", "9:20", "20:9", "1:2", "2:1",
  105. "auto"),
  106. resolutions=("1K", "2K"), quality=("low", "medium"), max_n=1, max_refs=3),
  107. "krea/krea-2-medium": _image_api_model(
  108. "Krea 2 Medium", "Realistic, expressive styles; deterministic via seed",
  109. aspect_ratios=_KREA_RATIOS, resolutions=("1K",), seed=True, max_n=1, max_refs=1),
  110. "krea/krea-2-medium-turbo": _image_api_model(
  111. "Krea 2 Medium Turbo", "Cheapest here — bulk content, cards, thumbnails; seed support",
  112. aspect_ratios=_KREA_RATIOS, resolutions=("1K",), seed=True, max_n=1, max_refs=1),
  113. "qwen/qwen-image-3-pro": _image_api_model(
  114. "Qwen Image 3 Pro", "Precise small text and detail rendering; n up to 6; 1K/2K; seed",
  115. aspect_ratios=("1:1", "1:2", "1:4", "2:1", "2:3", "3:2", "3:4", "4:1", "4:3", "4:5", "5:4", "9:16", "16:9"),
  116. resolutions=("1K", "2K"), seed=True, max_n=6, max_refs=4),
  117. }
  118. # For catalog models the table doesn't describe; empty ``aspect_ratios`` = unknown enum → omitted.
  119. _UNKNOWN_IMAGE_API_MODEL = _image_api_model("", "", aspect_ratios=(), max_n=1, max_refs=16)
  120. # Union of exact ratios across all models; sanity-checks an override for a model with unknown enum.
  121. _ENDPOINT_ASPECT_RATIOS = frozenset({
  122. "1:1", "1:2", "1:4", "1:8", "2:1", "2:3", "3:2", "3:4", "4:1", "4:3", "4:5", "5:4", "8:1", "9:16",
  123. "16:9", "9:19.5", "19.5:9", "9:20", "20:9", "9:21", "21:9", "auto",
  124. })
  125. # Semantic ratio → exact ratios, best first (``landscape`` degrades to 3:2 where 16:9 is missing).
  126. _ASPECT_PREFERENCES: dict[str, tuple[str, ...]] = {
  127. "landscape": ("16:9", "3:2", "4:3", "5:4", "21:9", "2:1", "19.5:9", "20:9", "4:1", "8:1"),
  128. "portrait": ("9:16", "2:3", "3:4", "4:5", "9:21", "1:2", "9:19.5", "9:20", "1:4", "1:8"),
  129. "square": ("1:1",),
  130. }
  131. _MEDIA_TYPE_EXTENSIONS = {
  132. "image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg", "image/webp": "webp", "image/gif": "gif",
  133. "image/svg+xml": "svg",
  134. }
  135. # Statuses worth retrying on the next chain model. 400 is our own payload (would repeat);
  136. # 401/403 are account-level and answered first; 502 is how the Image API reports an unbilled failure.
  137. _IMAGE_API_FALLBACK_STATUSES = frozenset({402, 404, 408, 409, 425, 429, 500, 502, 503, 504})
  138. # Stay on ``/chat/completions`` whatever the image catalog says — the tested defaults.
  139. _CHAT_ONLY_MODELS = frozenset({DEFAULT_MODEL, _FALLBACK_MODEL})
  140. _IMAGE_API_EXTRA_KEYS = ("resolution", "quality", "background", "output_format", "seed", "n")
  141. # Enum knobs: payload field → catalog key holding the allowed values.
  142. _IMAGE_API_ENUMS = (
  143. ("resolution", "resolutions"), ("quality", None), ("background", None), ("output_format", None))
  144. # Integer knobs: payload field → (catalog gate flag, clamp).
  145. _IMAGE_API_INTS = (
  146. ("output_compression", "compression", lambda v: max(0, min(100, v))), ("seed", "seed", lambda v: v),
  147. )
  148. _ATTRIBUTION_HEADERS = {
  149. "Content-Type": "application/json",
  150. # OpenRouter attribution headers (harmless against Nous Portal).
  151. "HTTP-Referer": "https://github.com/NousResearch/hermes-agent",
  152. "X-Title": "Hermes Agent",
  153. }
  154. def _to_image_url_part(ref: str) -> Optional[str]:
  155. """URLs pass through; local files inline as base64 data URIs (``None`` when unreadable)."""
  156. ref = str(ref or "").strip()
  157. if not ref:
  158. return None
  159. if ref.startswith(("http://", "https://", "data:")):
  160. return ref
  161. path = Path(ref)
  162. from agent.file_safety import raise_if_read_blocked # credential-read guard before inlining
  163. raise_if_read_blocked(ref)
  164. try:
  165. raw = path.read_bytes()
  166. except OSError as exc:
  167. logger.debug("could not read reference image %s: %s", ref, exc)
  168. return None
  169. mime = mimetypes.guess_type(path.name)[0] or "image/png"
  170. return f"data:{mime};base64,{base64.b64encode(raw).decode('ascii')}"
  171. def _dict_at(node: Any, key: str) -> dict[str, Any]:
  172. value = node.get(key) if isinstance(node, dict) else None
  173. return value if isinstance(value, dict) else {}
  174. def _list_at(node: Any, key: str) -> list[Any]:
  175. value = node.get(key) if isinstance(node, dict) else None
  176. return value if isinstance(value, list) else []
  177. def _extract_images(payload: dict[str, Any]) -> list[str]:
  178. """Generated image URLs from ``choices[].message.images[].image_url.url``."""
  179. out: list[str] = []
  180. for choice in _list_at(payload, "choices"):
  181. for image in _list_at(_dict_at(choice, "message"), "images"):
  182. url = _dict_at(image, "image_url").get("url")
  183. if isinstance(url, str) and url.strip():
  184. out.append(url.strip())
  185. return out
  186. def _access_error_hint(display: str, model_id: str, env_var: str, status: int, err_msg: str) -> Optional[str]:
  187. """Hint for an access-gated OpenAI image model (valid key, but the *model* needs account
  188. enablement / BYOK, so "check your key" would mislead). ``None`` otherwise."""
  189. if not model_id.startswith("openai/"):
  190. return None
  191. low = (err_msg or "").lower()
  192. gated = status in (402, 403, 404) or any(
  193. s in low for s in ("no endpoints", "no allowed", "not a valid model", "data policy"))
  194. if not gated:
  195. return None
  196. return (
  197. f"{display} can't reach image model '{model_id}' ({status}) — enable OpenAI "
  198. f"image access in your {display} account, or set {env_var}={_FALLBACK_MODEL}.")
  199. def _get_catalog(base_url: str, path: str, api_key: str, timeout: Any) -> list[tuple[str, dict[str, Any]]]:
  200. """``(model_id, entry)`` pairs from ``GET {base_url}{path}``'s ``data[]``; raises on HTTP failure."""
  201. import requests
  202. response = requests.get(
  203. f"{base_url}{path}", headers={"Authorization": f"Bearer {api_key}"} if api_key else {}, timeout=timeout,
  204. )
  205. response.raise_for_status()
  206. out: list[tuple[str, dict[str, Any]]] = []
  207. for entry in _list_at(response.json(), "data"):
  208. model_id = entry.get("id") if isinstance(entry, dict) else None
  209. if isinstance(model_id, str) and model_id.strip():
  210. out.append((model_id.strip(), entry))
  211. return out
  212. def _fetch_catalog(
  213. base_url: str, api_key: str, *, path: str, meta: dict[str, dict[str, Any]], generic: str,
  214. image_output_only: bool,
  215. ) -> list[dict[str, Any]]:
  216. """Picker rows from ``GET {base_url}{path}``; raises on failure. ``image_output_only`` keeps
  217. image-output models minus router pseudo-models; curated ``meta`` wins for known ids."""
  218. out: list[dict[str, Any]] = []
  219. for model_id, entry in _get_catalog(base_url, path, api_key, _LIVE_TIMEOUT):
  220. arch = _dict_at(entry, "architecture")
  221. if image_output_only and (
  222. model_id.startswith(_EXCLUDED_MODEL_PREFIXES) or "image" not in (arch.get("output_modalities") or [])
  223. ):
  224. continue
  225. known = meta.get(model_id, {})
  226. out.append({
  227. "id": model_id,
  228. "display": known.get("display", entry.get("name") or model_id),
  229. "strengths": known.get("strengths", generic),
  230. "input_modalities": arch.get("input_modalities") or [],
  231. })
  232. return out
  233. def _fetch_image_api_catalog(base_url: str, api_key: str) -> frozenset:
  234. """Model ids from ``GET {base_url}/images/models``, cached per (base URL, key). Any failure caches
  235. an empty set (→ chat-completions): guessing "images" would 404 a working chat setup."""
  236. from agent.credential_persistence import fingerprint_secret_value
  237. cache_key = (base_url, fingerprint_secret_value(api_key))
  238. cached = _CATALOG_CACHE.get(cache_key)
  239. if cached and (time.monotonic() - cached[0]) < _CATALOG_TTL_SECONDS:
  240. return cached[1]
  241. ids: set = set()
  242. try:
  243. catalog = _get_catalog(base_url, "/images/models", api_key, (_IMAGE_API_CONNECT_TIMEOUT, 30.0))
  244. ids = {model_id for model_id, _entry in catalog}
  245. except Exception as exc:
  246. logger.debug("image API catalog probe failed for %s: %s", base_url, exc)
  247. resolved = frozenset(ids)
  248. _CATALOG_CACHE[cache_key] = (time.monotonic(), resolved)
  249. return resolved
  250. def _image_api_model_meta(model_id: str) -> dict[str, Any]:
  251. """Catalog metadata for *model_id*, or permissive defaults when unknown."""
  252. return _IMAGE_API_MODELS.get(model_id, _UNKNOWN_IMAGE_API_MODEL)
  253. def _select_surface(model_id: str, base_url: str, api_key: str, config_key: str) -> str:
  254. """``"images"`` or ``"chat"``: config/env ``surface`` forces it; curated ids are offline and
  255. deterministic; unknown ids consult the cached live catalog (a positive probe must route there
  256. or a model the live picker offered would 404; offline they stay on chat)."""
  257. if not model_id:
  258. return "chat"
  259. forced = _image_api_setting("surface", None, config_key)
  260. if isinstance(forced, str) and forced.strip().lower() in {"images", "chat"}:
  261. return forced.strip().lower()
  262. if model_id in _CHAT_ONLY_MODELS:
  263. return "chat"
  264. if model_id in _IMAGE_API_MODELS or model_id in _fetch_image_api_catalog(base_url, api_key):
  265. return "images"
  266. return "chat"
  267. def _image_api_setting(name: str, explicit: Any, config_key: str) -> Any:
  268. """Knob: call kwarg → ``OPENROUTER_IMAGE_API_<NAME>`` env → scoped config; blanks are unset."""
  269. if explicit is not None and not (isinstance(explicit, str) and not explicit.strip()):
  270. return explicit
  271. env_value = os.environ.get(f"{_IMAGE_API_ENV_PREFIX}{name.upper()}", "").strip()
  272. if env_value:
  273. return env_value
  274. value = _dict_at(_load_image_gen_config(), config_key).get(name)
  275. return (value.strip() or None) if isinstance(value, str) else value
  276. def _coerce_int(value: Any) -> Optional[int]:
  277. """Best-effort int (env vars arrive as strings); ``None`` when not numeric."""
  278. if isinstance(value, bool):
  279. return None
  280. if isinstance(value, (int, float)):
  281. return int(value)
  282. try:
  283. return int(value.strip()) if isinstance(value, str) else None
  284. except ValueError:
  285. return None
  286. def _pick_exact_aspect_ratio(
  287. semantic: str, meta: dict[str, Any], forced: Optional[str], notes: list[str]
  288. ) -> Optional[str]:
  289. """Exact ``aspect_ratio`` or ``None`` to omit. *forced* wins when supported; otherwise the
  290. downgrade is noted and the per-model mapping of *semantic* applies."""
  291. supported: tuple[str, ...] = tuple(meta.get("aspect_ratios") or ())
  292. if isinstance(forced, str) and forced.strip():
  293. value = forced.strip()
  294. if (supported and value in supported) or (not supported and value in _ENDPOINT_ASPECT_RATIOS):
  295. return value
  296. notes.append(
  297. f"requested aspect_ratio '{value}' is unsupported by this model; "
  298. f"used the '{semantic}' mapping instead")
  299. if not supported:
  300. # Out-of-enum aspect_ratio is a hard 400 (unknown *parameters* are ignored) — omit it.
  301. notes.append(
  302. "model is not in this backend's catalog, so its aspect_ratio enum is "
  303. f"unknown; the field was omitted and '{semantic}' was not applied")
  304. return None
  305. for candidate in _ASPECT_PREFERENCES.get(semantic, ()):
  306. if candidate in supported:
  307. return candidate
  308. return "auto" if "auto" in supported else supported[0]
  309. def _image_api_enum(
  310. name: str, explicit: Any, meta: dict[str, Any], config_key: str, notes: list[str], *,
  311. meta_key: Optional[str] = None,
  312. ) -> Optional[str]:
  313. """Resolve an enum knob and drop it when this model doesn't accept it."""
  314. value = _image_api_setting(name, explicit, config_key)
  315. if not isinstance(value, str) or not value.strip():
  316. return None
  317. value = value.strip()
  318. allowed: tuple[str, ...] = tuple(meta.get(meta_key or name) or ())
  319. if not allowed:
  320. notes.append(f"'{name}' is not supported by this model; dropped")
  321. return None
  322. if value not in allowed:
  323. notes.append(f"'{name}={value}' is not valid for this model (accepts {', '.join(allowed)}); dropped")
  324. return None
  325. return value
  326. def _build_image_api_payload(
  327. *, model_id: str, prompt: str, semantic_aspect: str, references: list[str], config_key: str,
  328. kwargs: dict[str, Any],
  329. ) -> tuple[dict[str, Any], list[str]]:
  330. """``/images/generations`` body: ``(payload, notes)``. Every knob is filtered against what the
  331. model declares: the endpoint silently *ignores* unknown parameters, which would let a caller
  332. believe ``background=transparent`` took effect on a model without it."""
  333. meta = _image_api_model_meta(model_id)
  334. notes: list[str] = []
  335. payload: dict[str, Any] = {"model": model_id, "prompt": prompt}
  336. forced_ratio = _image_api_setting("aspect_ratio", kwargs.get("aspect_ratio_exact"), config_key)
  337. ratio = _pick_exact_aspect_ratio(semantic_aspect, meta, forced_ratio, notes)
  338. if ratio:
  339. payload["aspect_ratio"] = ratio
  340. for enum_name, meta_key in _IMAGE_API_ENUMS:
  341. value = _image_api_enum(enum_name, kwargs.get(enum_name), meta, config_key, notes, meta_key=meta_key)
  342. if value:
  343. payload[enum_name] = value
  344. for name, gate, clamp in _IMAGE_API_INTS:
  345. value = _coerce_int(_image_api_setting(name, kwargs.get(name), config_key))
  346. if value is None:
  347. continue
  348. if meta.get(gate):
  349. payload[name] = clamp(value)
  350. else:
  351. notes.append(f"'{name}' is not supported by this model; dropped")
  352. count = _coerce_int(_image_api_setting("n", kwargs.get("n"), config_key))
  353. if count is not None and count > 1:
  354. max_n = int(meta.get("max_n") or 1)
  355. if count > max_n:
  356. notes.append(f"'n={count}' exceeds this model's cap of {max_n}; clamped")
  357. payload["n"] = max(1, min(count, max_n))
  358. if references:
  359. max_refs = int(meta.get("max_refs") or 0)
  360. usable = references[:max_refs] if max_refs else []
  361. if len(references) > len(usable):
  362. notes.append(
  363. f"{len(references)} reference image(s) supplied but this model "
  364. f"accepts {max_refs}; extras dropped")
  365. if usable:
  366. payload["input_references"] = [{"type": "image_url", "image_url": {"url": url}} for url in usable]
  367. return payload, notes
  368. def _extract_image_api_error(response: Any, fallback: str) -> str:
  369. """One-line error from ``{"error": {"message"}}`` (routing/auth) or the ZodError shape
  370. ``{"error": {"name": "ZodError", "message": "<json issue array>"}}`` (validation)."""
  371. if response is None:
  372. return fallback
  373. try:
  374. body = response.json()
  375. except Exception:
  376. return (getattr(response, "text", "") or "")[:300] or fallback
  377. error = body.get("error") if isinstance(body, dict) else None
  378. if isinstance(error, str) and error.strip():
  379. return error.strip()
  380. if not isinstance(error, dict):
  381. return json.dumps(body)[:300] if body else fallback
  382. message = error.get("message")
  383. if error.get("name") == "ZodError" and isinstance(message, str):
  384. try:
  385. issues = json.loads(message)
  386. except Exception:
  387. return message[:300]
  388. parts = [
  389. f"{'.'.join(str(p) for p in (issue.get('path') or [])) or 'request'}: "
  390. f"{issue.get('message') or 'invalid'}"
  391. for issue in (issues if isinstance(issues, list) else []) if isinstance(issue, dict)
  392. ]
  393. return "; ".join(parts)[:400] or message[:300]
  394. if isinstance(message, str) and message.strip():
  395. return message.strip()[:300]
  396. return json.dumps(error)[:300]
  397. def _extension_for(media_type: Optional[str], fallback: str = "png") -> str:
  398. if isinstance(media_type, str):
  399. return _MEDIA_TYPE_EXTENSIONS.get(media_type.split(";", 1)[0].strip().lower(), fallback)
  400. return fallback
  401. def _model_slug(model_id: str) -> str:
  402. """Filename-safe fragment for the cache prefix (``openai/gpt-image-2`` …)."""
  403. return "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in model_id)
  404. def _save_image_api_entry(entry: dict[str, Any], prefix: str) -> Optional[str]:
  405. """Cache one ``data[]`` entry; ``None`` without b64/URL. Raises on write failure."""
  406. b64 = entry.get("b64_json")
  407. if isinstance(b64, str) and b64.strip():
  408. return str(save_b64_image(b64, prefix=prefix, extension=_extension_for(entry.get("media_type"))))
  409. url = entry.get("url")
  410. if isinstance(url, str) and url.strip():
  411. return str(save_url_image(url.strip(), prefix=prefix))
  412. return None
  413. def _image_api_extra(
  414. payload: dict[str, Any], saved: list[str], usable_refs: list[str], notes: list[str], body: Any
  415. ) -> dict[str, Any]:
  416. """Success ``extra`` for an Image API result (knobs sent, extra images, usage)."""
  417. extra: dict[str, Any] = {"endpoint": "images/generations", "exact_aspect_ratio": payload.get("aspect_ratio")}
  418. extra.update({key: payload[key] for key in _IMAGE_API_EXTRA_KEYS if key in payload})
  419. if len(saved) > 1:
  420. extra["additional_images"] = saved[1:]
  421. if usable_refs:
  422. extra["reference_images_used"] = len(payload.get("input_references") or [])
  423. if notes:
  424. extra["notes"] = notes
  425. usage = _dict_at(body, "usage")
  426. if isinstance(usage.get("cost"), (int, float)):
  427. extra["cost_usd"] = usage["cost"]
  428. if isinstance(usage.get("total_tokens"), int):
  429. extra["total_tokens"] = usage["total_tokens"]
  430. return extra
  431. class OpenRouterCompatImageProvider(ImageGenProvider):
  432. """One instance per backend (OpenRouter, Nous Portal); they differ only in the runtime
  433. provider supplying ``(base_url, api_key)``, the config namespace, and Image API support."""
  434. def __init__(
  435. self, *, provider_name: str, display_name: str, runtime_name: str, config_key: str,
  436. model_env_var: str, setup_schema: Optional[dict[str, Any]], supports_image_api: bool = False,
  437. ) -> None:
  438. self._name = provider_name
  439. self._display = display_name
  440. self._runtime_name = runtime_name
  441. self._config_key = config_key
  442. self._model_env_var = model_env_var
  443. self._setup_schema = setup_schema
  444. self._live_models_cache: Optional[tuple] = None
  445. self._image_api_models_cache: Optional[tuple] = None
  446. self._supports_image_api = supports_image_api
  447. @property
  448. def name(self) -> str:
  449. return self._name
  450. @property
  451. def display_name(self) -> str:
  452. return self._display
  453. def _credentials(self) -> tuple[str, str]:
  454. """``(api_key, base_url)`` — either may be ``""``; raises on resolution failure."""
  455. from hermes_cli.runtime_provider import resolve_runtime_provider
  456. runtime = resolve_runtime_provider(requested=self._runtime_name)
  457. return (
  458. str(runtime.get("api_key") or "").strip(), str(runtime.get("base_url") or "").strip().rstrip("/"),
  459. )
  460. def is_available(self) -> bool:
  461. try:
  462. return bool(self._credentials()[0])
  463. except Exception as exc:
  464. logger.debug("%s runtime resolution failed: %s", self._name, exc)
  465. return False
  466. def capabilities(self) -> dict[str, Any]:
  467. # Report the reference cap of the model that would service the next call.
  468. max_refs = _MAX_REFERENCE_IMAGES
  469. if self._supports_image_api:
  470. resolved = self._resolve_model_chain()[0]
  471. if resolved in _IMAGE_API_MODELS:
  472. max_refs = int(_IMAGE_API_MODELS[resolved].get("max_refs") or max_refs)
  473. return {"modalities": ["text", "image"], "max_reference_images": max_refs}
  474. def list_models(self) -> list[dict[str, Any]]:
  475. """Live catalog: OpenRouter = ``GET /images/models`` ∪ chat-completions image models (new
  476. releases selectable); Nous Portal = chat-completions only. Offline: default chain + snapshot."""
  477. merged: dict[str, dict[str, Any]] = {}
  478. if self._supports_image_api:
  479. merged = {entry["id"]: entry for entry in self._image_api_live_models()}
  480. for entry in self._live_models():
  481. merged.setdefault(entry["id"], entry)
  482. if merged:
  483. return sorted(merged.values(), key=lambda m: (_MODEL_PRIORITY.get(m["id"], 2), m["id"]))
  484. models = [{"id": model_id, **meta} for model_id, meta in _KNOWN_MODEL_META.items()]
  485. if self._supports_image_api:
  486. models.extend(
  487. {"id": model_id, "display": meta["display"], "strengths": f"{meta['strengths']} (Image API)"}
  488. for model_id, meta in _IMAGE_API_MODELS.items())
  489. return models
  490. def _cached_catalog(self, attr: str, label: str, **fetch: Any) -> list[dict[str, Any]]:
  491. """Cached (per TTL) live catalog rows for this backend; ``[]`` when unreachable."""
  492. cached = getattr(self, attr)
  493. if cached is not None and time.monotonic() - cached[1] < _LIVE_CACHE_TTL:
  494. return cached[0]
  495. models: list[dict[str, Any]] = []
  496. try:
  497. api_key, base_url = self._credentials()
  498. if base_url:
  499. models = _fetch_catalog(base_url, api_key, **fetch)
  500. except Exception as exc:
  501. logger.debug("%s live %s unavailable: %s", self._name, label, exc)
  502. models = []
  503. setattr(self, attr, (models, time.monotonic()))
  504. return models
  505. def _image_api_live_models(self) -> list[dict[str, Any]]:
  506. return self._cached_catalog(
  507. "_image_api_models_cache", "Image API catalog", path="/images/models", meta=_IMAGE_API_MODELS,
  508. generic="Image API model (from live OpenRouter catalog)", image_output_only=False)
  509. def _live_models(self) -> list[dict[str, Any]]:
  510. return self._cached_catalog(
  511. "_live_models_cache", "image model catalog", path="/models", meta=_KNOWN_MODEL_META,
  512. generic="Image-output model (from live OpenRouter catalog)", image_output_only=True)
  513. def default_model(self) -> Optional[str]:
  514. # The catalog default, not the effective runtime model (_resolve_model_chain).
  515. return DEFAULT_MODEL
  516. def get_setup_schema(self) -> Optional[dict[str, Any]]:
  517. return dict(self._setup_schema) if self._setup_schema else None
  518. def _resolve_model(self, explicit: Optional[str] = None) -> str:
  519. return self._resolve_model_chain(explicit)[0]
  520. def _resolve_model_chain(self, explicit: Optional[str] = None) -> list[str]:
  521. """``model`` kwarg → ``*_IMAGE_MODEL`` env → ``image_gen.<provider>.model`` →
  522. ``image_gen.model`` → default chain. Only the bare default chain carries a fallback."""
  523. for candidate in (explicit, os.environ.get(self._model_env_var)):
  524. if isinstance(candidate, str) and candidate.strip():
  525. return [candidate.strip()]
  526. cfg = _load_image_gen_config()
  527. for candidate in (_dict_at(cfg, self._config_key).get("model"), cfg.get("model")):
  528. if isinstance(candidate, str) and candidate.strip():
  529. return [candidate.strip()]
  530. return list(_DEFAULT_MODEL_CHAIN)
  531. def _generate_via_image_api(
  532. self, *, model_id: str, prompt: str, semantic_aspect: str, references: list[str],
  533. base_url: str, headers: dict[str, str], kwargs: dict[str, Any],
  534. ) -> dict[str, Any]:
  535. """One ``POST {base_url}/images/generations`` attempt; a chain-retryable failure carries a
  536. private ``_retryable`` flag that :meth:`generate` strips."""
  537. base_fail = error_factory(self._name, semantic_aspect, model=model_id, prompt=prompt)
  538. def _fail(error: str, error_type: str, retryable: bool = False) -> dict[str, Any]:
  539. response = base_fail(error, error_type)
  540. if retryable:
  541. response["_retryable"] = True
  542. return response
  543. # Not the chat path's `content`: that is clamped to 3 refs, these models take up to 16.
  544. usable_refs: list[str] = []
  545. unreadable: list[str] = []
  546. try:
  547. for ref in references:
  548. part = _to_image_url_part(ref)
  549. (usable_refs if part else unreadable).append(part or str(ref))
  550. except Exception as exc:
  551. return _fail(f"Could not load reference image: {exc}", "io_error")
  552. # An edit with no readable source must not silently bill a text-to-image picture.
  553. if unreadable and not usable_refs:
  554. return _fail(
  555. "Could not read the reference image(s) requested for editing: "
  556. + ", ".join(unreadable) + ". Refusing to silently fall back to text-to-image.",
  557. "io_error")
  558. payload, notes = _build_image_api_payload(
  559. model_id=model_id, prompt=prompt, semantic_aspect=semantic_aspect,
  560. references=usable_refs, config_key=self._config_key, kwargs=kwargs)
  561. if unreadable:
  562. notes.insert(0, f"dropped unreadable reference image(s): {', '.join(unreadable)}")
  563. timeout = _REQUEST_TIMEOUT
  564. configured = _image_api_setting("timeout", kwargs.get("timeout"), self._config_key)
  565. try:
  566. if configured is not None:
  567. timeout = max(1.0, float(configured))
  568. except (TypeError, ValueError):
  569. logger.debug("%s: ignoring non-numeric image API timeout %r", self._name, configured)
  570. # (connect, read): an unreachable endpoint fails in seconds, not the whole read budget.
  571. body, failure = post_json(
  572. f"{base_url}/images/generations", headers=headers, payload=payload,
  573. timeout=(min(_IMAGE_API_CONNECT_TIMEOUT, timeout), timeout), label=self._display,
  574. error_message=lambda resp, exc: _extract_image_api_error(resp, str(exc)),
  575. catch_request_exception=True)
  576. if failure is not None:
  577. if failure.kind != "http":
  578. return _fail(failure.error, failure.error_type, retryable=failure.kind == "timeout")
  579. status, message = failure.status, failure.message
  580. logger.error("%s image API generation failed (%s) on %s: %s", self._name, status, model_id, message)
  581. # 401/403 first so an account-level rejection gets one error_type anywhere in the chain.
  582. if status in (401, 403):
  583. return _fail(f"{self._display} rejected the API key ({status}): {message}", "auth_error")
  584. if status == 404:
  585. return _fail(
  586. f"Model '{model_id}' does not exist on the OpenRouter Image API "
  587. f"(its catalog is separate from chat-completions — check "
  588. f"GET {base_url}/images/models).",
  589. "model_access", retryable=True)
  590. return _fail(failure.error, "api_error", retryable=status in _IMAGE_API_FALLBACK_STATUSES)
  591. # The provider billed these tokens on HTTP 200 whether or not an image came back / saved.
  592. record_token_usage(_dict_at(body, "usage"), model=model_id, provider=self._name, base_url=base_url)
  593. entries = [e for e in _list_at(body, "data") if isinstance(e, dict)]
  594. if not entries:
  595. return _fail(
  596. f"{self._display} returned no image data for '{model_id}'.", "empty_response", retryable=True,
  597. )
  598. prefix = f"{self._name}_{_model_slug(model_id)}"
  599. try:
  600. saved = [p for p in (_save_image_api_entry(e, prefix) for e in entries) if p]
  601. except Exception as exc:
  602. return _fail(f"Could not save generated image: {exc}", "io_error")
  603. if not saved:
  604. return _fail(f"{self._display} response carried neither b64_json nor url.", "empty_response")
  605. return success_response(
  606. image=saved[0], model=model_id, prompt=prompt, aspect_ratio=semantic_aspect, provider=self._name,
  607. modality="image" if usable_refs else "text",
  608. extra=_image_api_extra(payload, saved, usable_refs, notes, body))
  609. def _generate_via_chat(
  610. self, *, model_id: str, prompt: str, aspect: str, content: list[dict[str, Any]],
  611. base_url: str, headers: dict[str, str],
  612. ) -> tuple[dict[str, Any], Optional[str]]:
  613. """One ``/chat/completions`` attempt: ``(result, retry_reason)``; reason set when the
  614. chain may continue with the fallback model."""
  615. fail = error_factory(self._name, aspect, model=model_id, prompt=prompt)
  616. payload: dict[str, Any] = {
  617. "model": model_id,
  618. "modalities": ["image", "text"],
  619. "messages": [{"role": "user", "content": content}],
  620. "image_config": {"aspect_ratio": _ASPECT_RATIOS.get(aspect, "1:1")},
  621. }
  622. result, failure = post_json(
  623. f"{base_url}/chat/completions", headers=headers, payload=payload, timeout=_REQUEST_TIMEOUT,
  624. label=self._display)
  625. if failure is not None:
  626. if failure.kind != "http":
  627. reason = "timed out" if failure.kind == "timeout" else None
  628. return fail(failure.error, failure.error_type), reason
  629. status, message = failure.status, failure.message
  630. logger.error("%s image gen failed (%d) on %s: %s", self._name, status, model_id, message)
  631. hint = _access_error_hint(self._display, model_id, self._model_env_var, status, message)
  632. if hint:
  633. return fail(hint, "model_access"), "unavailable"
  634. return fail(failure.error, "api_error"), None
  635. # The provider billed these tokens on HTTP 200 whether or not an image came back / saved.
  636. record_token_usage(_dict_at(result, "usage"), model=model_id, provider=self._name, base_url=base_url)
  637. images = _extract_images(result)
  638. if not images:
  639. # Text but no image usually means the model didn't honor image output.
  640. return fail(
  641. f"{self._display} returned no image. Ensure the model '{model_id}' supports image output.",
  642. "empty_response",
  643. ), "returned no image"
  644. first = images[0]
  645. try:
  646. if first.startswith("data:"):
  647. b64 = first.split(",", 1)[1] if "," in first else ""
  648. saved_path = save_b64_image(b64, prefix=f"{self._name}_gen")
  649. else:
  650. saved_path = save_url_image(first, prefix=f"{self._name}_gen")
  651. except Exception as exc:
  652. return fail(f"Could not save generated image: {exc}", "io_error"), None
  653. return success_response(
  654. image=str(saved_path), model=model_id, prompt=prompt, aspect_ratio=aspect, provider=self._name,
  655. ), None
  656. def generate(
  657. self, prompt: str, aspect_ratio: str = DEFAULT_ASPECT_RATIO, *,
  658. image_url: Optional[str] = None, reference_image_urls: Optional[list[str]] = None,
  659. **kwargs: Any,
  660. ) -> dict[str, Any]:
  661. fail = error_factory(self._name, aspect_ratio)
  662. try:
  663. api_key, base_url = self._credentials()
  664. except Exception as exc:
  665. return fail(f"Could not resolve {self._display} credentials: {exc}", "missing_api_key")
  666. if not api_key or not base_url:
  667. return fail(
  668. f"No {self._display} credentials found. "
  669. f"Configure {self._display} in `hermes tools` → Image Generation.",
  670. "missing_api_key")
  671. model_chain = self._resolve_model_chain(kwargs.get("model"))
  672. aspect = resolve_aspect_ratio(aspect_ratio)
  673. # ``reference_images`` (pet generator, local paths) + the generic ``image_url`` surface.
  674. references = [str(ref) for ref in kwargs.get("reference_images") or []]
  675. if image_url:
  676. references.append(str(image_url))
  677. references.extend(str(ref) for ref in reference_image_urls or [])
  678. content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
  679. for ref in references[:_MAX_REFERENCE_IMAGES]:
  680. part = _to_image_url_part(ref)
  681. if part:
  682. content.append({"type": "image_url", "image_url": {"url": part}})
  683. headers = {"Authorization": f"Bearer {api_key}", **_ATTRIBUTION_HEADERS}
  684. last_error: Optional[dict[str, Any]] = None
  685. for i, model_id in enumerate(model_chain):
  686. # Image API and chat models are not interchangeable: this is routing, not preference.
  687. surface = "chat"
  688. if self._supports_image_api:
  689. surface = _select_surface(model_id, base_url, api_key, self._config_key)
  690. if surface == "images":
  691. outcome = self._generate_via_image_api(
  692. model_id=model_id, prompt=prompt, semantic_aspect=aspect,
  693. references=references, base_url=base_url, headers=headers, kwargs=kwargs)
  694. reason = "failed on the image API" if outcome.pop("_retryable", False) else None
  695. else:
  696. outcome, reason = self._generate_via_chat(
  697. model_id=model_id, prompt=prompt, aspect=aspect, content=content,
  698. base_url=base_url, headers=headers)
  699. if outcome.get("success") or reason is None or i == len(model_chain) - 1:
  700. return outcome
  701. logger.info(
  702. "%s model %s %s; retrying with fallback %s", self._name, model_id, reason, model_chain[i + 1],
  703. )
  704. last_error = outcome
  705. return last_error or error_response(
  706. error=f"{self._display} image generation failed after trying all candidate models.",
  707. error_type="api_error", provider=self._name,
  708. model=model_chain[-1] if model_chain else "", prompt=prompt, aspect_ratio=aspect)
  709. def _build_providers() -> list[OpenRouterCompatImageProvider]:
  710. return [
  711. OpenRouterCompatImageProvider(
  712. provider_name="openrouter", display_name="OpenRouter", runtime_name="openrouter",
  713. config_key="openrouter", model_env_var="OPENROUTER_IMAGE_MODEL", supports_image_api=True,
  714. setup_schema={
  715. "name": "OpenRouter (image)",
  716. "badge": "paid",
  717. "tag": "Gemini Flash Image, gpt-image-2, Krea 2, Qwen Image 3 & more via OpenRouter; uses OPENROUTER_API_KEY",
  718. "env_vars": [{
  719. "key": "OPENROUTER_API_KEY", "prompt": "OpenRouter API key", "url": "https://openrouter.ai/keys",
  720. }],
  721. }),
  722. # No picker row: Portal models are offered inside the single managed "Nous Subscription" row
  723. # (tools/image_generation_managed.py). A row of its own also wrote `provider: nous` and so
  724. # read "active" alongside the managed FAL row while the runtime routed its pick to FAL.
  725. OpenRouterCompatImageProvider(
  726. provider_name="nous", display_name="Nous Portal", runtime_name="nous", config_key="nous",
  727. model_env_var="NOUS_IMAGE_MODEL", setup_schema=None),
  728. ]
  729. def register(ctx: Any) -> None:
  730. """Register the OpenRouter + Nous Portal image gen providers."""
  731. for provider in _build_providers():
  732. ctx.register_image_gen_provider(provider)