__init__.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355
  1. """xAI ``grok-imagine-image`` backend: text-to-image (``/v1/images/generations``) and editing
  2. (``/v1/images/edits``), base64 output saved to cache. Selection: ``model`` kwarg →
  3. ``XAI_IMAGE_MODEL`` → ``image_gen.xai.model`` → :data:`DEFAULT_MODEL`."""
  4. from __future__ import annotations
  5. import logging
  6. import os
  7. import time
  8. from pathlib import Path
  9. from typing import Any, Dict, List, Optional, Tuple
  10. import requests
  11. from agent.image_gen_provider import DEFAULT_ASPECT_RATIO, resolve_aspect_ratio, success_response
  12. from plugins.image_gen._common import (
  13. StaticImageGenProvider, catalog_rows, collect_source_images, error_factory,
  14. load_image_gen_config, materialize_image, post_json)
  15. from tools.xai_http import (
  16. build_xai_storage_options, hermes_xai_user_agent, maybe_mark_xai_storage_notice_seen,
  17. read_xai_imagine_storage_config, resolve_xai_http_credentials, xai_storage_notice_text)
  18. logger = logging.getLogger(__name__)
  19. _MODELS: dict[str, dict[str, Any]] = {
  20. "grok-imagine-image": {
  21. "display": "Grok Imagine Image", "speed": "~5-10s", "strengths": "Fast, high-quality",
  22. },
  23. "grok-imagine-image-2.0": {
  24. "display": "Grok Imagine Image 2.0", "speed": "~10-20s",
  25. "strengths": "Typography/layout-aware; legible small text; strongest quality.",
  26. },
  27. "grok-imagine-image-quality": {
  28. "display": "Grok Imagine Image (Quality)", "speed": "~10-20s",
  29. "strengths": "Higher fidelity / detail; slower than the standard model.",
  30. },
  31. }
  32. DEFAULT_MODEL = "grok-imagine-image"
  33. # xAI documents the quality model as the edit-capable baseline.
  34. _EDIT_FALLBACK_MODEL = "grok-imagine-image-quality"
  35. # Live catalog cache ``(models, fetched_monotonic)``: ``/image-generation-models`` is the source of
  36. # truth (new models need no code change); ``_MODELS`` is the offline fallback + curated text.
  37. _LIVE_CACHE: Optional[tuple[dict[str, dict[str, Any]], float]] = None
  38. # Under a multiplexed profile override the catalog is keyed by (base_url, key fingerprint): the
  39. # endpoint is credential-scoped, so one slot would hand profile A's models (or its cached auth
  40. # failure) to profile B. The unscoped slot above stays for the single-profile path and its tests.
  41. _LIVE_CACHE_BY_CREDENTIAL: dict[tuple[str, Optional[str]], tuple[dict[str, dict[str, Any]], float]] = {}
  42. _LIVE_CACHE_TTL = 300.0
  43. _LIVE_TIMEOUT = 10.0
  44. _XAI_ASPECT_RATIOS = {
  45. "landscape": "16:9", "square": "1:1", "portrait": "9:16",
  46. "4:3": "4:3", "3:4": "3:4", "3:2": "3:2", "2:3": "2:3",
  47. }
  48. _XAI_RESOLUTIONS = {"1k", "2k"}
  49. DEFAULT_RESOLUTION = "1k"
  50. _MAX_SOURCE_IMAGES = 3
  51. _REQUEST_TIMEOUT = 120
  52. _REMOTE_PREFIXES = ("http://", "https://", "data:")
  53. _FILE_OUTPUT_EXTRA_KEYS = (
  54. "filename", "expires_at", "public_url_expires_at", "public_url_error", "storage_error")
  55. def _base_url(creds: dict[str, Any]) -> str:
  56. return str(creds.get("base_url") or "https://api.x.ai/v1").strip().rstrip("/")
  57. def _fetch_live_models(creds: Optional[dict[str, Any]] = None) -> dict[str, dict[str, Any]]:
  58. """``{model_id: {"input_modalities", "aliases"}}`` from the live endpoint; raises on failure."""
  59. if creds is None:
  60. creds = resolve_xai_http_credentials()
  61. api_key = str(creds.get("api_key") or "").strip()
  62. if not api_key:
  63. raise RuntimeError("no xAI credentials")
  64. response = requests.get(
  65. f"{_base_url(creds)}/image-generation-models",
  66. headers={"Authorization": f"Bearer {api_key}", "User-Agent": hermes_xai_user_agent()},
  67. timeout=_LIVE_TIMEOUT)
  68. response.raise_for_status()
  69. payload = response.json()
  70. out: dict[str, dict[str, Any]] = {}
  71. for entry in payload.get("models") or payload.get("data") or []:
  72. model_id = entry.get("id") or entry.get("name") if isinstance(entry, dict) else None
  73. if isinstance(model_id, str) and model_id.strip():
  74. out[model_id.strip()] = {
  75. "input_modalities": entry.get("input_modalities") or [], "aliases": entry.get("aliases") or [],
  76. }
  77. return out
  78. def _live_models() -> dict[str, dict[str, Any]]:
  79. """Cached live catalog (``{}`` when unreachable)."""
  80. global _LIVE_CACHE
  81. from hermes_constants import get_hermes_home_override
  82. if get_hermes_home_override() is None:
  83. if _LIVE_CACHE is not None and time.monotonic() - _LIVE_CACHE[1] < _LIVE_CACHE_TTL:
  84. return _LIVE_CACHE[0]
  85. _LIVE_CACHE = (_fetch_live_models_or_empty(None), time.monotonic())
  86. return _LIVE_CACHE[0]
  87. from agent.credential_persistence import fingerprint_secret_value
  88. try:
  89. creds = resolve_xai_http_credentials()
  90. except Exception as exc:
  91. logger.debug("xAI live image model catalog unavailable: %s", exc)
  92. creds = {}
  93. key = (_base_url(creds), fingerprint_secret_value(creds.get("api_key")))
  94. cached = _LIVE_CACHE_BY_CREDENTIAL.get(key)
  95. if cached is not None and time.monotonic() - cached[1] < _LIVE_CACHE_TTL:
  96. return cached[0]
  97. live = _fetch_live_models_or_empty(creds)
  98. _LIVE_CACHE_BY_CREDENTIAL[key] = (live, time.monotonic())
  99. return live
  100. def _fetch_live_models_or_empty(creds: Optional[dict[str, Any]]) -> dict[str, dict[str, Any]]:
  101. try:
  102. return _fetch_live_models() if creds is None else _fetch_live_models(creds)
  103. except Exception as exc:
  104. logger.debug("xAI live image model catalog unavailable: %s", exc)
  105. return {}
  106. def _catalog() -> dict[str, dict[str, Any]]:
  107. """Live ids + curated metadata (unknown live models get generic text; curated entries the live
  108. list omits are kept); the static table alone when the API is unreachable."""
  109. live = _live_models()
  110. if not live:
  111. return dict(_MODELS)
  112. merged: dict[str, dict[str, Any]] = {}
  113. for model_id in live:
  114. meta = _MODELS.get(model_id) or {
  115. "display": model_id, "speed": "", "strengths": "New xAI Imagine model (from live xAI catalog)",
  116. }
  117. merged[model_id] = {**meta, "input_modalities": live[model_id].get("input_modalities") or []}
  118. for model_id, meta in _MODELS.items():
  119. merged.setdefault(model_id, dict(meta))
  120. return merged
  121. def _configured_model() -> Optional[str]:
  122. value = load_image_gen_config("xai").get("model")
  123. return value if isinstance(value, str) else None
  124. def _resolve_model(caller_model: Optional[str] = None) -> tuple[str, dict[str, Any]]:
  125. """caller kwarg → ``XAI_IMAGE_MODEL`` → config → default, validated against the merged catalog."""
  126. catalog = _catalog()
  127. if caller_model and caller_model not in catalog:
  128. raise ValueError(f"Selected model {caller_model!r} is not in the image model catalog")
  129. for candidate in (caller_model, os.environ.get("XAI_IMAGE_MODEL"), _configured_model()):
  130. if candidate and candidate in catalog:
  131. return candidate, catalog[candidate]
  132. return DEFAULT_MODEL, catalog.get(DEFAULT_MODEL, _MODELS[DEFAULT_MODEL])
  133. def _resolve_edit_model(caller_model: Optional[str] = None) -> str:
  134. """Edit model: an explicit selection that accepts image input, else the documented quality baseline."""
  135. catalog = _catalog()
  136. explicit = caller_model or os.environ.get("XAI_IMAGE_MODEL") or _configured_model()
  137. if explicit:
  138. if explicit in catalog and "image" in (catalog[explicit].get("input_modalities") or []):
  139. return explicit
  140. raise ValueError(f"Selected model {explicit!r} does not support image editing")
  141. return _EDIT_FALLBACK_MODEL
  142. def _resolve_resolution() -> str:
  143. res = load_image_gen_config("xai").get("resolution")
  144. return res if isinstance(res, str) and res in _XAI_RESOLUTIONS else DEFAULT_RESOLUTION
  145. def _xai_image_field(source: str) -> dict[str, str]:
  146. """Edit ``image`` field: URL / data URI pass through; local paths are inlined as ``data:`` URIs."""
  147. source = source.strip()
  148. if source.lower().startswith(_REMOTE_PREFIXES):
  149. return {"url": source, "type": "image_url"}
  150. import base64
  151. from agent.file_safety import raise_if_read_blocked # credential-read guard before local bytes
  152. raise_if_read_blocked(source)
  153. with open(os.path.expanduser(source), "rb") as fh: # windows-footgun: ok
  154. raw = fh.read()
  155. ext = (os.path.splitext(source)[1].lstrip(".") or "png").lower()
  156. if ext == "jpg":
  157. ext = "jpeg"
  158. return {"url": f"data:image/{ext};base64,{base64.b64encode(raw).decode('utf-8')}", "type": "image_url"}
  159. def _check_source_images(
  160. source_images: list[str], image_url: Optional[str], fail: Any
  161. ) -> Optional[dict[str, Any]]:
  162. """Edit-request guard: at most 3 sources, each a remote URL/data URI or an existing local file."""
  163. if len(source_images) > _MAX_SOURCE_IMAGES:
  164. return fail(
  165. f"xAI image editing supports at most {_MAX_SOURCE_IMAGES} source images", "too_many_references",
  166. )
  167. for index, source in enumerate(source_images):
  168. if source.lower().startswith(_REMOTE_PREFIXES) or Path(source).expanduser().is_file():
  169. continue
  170. is_primary = index == 0 and image_url and image_url.strip() == source
  171. field = "image_url" if is_primary else "reference_image_urls"
  172. return fail(
  173. f"{field} must be a public HTTPS URL or data URI "
  174. "(e.g. the `image`/`public_url` from a prior Imagine result)",
  175. "invalid_image_url")
  176. return None
  177. class XAIImageGenProvider(StaticImageGenProvider):
  178. """xAI ``grok-imagine-image`` backend."""
  179. provider_id = "xai"
  180. label = "xAI (Grok)"
  181. def is_available(self) -> bool:
  182. return bool(resolve_xai_http_credentials().get("api_key"))
  183. def list_models(self) -> list[dict[str, Any]]:
  184. return catalog_rows(_catalog(), ("display", "speed", "strengths"))
  185. def default_model(self) -> Optional[str]:
  186. # First live/static catalog row (inherited ImageGenProvider behaviour).
  187. return next(iter(_catalog()), None)
  188. def get_setup_schema(self) -> dict[str, Any]:
  189. # Auth goes through the shared ``xai_grok`` post_setup hook (same OAuth-or-key choice everywhere).
  190. storage_notice = xai_storage_notice_text("image_gen")
  191. tag = "grok-imagine-image - text-to-image & image editing; uses xAI Grok OAuth or XAI_API_KEY"
  192. if storage_notice:
  193. tag += f". {storage_notice}"
  194. return {
  195. "name": "xAI Grok Imagine (image)", "badge": "paid", "tag": tag, "env_vars": [],
  196. "post_setup": "xai_grok",
  197. }
  198. def capabilities(self) -> dict[str, Any]:
  199. # /v1/images/edits accepts up to 3 total source images.
  200. return {
  201. "modalities": ["text", "image"], "max_reference_images": 2,
  202. "max_source_images": _MAX_SOURCE_IMAGES,
  203. }
  204. def generate(
  205. self, prompt: str, aspect_ratio: str = DEFAULT_ASPECT_RATIO, *,
  206. image_url: Optional[str] = None, reference_image_urls: Optional[list[str]] = None,
  207. **kwargs: Any,
  208. ) -> dict[str, Any]:
  209. """Text-to-image, or editing via ``/v1/images/edits`` (JSON body — xAI does not support the
  210. SDK's multipart ``images.edit()``) when source images are supplied."""
  211. creds = resolve_xai_http_credentials()
  212. api_key = str(creds.get("api_key") or "").strip()
  213. provider_name = str(creds.get("provider") or "xai").strip() or "xai"
  214. if not api_key:
  215. return error_factory(provider_name, aspect_ratio)(
  216. "No xAI credentials found. Configure xAI OAuth in `hermes model` or set XAI_API_KEY.",
  217. "missing_api_key")
  218. try:
  219. model_id, _meta = _resolve_model(kwargs.get("model"))
  220. except ValueError as exc:
  221. return error_factory(provider_name, aspect_ratio)(str(exc), "unsupported_model", model=kwargs.get("model"))
  222. aspect = resolve_aspect_ratio(aspect_ratio)
  223. xai_res = _resolve_resolution()
  224. source_images = collect_source_images(image_url, reference_image_urls)
  225. edit_fail = error_factory(provider_name, aspect, model=_EDIT_FALLBACK_MODEL, prompt=prompt)
  226. err = _check_source_images(source_images, image_url, edit_fail)
  227. if err:
  228. return err
  229. is_edit = bool(source_images)
  230. headers = {
  231. "Authorization": f"Bearer {api_key}", "Content-Type": "application/json",
  232. "User-Agent": hermes_xai_user_agent(),
  233. }
  234. base_url = _base_url(creds)
  235. storage_options = build_xai_storage_options(
  236. "image_gen", filename_prefix="hermes-xai-image", extension="png")
  237. storage_notice = maybe_mark_xai_storage_notice_seen("image_gen")
  238. storage_cfg = read_xai_imagine_storage_config("image_gen")
  239. if is_edit:
  240. try:
  241. model_id = _resolve_edit_model(kwargs.get("model"))
  242. except ValueError as exc:
  243. return edit_fail(str(exc), "unsupported_model", model=kwargs.get("model"))
  244. try:
  245. image_fields = [_xai_image_field(source) for source in source_images]
  246. except Exception as exc:
  247. return edit_fail(f"Could not load source image for editing: {exc}", "io_error", model=model_id)
  248. payload: dict[str, Any] = {"model": model_id, "prompt": prompt}
  249. if len(image_fields) == 1:
  250. payload["image"] = image_fields[0]
  251. else:
  252. payload["images"] = image_fields
  253. endpoint_url = f"{base_url}/images/edits"
  254. else:
  255. payload = {
  256. "model": model_id, "prompt": prompt, "aspect_ratio": _XAI_ASPECT_RATIOS.get(aspect, "1:1"),
  257. "resolution": xai_res,
  258. }
  259. endpoint_url = f"{base_url}/images/generations"
  260. if storage_options is not None:
  261. payload["storage_options"] = storage_options
  262. fail = error_factory(provider_name, aspect, model=model_id, prompt=prompt)
  263. result, failure = post_json(
  264. endpoint_url, headers=headers, payload=payload, timeout=_REQUEST_TIMEOUT, label="xAI")
  265. if failure:
  266. if failure.kind == "http":
  267. logger.error("xAI image gen failed (%d): %s", failure.status, failure.message)
  268. return fail(failure.error, failure.error_type)
  269. # data[0] carries b64_json / url, plus file_output when storage_options were requested.
  270. data = result.get("data", [])
  271. if not data:
  272. return fail("xAI returned no image data", "empty_response")
  273. first = data[0]
  274. file_output = first.get("file_output") if isinstance(first, dict) else None
  275. file_output = file_output if isinstance(file_output, dict) else {}
  276. public_url = file_output.get("public_url")
  277. public_url = public_url if isinstance(public_url, str) else None
  278. if public_url:
  279. image_ref = public_url
  280. else:
  281. # ``imgen.x.ai/xai-tmp-*`` URLs 404 within minutes; materialise locally for a stable path.
  282. image_ref, err = materialize_image(
  283. first.get("b64_json"), first.get("url"), prefix=f"xai_{model_id}", label="xAI", provider="xai",
  284. model=model_id, prompt=prompt, aspect=aspect, log=logger)
  285. if err:
  286. return err
  287. extra: dict[str, Any] = {"storage_enabled": bool(storage_cfg["enabled"])}
  288. if not is_edit:
  289. extra["resolution"] = xai_res
  290. if storage_notice:
  291. extra["storage_notice"] = storage_notice
  292. if public_url:
  293. extra["public_url"] = public_url
  294. extra.update({key: file_output[key] for key in _FILE_OUTPUT_EXTRA_KEYS if key in file_output})
  295. if result.get("usage"):
  296. extra["usage"] = result["usage"]
  297. return success_response(
  298. image=image_ref, model=model_id, prompt=prompt, aspect_ratio=aspect, provider="xai",
  299. modality="image" if is_edit else "text", extra=extra)
  300. def register(ctx: Any) -> None:
  301. """Register this provider with the image gen registry."""
  302. ctx.register_image_gen_provider(XAIImageGenProvider())