__init__.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319
  1. """OpenAI image generation — ChatGPT/Codex OAuth variant.
  2. Same catalog/tiers as the ``openai`` plugin (``gpt-image-2`` low/medium/high), posted to the
  3. Codex backend's native ``images/generations`` and ``images/edits`` endpoints, the same route the
  4. official Codex client uses (``codex-rs/ext/image-generation``). No ``OPENAI_API_KEY`` is needed.
  5. There is deliberately NO chat/host model here. An earlier version rode a Responses call with a
  6. hosted ``image_generation`` tool on a pinned chat model (``gpt-5.5``): when OpenAI withdrew that id
  7. from an account cohort every image call 404'd while chat kept working (#105398, #107076), and the
  8. host model was free to answer in text instead of calling the tool. The native route has neither
  9. failure mode.
  10. The backend does not enforce ``model``/``quality``/``size`` — it accepts unknown model ids and
  11. returns its own quality/size (#107233). We send the catalog values and report what came back
  12. (``reported_quality``/``reported_size``) so a request that was not honoured is diagnosable.
  13. """
  14. from __future__ import annotations
  15. import base64
  16. import json
  17. import logging
  18. import os
  19. import uuid
  20. from pathlib import Path
  21. from typing import Any, Dict, List, Optional, Tuple
  22. from agent.image_gen_provider import DEFAULT_ASPECT_RATIO, resolve_aspect_ratio, save_b64_image, success_response
  23. from plugins.image_gen._common import (
  24. GPT_IMAGE_2_API_MODEL as API_MODEL, GPT_IMAGE_2_DEFAULT as DEFAULT_MODEL, GPT_IMAGE_2_TIERS,
  25. StaticImageGenProvider, collect_source_images, error_factory, prompt_required_error,
  26. resolve_static_model, size_for)
  27. logger = logging.getLogger(__name__)
  28. _MAX_ERROR_BODY_CHARS = 500
  29. _MAX_REFERENCE_IMAGES = 16
  30. _MAX_INPUT_IMAGE_BYTES = 25 * 1024 * 1024
  31. # The edit endpoint accepts raster only; the shared sniffer also knows SVG/TIFF/ICO, which it rejects.
  32. _ACCEPTED_INPUT_MIME = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"})
  33. _NO_AUTH = (
  34. "No Codex/ChatGPT OAuth credentials available. Run "
  35. "`hermes auth add openai-codex` (or `hermes setup` → Codex) to sign in.")
  36. def _summarize_error_body(body: str) -> str:
  37. """Bounded summary preferring parsed ``error.message`` (Codex bodies carry leading metadata)."""
  38. text = body or ""
  39. try:
  40. payload = json.loads(text)
  41. error = payload.get("error") if isinstance(payload, dict) else None
  42. message = error.get("message") if isinstance(error, dict) else None
  43. if isinstance(message, str) and message.strip():
  44. return message.strip()[:_MAX_ERROR_BODY_CHARS]
  45. except (TypeError, ValueError):
  46. pass
  47. return text[:_MAX_ERROR_BODY_CHARS]
  48. def _resolve_model() -> tuple[str, dict[str, Any]]:
  49. return resolve_static_model(
  50. GPT_IMAGE_2_TIERS, DEFAULT_MODEL, env_var="OPENAI_IMAGE_MODEL", config_key="openai-codex")
  51. def _read_codex_credential() -> tuple[Optional[str], Optional[str]]:
  52. """``(token, base_url)`` from one resolution (``agent.auxiliary_client`` owns expiry/pool/JWT):
  53. the image request goes to the host the token's credential routes to (pool row /
  54. ``model.base_url`` / profile override), never a default it does not belong to (#121486).
  55. ``(None, None)`` without a usable token."""
  56. try:
  57. from agent.auxiliary_client import _resolve_codex_credential_and_base
  58. token, base_url = _resolve_codex_credential_and_base()
  59. if isinstance(token, str) and token.strip():
  60. return token.strip(), base_url
  61. return None, None
  62. except Exception as exc:
  63. logger.debug("Could not resolve Codex credential: %s", exc)
  64. return None, None
  65. def _httpx_available() -> bool:
  66. try:
  67. import httpx
  68. except ImportError:
  69. return False
  70. return True
  71. def _sniff_image_mime(raw: bytes) -> Optional[str]:
  72. from agent.image_routing import _sniff_mime_from_bytes
  73. mime = _sniff_mime_from_bytes(raw)
  74. return mime if mime in _ACCEPTED_INPUT_MIME else None
  75. def _encode_input_image(raw: bytes, too_big: str, unsupported: str) -> str:
  76. """Size- and MIME-check raw image bytes, then return a canonical ``data:`` URL."""
  77. if len(raw) > _MAX_INPUT_IMAGE_BYTES:
  78. raise ValueError(too_big)
  79. mime = _sniff_image_mime(raw)
  80. if mime is None:
  81. raise ValueError(unsupported)
  82. return f"data:{mime};base64,{base64.b64encode(raw).decode('ascii')}"
  83. def _data_url_to_input_image_url(value: str) -> str:
  84. if "," not in value:
  85. raise ValueError("Image data URL is missing a comma separator")
  86. header, data = value.split(",", 1)
  87. header_lc = header.lower()
  88. if not header_lc.startswith("data:image/") or ";base64" not in header_lc:
  89. raise ValueError("Only base64 data:image URLs are supported as Codex image inputs")
  90. return _encode_input_image(
  91. base64.b64decode(data, validate=True),
  92. "Image data URL exceeds 25MB cap",
  93. "Image data URL does not contain supported image bytes")
  94. def _remote_image_to_data_url(value: str) -> str:
  95. """The edit endpoint takes inline data URLs only (as the official client sends), so fetch."""
  96. from tools.url_safety import create_ssrf_safe_client, is_safe_url
  97. if not is_safe_url(value):
  98. raise ValueError(f"Image URL failed the SSRF safety check: {value}")
  99. with create_ssrf_safe_client(timeout=60.0, follow_redirects=True) as client:
  100. response = client.get(value)
  101. response.raise_for_status()
  102. return _encode_input_image(
  103. response.content,
  104. f"Image URL exceeds 25MB cap: {value}",
  105. f"Image URL did not return a supported image: {value}")
  106. def _local_image_to_data_url(value: str) -> str:
  107. from agent.file_safety import get_read_block_error
  108. blocked = get_read_block_error(value)
  109. if blocked:
  110. raise ValueError(blocked)
  111. path = Path(os.path.expanduser(value)).resolve()
  112. if not path.is_file():
  113. raise ValueError(f"Image input path does not exist or is not a file: {value}")
  114. if path.stat().st_size <= 0:
  115. raise ValueError(f"Image input path is empty: {value}")
  116. return _encode_input_image(
  117. path.read_bytes(),
  118. f"Image input path exceeds 25MB cap: {value}",
  119. f"Image input path is not a supported image: {value}")
  120. def _to_input_image(value: str) -> dict[str, str]:
  121. """Convert a URL/data URL/local path into an ``images[]`` entry for ``images/edits``."""
  122. candidate = (value or "").strip()
  123. if not candidate:
  124. raise ValueError("Blank image input")
  125. lowered = candidate.lower()
  126. if lowered.startswith(("http://", "https://")):
  127. image_url = _remote_image_to_data_url(candidate)
  128. elif lowered.startswith("data:"):
  129. image_url = _data_url_to_input_image_url(candidate)
  130. else:
  131. image_url = _local_image_to_data_url(candidate)
  132. return {"image_url": image_url}
  133. def _normalize_input_images(
  134. image_url: Optional[str], reference_image_urls: Optional[list[str]]
  135. ) -> list[dict[str, str]]:
  136. values = collect_source_images(image_url, reference_image_urls, limit=_MAX_REFERENCE_IMAGES)
  137. return [_to_input_image(value) for value in values]
  138. def _build_image_request(
  139. *, prompt: str, size: str, quality: str, input_images: Optional[list[dict[str, str]]] = None
  140. ) -> tuple[str, dict[str, Any]]:
  141. """``(endpoint_path, json_body)`` — ``images/edits`` when sources are present, else
  142. ``images/generations``. Field set mirrors the official client's ``ImageGenerationRequest`` /
  143. ``ImageEditRequest``."""
  144. body: dict[str, Any] = {
  145. "prompt": prompt, "model": API_MODEL, "n": 1, "quality": quality, "size": size,
  146. "background": "opaque",
  147. }
  148. if input_images:
  149. body["images"] = input_images
  150. return "images/edits", body
  151. return "images/generations", body
  152. def _post_image_request(
  153. token: str, *, prompt: str, size: str, quality: str, input_images: Optional[list[dict[str, str]]] = None,
  154. base_url: str,
  155. ) -> dict[str, Any]:
  156. """POST to the native Codex images endpoint; return the decoded JSON body plus
  157. ``imagegen_request_id`` (backend correlation id, for support tickets).
  158. ``base_url`` must come from the same resolution as ``token`` (``_read_codex_credential``)."""
  159. import httpx
  160. from agent.codex_headers import codex_cloudflare_headers
  161. base_url = base_url.strip().rstrip("/")
  162. headers = codex_cloudflare_headers(token, base_url=base_url)
  163. headers.update({
  164. "Authorization": f"Bearer {token}",
  165. "Content-Type": "application/json",
  166. "x-codex-image-turn-id": str(uuid.uuid4()),
  167. })
  168. path, body = _build_image_request(prompt=prompt, size=size, quality=quality, input_images=input_images)
  169. timeout = httpx.Timeout(300.0, connect=30.0, read=300.0, write=60.0, pool=30.0)
  170. with httpx.Client(timeout=timeout, headers=headers) as http:
  171. response = http.post(f"{base_url}/{path}", json=body)
  172. if response.status_code >= 400:
  173. raise RuntimeError(
  174. f"Codex images API returned HTTP {response.status_code}: "
  175. f"{_summarize_error_body(response.text)}")
  176. payload = response.json()
  177. if not isinstance(payload, dict):
  178. raise RuntimeError("Codex images API returned a non-object body")
  179. payload["imagegen_request_id"] = response.headers.get("x-codex-imagegen-request-id")
  180. return payload
  181. def _png_pixel_size(raw: bytes) -> Optional[str]:
  182. """``"{w}x{h}"`` for a PNG payload, or None if not a PNG IHDR."""
  183. import struct
  184. if len(raw) < 24 or raw[:8] != b"\x89PNG\r\n\x1a\n" or raw[12:16] != b"IHDR":
  185. return None
  186. width, height = struct.unpack(">II", raw[16:24])
  187. return f"{width}x{height}"
  188. class OpenAICodexImageGenProvider(StaticImageGenProvider):
  189. """gpt-image-2 routed through ChatGPT/Codex OAuth instead of an API key."""
  190. provider_id = "openai-codex"
  191. label = "OpenAI (Codex auth)"
  192. models = GPT_IMAGE_2_TIERS
  193. default_model_id = DEFAULT_MODEL
  194. price = "varies"
  195. def is_available(self) -> bool:
  196. return bool(_read_codex_credential()[0]) and _httpx_available()
  197. def get_setup_schema(self) -> dict[str, Any]:
  198. return {
  199. "name": "OpenAI (Codex auth)",
  200. "badge": "free",
  201. "tag": "gpt-image-2 via ChatGPT/Codex OAuth — no API key required; supports text and image inputs",
  202. "env_vars": [],
  203. # Empty env_vars means the picker writes the selection without a credential prompt; the shared
  204. # Codex OAuth bootstrap hook (hermes_cli/tools_config_post_setup.py) starts the sign-in (#102144).
  205. "post_setup": "openai_codex",
  206. "post_setup_hint": (
  207. "Sign in with `hermes auth add openai-codex` (or `hermes setup` → Codex) "
  208. "if you haven't already. No API key needed."),
  209. }
  210. def capabilities(self) -> dict[str, Any]:
  211. return {"modalities": ["text", "image"], "max_reference_images": _MAX_REFERENCE_IMAGES}
  212. def generate(
  213. self, prompt: str, aspect_ratio: str = DEFAULT_ASPECT_RATIO, *,
  214. image_url: Optional[str] = None, reference_image_urls: Optional[list[str]] = None,
  215. **kwargs: Any,
  216. ) -> dict[str, Any]:
  217. prompt = (prompt or "").strip()
  218. aspect = resolve_aspect_ratio(aspect_ratio)
  219. if not prompt:
  220. return prompt_required_error("openai-codex", aspect)
  221. token, base_url = _read_codex_credential()
  222. if not token:
  223. return error_factory("openai-codex", aspect)(_NO_AUTH, "auth_required")
  224. if not _httpx_available():
  225. return error_factory("openai-codex", aspect)(
  226. "httpx Python package not installed (pip install httpx)", "missing_dependency")
  227. tier_id, meta = _resolve_model()
  228. size = size_for(aspect)
  229. fail = error_factory("openai-codex", aspect, model=tier_id, prompt=prompt)
  230. try:
  231. input_images = _normalize_input_images(image_url, reference_image_urls)
  232. except Exception as exc:
  233. return fail(f"Invalid image input for Codex image editing: {exc}", "invalid_image_input")
  234. try:
  235. payload = _post_image_request(
  236. token, prompt=prompt, size=size, quality=meta["quality"], input_images=input_images or None,
  237. base_url=base_url)
  238. except Exception as exc:
  239. logger.debug("Codex image generation failed", exc_info=True)
  240. return fail(f"OpenAI image generation via Codex auth failed: {exc}", "api_error")
  241. data = payload.get("data")
  242. b64 = data[0].get("b64_json") if isinstance(data, list) and data and isinstance(data[0], dict) else None
  243. if not isinstance(b64, str) or not b64:
  244. return fail("Codex images API response contained no image data", "empty_response")
  245. try:
  246. pixel_size = _png_pixel_size(base64.b64decode(b64))
  247. saved_path = save_b64_image(b64, prefix=f"openai_codex_{tier_id}")
  248. except Exception as exc:
  249. return fail(f"Could not save image to cache: {exc}", "io_error")
  250. return success_response(
  251. image=str(saved_path), model=tier_id, prompt=prompt, aspect_ratio=aspect,
  252. provider="openai-codex", modality="image" if input_images else "text",
  253. extra={
  254. "size": size, "quality": meta["quality"], "input_image_count": len(input_images),
  255. "requested_size": size, "pixel_size": pixel_size,
  256. "reported_quality": payload.get("quality"), "reported_size": payload.get("size"),
  257. "imagegen_request_id": payload.get("imagegen_request_id"),
  258. })
  259. def register(ctx) -> None:
  260. """Plugin entry point — register the Codex-backed image-gen provider."""
  261. ctx.register_image_gen_provider(OpenAICodexImageGenProvider())