| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319 |
- """OpenAI image generation — ChatGPT/Codex OAuth variant.
- Same catalog/tiers as the ``openai`` plugin (``gpt-image-2`` low/medium/high), posted to the
- Codex backend's native ``images/generations`` and ``images/edits`` endpoints, the same route the
- official Codex client uses (``codex-rs/ext/image-generation``). No ``OPENAI_API_KEY`` is needed.
- There is deliberately NO chat/host model here. An earlier version rode a Responses call with a
- hosted ``image_generation`` tool on a pinned chat model (``gpt-5.5``): when OpenAI withdrew that id
- from an account cohort every image call 404'd while chat kept working (#105398, #107076), and the
- host model was free to answer in text instead of calling the tool. The native route has neither
- failure mode.
- The backend does not enforce ``model``/``quality``/``size`` — it accepts unknown model ids and
- returns its own quality/size (#107233). We send the catalog values and report what came back
- (``reported_quality``/``reported_size``) so a request that was not honoured is diagnosable.
- """
- from __future__ import annotations
- import base64
- import json
- import logging
- import os
- import uuid
- from pathlib import Path
- from typing import Any, Dict, List, Optional, Tuple
- from agent.image_gen_provider import DEFAULT_ASPECT_RATIO, resolve_aspect_ratio, save_b64_image, success_response
- from plugins.image_gen._common import (
- GPT_IMAGE_2_API_MODEL as API_MODEL, GPT_IMAGE_2_DEFAULT as DEFAULT_MODEL, GPT_IMAGE_2_TIERS,
- StaticImageGenProvider, collect_source_images, error_factory, prompt_required_error,
- resolve_static_model, size_for)
- logger = logging.getLogger(__name__)
- _MAX_ERROR_BODY_CHARS = 500
- _MAX_REFERENCE_IMAGES = 16
- _MAX_INPUT_IMAGE_BYTES = 25 * 1024 * 1024
- # The edit endpoint accepts raster only; the shared sniffer also knows SVG/TIFF/ICO, which it rejects.
- _ACCEPTED_INPUT_MIME = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"})
- _NO_AUTH = (
- "No Codex/ChatGPT OAuth credentials available. Run "
- "`hermes auth add openai-codex` (or `hermes setup` → Codex) to sign in.")
- def _summarize_error_body(body: str) -> str:
- """Bounded summary preferring parsed ``error.message`` (Codex bodies carry leading metadata)."""
- text = body or ""
- try:
- payload = json.loads(text)
- error = payload.get("error") if isinstance(payload, dict) else None
- message = error.get("message") if isinstance(error, dict) else None
- if isinstance(message, str) and message.strip():
- return message.strip()[:_MAX_ERROR_BODY_CHARS]
- except (TypeError, ValueError):
- pass
- return text[:_MAX_ERROR_BODY_CHARS]
- def _resolve_model() -> tuple[str, dict[str, Any]]:
- return resolve_static_model(
- GPT_IMAGE_2_TIERS, DEFAULT_MODEL, env_var="OPENAI_IMAGE_MODEL", config_key="openai-codex")
- def _read_codex_credential() -> tuple[Optional[str], Optional[str]]:
- """``(token, base_url)`` from one resolution (``agent.auxiliary_client`` owns expiry/pool/JWT):
- the image request goes to the host the token's credential routes to (pool row /
- ``model.base_url`` / profile override), never a default it does not belong to (#121486).
- ``(None, None)`` without a usable token."""
- try:
- from agent.auxiliary_client import _resolve_codex_credential_and_base
- token, base_url = _resolve_codex_credential_and_base()
- if isinstance(token, str) and token.strip():
- return token.strip(), base_url
- return None, None
- except Exception as exc:
- logger.debug("Could not resolve Codex credential: %s", exc)
- return None, None
- def _httpx_available() -> bool:
- try:
- import httpx
- except ImportError:
- return False
- return True
- def _sniff_image_mime(raw: bytes) -> Optional[str]:
- from agent.image_routing import _sniff_mime_from_bytes
- mime = _sniff_mime_from_bytes(raw)
- return mime if mime in _ACCEPTED_INPUT_MIME else None
- def _encode_input_image(raw: bytes, too_big: str, unsupported: str) -> str:
- """Size- and MIME-check raw image bytes, then return a canonical ``data:`` URL."""
- if len(raw) > _MAX_INPUT_IMAGE_BYTES:
- raise ValueError(too_big)
- mime = _sniff_image_mime(raw)
- if mime is None:
- raise ValueError(unsupported)
- return f"data:{mime};base64,{base64.b64encode(raw).decode('ascii')}"
- def _data_url_to_input_image_url(value: str) -> str:
- if "," not in value:
- raise ValueError("Image data URL is missing a comma separator")
- header, data = value.split(",", 1)
- header_lc = header.lower()
- if not header_lc.startswith("data:image/") or ";base64" not in header_lc:
- raise ValueError("Only base64 data:image URLs are supported as Codex image inputs")
- return _encode_input_image(
- base64.b64decode(data, validate=True),
- "Image data URL exceeds 25MB cap",
- "Image data URL does not contain supported image bytes")
- def _remote_image_to_data_url(value: str) -> str:
- """The edit endpoint takes inline data URLs only (as the official client sends), so fetch."""
- from tools.url_safety import create_ssrf_safe_client, is_safe_url
- if not is_safe_url(value):
- raise ValueError(f"Image URL failed the SSRF safety check: {value}")
- with create_ssrf_safe_client(timeout=60.0, follow_redirects=True) as client:
- response = client.get(value)
- response.raise_for_status()
- return _encode_input_image(
- response.content,
- f"Image URL exceeds 25MB cap: {value}",
- f"Image URL did not return a supported image: {value}")
- def _local_image_to_data_url(value: str) -> str:
- from agent.file_safety import get_read_block_error
- blocked = get_read_block_error(value)
- if blocked:
- raise ValueError(blocked)
- path = Path(os.path.expanduser(value)).resolve()
- if not path.is_file():
- raise ValueError(f"Image input path does not exist or is not a file: {value}")
- if path.stat().st_size <= 0:
- raise ValueError(f"Image input path is empty: {value}")
- return _encode_input_image(
- path.read_bytes(),
- f"Image input path exceeds 25MB cap: {value}",
- f"Image input path is not a supported image: {value}")
- def _to_input_image(value: str) -> dict[str, str]:
- """Convert a URL/data URL/local path into an ``images[]`` entry for ``images/edits``."""
- candidate = (value or "").strip()
- if not candidate:
- raise ValueError("Blank image input")
- lowered = candidate.lower()
- if lowered.startswith(("http://", "https://")):
- image_url = _remote_image_to_data_url(candidate)
- elif lowered.startswith("data:"):
- image_url = _data_url_to_input_image_url(candidate)
- else:
- image_url = _local_image_to_data_url(candidate)
- return {"image_url": image_url}
- def _normalize_input_images(
- image_url: Optional[str], reference_image_urls: Optional[list[str]]
- ) -> list[dict[str, str]]:
- values = collect_source_images(image_url, reference_image_urls, limit=_MAX_REFERENCE_IMAGES)
- return [_to_input_image(value) for value in values]
- def _build_image_request(
- *, prompt: str, size: str, quality: str, input_images: Optional[list[dict[str, str]]] = None
- ) -> tuple[str, dict[str, Any]]:
- """``(endpoint_path, json_body)`` — ``images/edits`` when sources are present, else
- ``images/generations``. Field set mirrors the official client's ``ImageGenerationRequest`` /
- ``ImageEditRequest``."""
- body: dict[str, Any] = {
- "prompt": prompt, "model": API_MODEL, "n": 1, "quality": quality, "size": size,
- "background": "opaque",
- }
- if input_images:
- body["images"] = input_images
- return "images/edits", body
- return "images/generations", body
- def _post_image_request(
- token: str, *, prompt: str, size: str, quality: str, input_images: Optional[list[dict[str, str]]] = None,
- base_url: str,
- ) -> dict[str, Any]:
- """POST to the native Codex images endpoint; return the decoded JSON body plus
- ``imagegen_request_id`` (backend correlation id, for support tickets).
- ``base_url`` must come from the same resolution as ``token`` (``_read_codex_credential``)."""
- import httpx
- from agent.codex_headers import codex_cloudflare_headers
- base_url = base_url.strip().rstrip("/")
- headers = codex_cloudflare_headers(token, base_url=base_url)
- headers.update({
- "Authorization": f"Bearer {token}",
- "Content-Type": "application/json",
- "x-codex-image-turn-id": str(uuid.uuid4()),
- })
- path, body = _build_image_request(prompt=prompt, size=size, quality=quality, input_images=input_images)
- timeout = httpx.Timeout(300.0, connect=30.0, read=300.0, write=60.0, pool=30.0)
- with httpx.Client(timeout=timeout, headers=headers) as http:
- response = http.post(f"{base_url}/{path}", json=body)
- if response.status_code >= 400:
- raise RuntimeError(
- f"Codex images API returned HTTP {response.status_code}: "
- f"{_summarize_error_body(response.text)}")
- payload = response.json()
- if not isinstance(payload, dict):
- raise RuntimeError("Codex images API returned a non-object body")
- payload["imagegen_request_id"] = response.headers.get("x-codex-imagegen-request-id")
- return payload
- def _png_pixel_size(raw: bytes) -> Optional[str]:
- """``"{w}x{h}"`` for a PNG payload, or None if not a PNG IHDR."""
- import struct
- if len(raw) < 24 or raw[:8] != b"\x89PNG\r\n\x1a\n" or raw[12:16] != b"IHDR":
- return None
- width, height = struct.unpack(">II", raw[16:24])
- return f"{width}x{height}"
- class OpenAICodexImageGenProvider(StaticImageGenProvider):
- """gpt-image-2 routed through ChatGPT/Codex OAuth instead of an API key."""
- provider_id = "openai-codex"
- label = "OpenAI (Codex auth)"
- models = GPT_IMAGE_2_TIERS
- default_model_id = DEFAULT_MODEL
- price = "varies"
- def is_available(self) -> bool:
- return bool(_read_codex_credential()[0]) and _httpx_available()
- def get_setup_schema(self) -> dict[str, Any]:
- return {
- "name": "OpenAI (Codex auth)",
- "badge": "free",
- "tag": "gpt-image-2 via ChatGPT/Codex OAuth — no API key required; supports text and image inputs",
- "env_vars": [],
- # Empty env_vars means the picker writes the selection without a credential prompt; the shared
- # Codex OAuth bootstrap hook (hermes_cli/tools_config_post_setup.py) starts the sign-in (#102144).
- "post_setup": "openai_codex",
- "post_setup_hint": (
- "Sign in with `hermes auth add openai-codex` (or `hermes setup` → Codex) "
- "if you haven't already. No API key needed."),
- }
- def capabilities(self) -> dict[str, Any]:
- return {"modalities": ["text", "image"], "max_reference_images": _MAX_REFERENCE_IMAGES}
- def generate(
- self, prompt: str, aspect_ratio: str = DEFAULT_ASPECT_RATIO, *,
- image_url: Optional[str] = None, reference_image_urls: Optional[list[str]] = None,
- **kwargs: Any,
- ) -> dict[str, Any]:
- prompt = (prompt or "").strip()
- aspect = resolve_aspect_ratio(aspect_ratio)
- if not prompt:
- return prompt_required_error("openai-codex", aspect)
- token, base_url = _read_codex_credential()
- if not token:
- return error_factory("openai-codex", aspect)(_NO_AUTH, "auth_required")
- if not _httpx_available():
- return error_factory("openai-codex", aspect)(
- "httpx Python package not installed (pip install httpx)", "missing_dependency")
- tier_id, meta = _resolve_model()
- size = size_for(aspect)
- fail = error_factory("openai-codex", aspect, model=tier_id, prompt=prompt)
- try:
- input_images = _normalize_input_images(image_url, reference_image_urls)
- except Exception as exc:
- return fail(f"Invalid image input for Codex image editing: {exc}", "invalid_image_input")
- try:
- payload = _post_image_request(
- token, prompt=prompt, size=size, quality=meta["quality"], input_images=input_images or None,
- base_url=base_url)
- except Exception as exc:
- logger.debug("Codex image generation failed", exc_info=True)
- return fail(f"OpenAI image generation via Codex auth failed: {exc}", "api_error")
- data = payload.get("data")
- b64 = data[0].get("b64_json") if isinstance(data, list) and data and isinstance(data[0], dict) else None
- if not isinstance(b64, str) or not b64:
- return fail("Codex images API response contained no image data", "empty_response")
- try:
- pixel_size = _png_pixel_size(base64.b64decode(b64))
- saved_path = save_b64_image(b64, prefix=f"openai_codex_{tier_id}")
- except Exception as exc:
- return fail(f"Could not save image to cache: {exc}", "io_error")
- return success_response(
- image=str(saved_path), model=tier_id, prompt=prompt, aspect_ratio=aspect,
- provider="openai-codex", modality="image" if input_images else "text",
- extra={
- "size": size, "quality": meta["quality"], "input_image_count": len(input_images),
- "requested_size": size, "pixel_size": pixel_size,
- "reported_quality": payload.get("quality"), "reported_size": payload.get("size"),
- "imagegen_request_id": payload.get("imagegen_request_id"),
- })
- def register(ctx) -> None:
- """Plugin entry point — register the Codex-backed image-gen provider."""
- ctx.register_image_gen_provider(OpenAICodexImageGenProvider())
|