__init__.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220
  1. """OpenAI GPT Image 2 and 2.5 Flare/Sunburst quality tiers;
  2. base64 output → image cache. Selection: ``OPENAI_IMAGE_MODEL`` → ``image_gen.openai.model`` →
  3. ``image_gen.model`` → :data:`DEFAULT_MODEL`; an id outside the catalog is sent verbatim.
  4. Endpoint: ``image_gen.openai.base_url`` → the named endpoint ``image_gen.openai.provider`` →
  5. ``OPENAI_BASE_URL`` → SDK default; key: env named by ``image_gen.openai.key_env`` → the named
  6. endpoint's credential → ``OPENAI_API_KEY``."""
  7. from __future__ import annotations
  8. import io
  9. import logging
  10. import os
  11. from typing import Any, Dict, List, Optional, Tuple
  12. from agent.secret_scope import get_secret, get_secret_str
  13. from agent.image_gen_provider import DEFAULT_ASPECT_RATIO, resolve_aspect_ratio, success_response
  14. from plugins.image_gen._common import (
  15. GPT_IMAGE_2_API_MODEL as API_MODEL, GPT_IMAGE_2_DEFAULT as DEFAULT_MODEL, GPT_IMAGE_2_TIERS,
  16. StaticImageGenProvider, collect_source_images, error_factory, import_openai, load_image_gen_config,
  17. materialize_image, openai_importable, prompt_required_error, record_token_usage, resolve_static_model,
  18. size_for)
  19. logger = logging.getLogger(__name__)
  20. # Keep subscription routing independent: Codex does not verify explicit image model selection.
  21. MODELS = {
  22. **{key: {**meta, "api_model": API_MODEL} for key, meta in GPT_IMAGE_2_TIERS.items()},
  23. **{
  24. model if quality == "auto" else f"{model}-{quality}": {
  25. "display": f"GPT Image 2.5 {name} ({quality.title()})",
  26. "speed": speed,
  27. "strengths": strengths,
  28. "api_model": model,
  29. "quality": quality,
  30. }
  31. for model, name, speed, strengths in (
  32. ("gpt-image-2.5-flare", "Flare", "Fast", "Everyday image generation and editing"),
  33. ("gpt-image-2.5-sunburst", "Sunburst", "Slower", "Precision generation and editing"),
  34. )
  35. for quality in ("auto", "low", "medium", "high", "xhigh", "max")
  36. },
  37. }
  38. def _resolve_model() -> tuple[str, dict[str, Any]]:
  39. return resolve_static_model(
  40. MODELS, DEFAULT_MODEL, env_var="OPENAI_IMAGE_MODEL", config_key="openai", passthrough=True)
  41. def _named_endpoint(name: str) -> tuple[str, str]:
  42. """``(base_url, api_key)`` of the user-declared custom endpoint *name* (``providers:`` /
  43. ``custom_providers:``), so image generation reuses a chat endpoint's URL and credential without
  44. duplicating the key into OpenAI variables (#83080). Unknown name → ``("", "")`` with a warning."""
  45. from hermes_cli.runtime_provider import _get_named_custom_provider
  46. entry = _get_named_custom_provider(name)
  47. if not entry:
  48. logger.warning("image_gen.openai.provider %r matches no custom endpoint in providers:", name)
  49. return "", ""
  50. key_env = str(entry.get("key_env") or "").strip()
  51. api_key = str(entry.get("api_key") or "").strip() or (get_secret(key_env) if key_env else None) or ""
  52. return str(entry.get("base_url") or "").strip().rstrip("/"), api_key
  53. def _resolve_endpoint() -> tuple[str, str]:
  54. """``(base_url, api_key)`` — ``image_gen.openai.base_url`` → ``OPENAI_BASE_URL`` → ``""`` (SDK default);
  55. the env var named by ``image_gen.openai.key_env`` → ``OPENAI_API_KEY``. Only the var NAME lives in
  56. config.yaml; ``is_available()`` and ``generate()`` share this so they cannot disagree (#65309)."""
  57. cfg = load_image_gen_config("openai")
  58. named = str(cfg.get("provider") or "").strip()
  59. named_base, named_key = _named_endpoint(named) if named else ("", "")
  60. base_url = (str(cfg.get("base_url") or "").strip().rstrip("/") or named_base
  61. or get_secret_str("OPENAI_BASE_URL").strip())
  62. key_env = str(cfg.get("key_env") or "").strip()
  63. api_key = (get_secret(key_env) if key_env else None) or named_key or get_secret("OPENAI_API_KEY") or ""
  64. return base_url, api_key
  65. def _build_client(openai: Any, base_url: str, api_key: str) -> Any:
  66. """``openai.OpenAI`` on Hermes' env-only-proxy httpx client, so a local/custom endpoint never
  67. routes through a macOS system proxy whose ExceptionsList httpx cannot see (#64888). The project
  68. header is blanked: an ``OPENAI_PROJECT_ID`` set for chat makes ``/images/generations`` 403 on
  69. projects with a model allow-list, and the key already carries the project (#60748)."""
  70. from agent.process_bootstrap import build_keepalive_http_client
  71. kwargs: dict[str, Any] = {"api_key": api_key, "default_headers": {"OpenAI-Project": ""}}
  72. if base_url:
  73. kwargs["base_url"] = base_url
  74. http_client = build_keepalive_http_client(base_url)
  75. if http_client is not None:
  76. kwargs["http_client"] = http_client
  77. return openai.OpenAI(**kwargs)
  78. def _load_image_bytes(ref: str) -> tuple[bytes, str]:
  79. """Load ``(data, filename)`` from a URL, data URI or local path; raises on IO/network error."""
  80. ref = ref.strip()
  81. lower = ref.lower()
  82. if lower.startswith(("http://", "https://")):
  83. from tools.url_safety import create_ssrf_safe_client, is_safe_url
  84. if not is_safe_url(ref):
  85. raise ValueError(f"Image reference URL failed the SSRF safety check: {ref}")
  86. with create_ssrf_safe_client(timeout=60, follow_redirects=True) as client:
  87. resp = client.get(ref)
  88. resp.raise_for_status()
  89. name = ref.split("?", 1)[0].rsplit("/", 1)[-1] or "image.png"
  90. return resp.content, name
  91. if lower.startswith("data:"):
  92. import base64
  93. header, _, b64 = ref.partition(",")
  94. ext = (header.split("image/", 1)[1].split(";", 1)[0] if "image/" in header else "") or "png"
  95. return base64.b64decode(b64), f"image.{ext}"
  96. from agent.file_safety import raise_if_read_blocked # credential-read guard before local bytes
  97. raise_if_read_blocked(ref)
  98. with open(ref, "rb") as fh:
  99. data = fh.read()
  100. return data, os.path.basename(ref) or "image.png"
  101. def _named_bytes_io(ref: str) -> io.BytesIO:
  102. """``images.edit()`` expects named file-like objects for correct multipart."""
  103. data, fname = _load_image_bytes(ref)
  104. bio = io.BytesIO(data)
  105. bio.name = fname
  106. return bio
  107. class OpenAIImageGenProvider(StaticImageGenProvider):
  108. """OpenAI ``images.generate`` / ``images.edit`` backend with selectable API models."""
  109. provider_id = "openai"
  110. label = "OpenAI"
  111. models = MODELS
  112. default_model_id = DEFAULT_MODEL
  113. price = "varies"
  114. setup = dict(
  115. name="OpenAI", badge="paid",
  116. tag="GPT Image 2 / 2.5 Flare / 2.5 Sunburst — text-to-image & image editing",
  117. key="OPENAI_API_KEY", prompt="OpenAI API key", url="https://platform.openai.com/api-keys")
  118. def is_available(self) -> bool:
  119. return bool(_resolve_endpoint()[1]) and openai_importable()
  120. def capabilities(self) -> dict[str, Any]:
  121. # images.edit() accepts up to 16 source images.
  122. return {"modalities": ["text", "image"], "max_reference_images": 16}
  123. def generate(
  124. self, prompt: str, aspect_ratio: str = DEFAULT_ASPECT_RATIO, *,
  125. image_url: Optional[str] = None, reference_image_urls: Optional[list[str]] = None,
  126. **kwargs: Any,
  127. ) -> dict[str, Any]:
  128. prompt = (prompt or "").strip()
  129. aspect = resolve_aspect_ratio(aspect_ratio)
  130. if not prompt:
  131. return prompt_required_error("openai", aspect)
  132. base_url, api_key = _resolve_endpoint()
  133. if not api_key:
  134. return error_factory("openai", aspect)(
  135. "OPENAI_API_KEY not set (or the variable named by image_gen.openai.key_env is empty). "
  136. "Run `hermes tools` → Image Generation → OpenAI to configure, or `hermes setup` "
  137. "to add the key.",
  138. "auth_required")
  139. openai, err = import_openai("openai", aspect)
  140. if err:
  141. return err
  142. tier_id, meta = _resolve_model()
  143. size = size_for(aspect)
  144. sources = collect_source_images(image_url, reference_image_urls, limit=16)
  145. is_edit = bool(sources)
  146. fail = error_factory("openai", aspect, model=tier_id, prompt=prompt)
  147. client = _build_client(openai, base_url, api_key)
  148. # gpt-image-2 returns b64_json unconditionally and REJECTS
  149. # ``response_format`` as an unknown parameter. Don't send it.
  150. # A custom (non-catalog) model id carries no quality tier: gateways reject unknown enum values.
  151. request: dict[str, Any] = dict(model=meta["api_model"], prompt=prompt, size=size, n=1)
  152. if meta["quality"] is not None:
  153. request["quality"] = meta["quality"]
  154. if is_edit:
  155. try:
  156. files = [_named_bytes_io(ref) for ref in sources]
  157. except Exception as exc:
  158. return fail(f"Could not load source image for editing: {exc}", "io_error")
  159. request["image"] = files if len(files) > 1 else files[0]
  160. verb, call = ("edit", client.images.edit) if is_edit else ("generation", client.images.generate)
  161. try:
  162. response = call(**request)
  163. except Exception as exc:
  164. logger.debug("OpenAI image %s failed", verb, exc_info=True)
  165. return fail(f"OpenAI image {'editing' if is_edit else 'generation'} failed: {exc}", "api_error")
  166. # gpt-image bills per text/image token; the tier id is a Hermes label, the API model prices.
  167. # Recorded before extraction/save: the tokens are billed whether or not an image came back.
  168. record_token_usage(getattr(response, "usage", None), model=meta["api_model"], provider="openai")
  169. data = getattr(response, "data", None) or []
  170. if not data:
  171. return fail("OpenAI returned no image data", "empty_response")
  172. first = data[0]
  173. image_ref, err = materialize_image(
  174. getattr(first, "b64_json", None), getattr(first, "url", None),
  175. prefix=f"openai_{tier_id}", label="OpenAI", provider="openai",
  176. model=tier_id, prompt=prompt, aspect=aspect, log=logger)
  177. if err:
  178. return err
  179. extra: dict[str, Any] = {"size": size, "quality": meta["quality"]}
  180. if getattr(first, "revised_prompt", None):
  181. extra["revised_prompt"] = first.revised_prompt
  182. return success_response(
  183. image=image_ref, model=tier_id, prompt=prompt, aspect_ratio=aspect, provider="openai",
  184. modality="image" if is_edit else "text", extra=extra)
  185. def register(ctx) -> None:
  186. """Plugin entry point — wire ``OpenAIImageGenProvider`` into the registry."""
  187. ctx.register_image_gen_provider(OpenAIImageGenProvider())