image_gen_provider.py 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139
  1. """Image generation provider ABC.
  2. Providers register via ``PluginContext.register_image_gen_provider()`` (from
  3. ``<repo>/plugins/image_gen/<name>/`` or ``~/.hermes/plugins/image_gen/<name>/``);
  4. the one selected by ``image_gen.provider`` services every ``image_generate`` call.
  5. One tool covers text-to-image and editing: ``image_url`` / ``reference_image_urls``
  6. route to the provider's edit endpoint, otherwise text-to-image. Mirrors
  7. ``agent/video_gen_provider.py``. Response dicts come from :func:`success_response`
  8. / :func:`error_response` (``success, image, model, prompt, aspect_ratio, modality,
  9. provider`` + ``error, error_type`` on failure).
  10. """
  11. from __future__ import annotations
  12. import abc
  13. import logging
  14. from pathlib import Path
  15. from typing import Any, Dict, List, Optional, Tuple
  16. from agent import provider_media
  17. from agent.provider_base import CatalogProviderBase
  18. logger = logging.getLogger(__name__)
  19. VALID_ASPECT_RATIOS: tuple[str, ...] = ("landscape", "square", "portrait")
  20. DEFAULT_ASPECT_RATIO = "landscape"
  21. class ImageGenProvider(CatalogProviderBase):
  22. """Abstract base class for an image generation backend. Subclasses implement
  23. :attr:`name` and :meth:`generate`; ``list_models`` entries may add
  24. ``speed`` / ``strengths`` / ``price`` for the picker."""
  25. def capabilities(self) -> dict[str, Any]:
  26. """``modalities`` (``"text"`` and/or ``"image"``) and ``max_reference_images``; optionally
  27. ``supports_upscale`` (bool), ``creative_controls`` (names from the tool's creative-control
  28. vocabulary: ``creativity``, ``intensity``, ``complexity``, ``movement``, ``thinking``,
  29. ``resolution``) and
  30. ``source_image_role`` (``"style"`` when image inputs set the look and are not edited). Surfaced in the
  31. dynamic tool schema so the model knows when ``image_url`` / ``upscale`` / each control is
  32. honored, and only declared controls are passed to :meth:`generate`; the text-only default
  33. keeps non-overriding providers backward compatible."""
  34. return {"modalities": ["text"], "max_reference_images": 0}
  35. @abc.abstractmethod
  36. def generate(
  37. self,
  38. prompt: str,
  39. aspect_ratio: str = DEFAULT_ASPECT_RATIO,
  40. *,
  41. image_url: Optional[str] = None,
  42. reference_image_urls: Optional[list[str]] = None,
  43. **kwargs: Any,
  44. ) -> dict[str, Any]:
  45. """Generate an image, or edit ``image_url`` (``reference_image_urls`` are extra
  46. style/composition refs, clamped to ``max_reference_images``); any source image
  47. routes to the edit endpoint. Return :func:`success_response` / :func:`error_response`.
  48. Unknown ``kwargs`` MUST be ignored (forward compat); ``upscale`` (bool) is a
  49. post-generation high-res pass, reported as ``upscaled: True`` in ``extra``."""
  50. def resolve_aspect_ratio(value: Optional[str]) -> str:
  51. """Clamp to :data:`VALID_ASPECT_RATIOS`; invalid values coerce to landscape so
  52. the tool surface forgives agent mistakes instead of rejecting them."""
  53. v = value.strip().lower() if isinstance(value, str) else ""
  54. return v if v in VALID_ASPECT_RATIOS else DEFAULT_ASPECT_RATIO
  55. def normalize_reference_images(value: Any) -> Optional[list[str]]:
  56. """Coerce a str or list into a clean list of non-blank strings; ``None`` when
  57. nothing usable remains so providers treat "no refs" as one sentinel."""
  58. if isinstance(value, str):
  59. value = [value]
  60. if not isinstance(value, (list, tuple)):
  61. return None
  62. return [item.strip() for item in value if isinstance(item, str) and item.strip()] or None
  63. _GENERATED_IMAGE_KIND = f"{provider_media.GENERATED_SUBDIR}/images"
  64. def save_b64_image(b64_data: str, *, prefix: str = "image", extension: str = "png") -> Path:
  65. """Decode base64 image data into ``$HERMES_HOME/cache/generated/images/``; return the path."""
  66. return provider_media.save_b64(_GENERATED_IMAGE_KIND, b64_data, prefix=prefix, extension=extension)
  67. _URL_IMAGE_CONTENT_TYPES = {
  68. "image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg", "image/webp": "webp", "image/gif": "gif",
  69. }
  70. def save_url_image(
  71. url: str, *, prefix: str = "image", timeout: float = 60.0, max_bytes: int = 25 * 1024 * 1024,
  72. ) -> Path:
  73. """Download an (often ephemeral) image URL into ``$HERMES_HOME/cache/generated/images/``. Raises on
  74. network / HTTP / oversize / empty errors so callers can fall back to the bare URL."""
  75. return provider_media.save_url(
  76. _GENERATED_IMAGE_KIND, url, prefix=prefix, timeout=timeout, max_bytes=max_bytes,
  77. chunk_size=64 * 1024, content_types=_URL_IMAGE_CONTENT_TYPES,
  78. url_extensions=("png", "jpg", "jpeg", "webp", "gif"), default_extension="png",
  79. label="Image", empty_error="Image at {url} returned 0 bytes; refusing to cache.",
  80. )
  81. def success_response(
  82. *,
  83. image: str,
  84. model: str,
  85. prompt: str,
  86. aspect_ratio: str,
  87. provider: str,
  88. modality: str = "text",
  89. extra: Optional[dict[str, Any]] = None,
  90. ) -> dict[str, Any]:
  91. """Uniform success dict; ``extra`` keys are added without overriding standard ones."""
  92. payload: dict[str, Any] = {
  93. "success": True, "image": image, "model": model, "prompt": prompt,
  94. "aspect_ratio": aspect_ratio, "modality": modality, "provider": provider,
  95. }
  96. for k, v in (extra or {}).items():
  97. payload.setdefault(k, v)
  98. return payload
  99. def error_response(
  100. *,
  101. error: str,
  102. error_type: str = "provider_error",
  103. provider: str = "",
  104. model: str = "",
  105. prompt: str = "",
  106. aspect_ratio: str = DEFAULT_ASPECT_RATIO,
  107. ) -> dict[str, Any]:
  108. """Build a uniform error response dict."""
  109. return {
  110. "success": False, "image": None, "error": error, "error_type": error_type,
  111. "model": model, "prompt": prompt, "aspect_ratio": aspect_ratio, "provider": provider,
  112. }