imagegen.py 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184
  1. """Thin image-generation layer for pet sprites.
  2. Wraps the active ``ImageGenProvider`` with what the ``image_generate`` tool
  3. doesn't expose: N variants and reference-image grounding (each animation row
  4. stays the same character as the chosen base). Grounding needs a ref-capable
  5. provider; we resolve to one or raise an actionable error rather than drift.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import os
  10. from dataclasses import dataclass
  11. from pathlib import Path
  12. logger = logging.getLogger(__name__)
  13. # Providers that can ground generation on a reference image, in preference order.
  14. # OpenRouter/Nous run a quality-first model chain and may fall back depending on
  15. # account access, so fidelity can vary by configured backend.
  16. _REF_CAPABLE = ("nous", "openai", "openai-codex", "openrouter", "krea")
  17. # Friendly display label per reference-capable provider (desktop pet-gen picker).
  18. _PROVIDER_LABELS = {"nous": "Nous Portal", "openrouter": "OpenRouter", "openai": "OpenAI", "openai-codex": "OpenAI (Codex)", "krea": "Krea"}
  19. class GenerationError(RuntimeError):
  20. """Raised on any image-generation failure (no provider, API error, IO)."""
  21. @dataclass(frozen=True)
  22. class SpriteProvider:
  23. """Resolved provider plus whether it can take reference images."""
  24. name: str
  25. provider: object
  26. supports_references: bool
  27. def _discover() -> None:
  28. try:
  29. from hermes_cli.plugins import _ensure_plugins_discovered
  30. _ensure_plugins_discovered()
  31. except Exception as exc:
  32. logger.debug("image-gen plugin discovery failed: %s", exc)
  33. def _available(name: str):
  34. """The registered provider *name* if it exists and has credentials, else ``None``."""
  35. from agent.image_gen_registry import get_provider
  36. provider = get_provider(name)
  37. return provider if provider is not None and provider.is_available() else None
  38. def resolve_provider(*, require_references: bool = True, prefer: str | None = None) -> SpriteProvider:
  39. """Pick the image provider for sprite work.
  40. Preference: ``HERMES_PET_IMAGE_PROVIDER`` (QA override, unknown values ignored),
  41. then *prefer* (desktop picker), then the active provider, then the first available
  42. — each only if ref-capable and configured. With *require_references* off, any
  43. available active provider is accepted (prompt-only base drafts).
  44. """
  45. _discover()
  46. from agent.image_gen_registry import get_active_provider
  47. forced = os.environ.get("HERMES_PET_IMAGE_PROVIDER", "").strip().lower()
  48. for name in (forced, prefer):
  49. if name in _REF_CAPABLE and (chosen := _available(name)) is not None:
  50. return SpriteProvider(name=name, provider=chosen, supports_references=True)
  51. try:
  52. active = get_active_provider()
  53. except Exception:
  54. active = None
  55. active_name = getattr(active, "name", "") if active is not None else ""
  56. if active_name in _REF_CAPABLE and active.is_available():
  57. return SpriteProvider(name=active_name, provider=active, supports_references=True)
  58. for name in _REF_CAPABLE:
  59. if (provider := _available(name)) is not None:
  60. return SpriteProvider(name=name, provider=provider, supports_references=True)
  61. if not require_references and active is not None and active.is_available():
  62. return SpriteProvider(name=getattr(active, "name", "unknown"), provider=active, supports_references=False)
  63. raise GenerationError(
  64. "Pet generation needs an image backend that supports reference images. "
  65. "Open `hermes tools` → Image Generation and configure Nous Portal, "
  66. "OpenRouter, or OpenAI (gpt-image-2) with an API key."
  67. )
  68. def list_sprite_providers() -> list[dict]:
  69. """``[{name, label, default}]`` per configured ref-capable provider, in preference order; empty hides the picker."""
  70. _discover()
  71. try:
  72. default_name = resolve_provider(require_references=True).name
  73. except GenerationError:
  74. default_name = ""
  75. return [
  76. {"name": name, "label": _PROVIDER_LABELS.get(name, name), "default": name == default_name}
  77. for name in _REF_CAPABLE
  78. if _available(name) is not None
  79. ]
  80. def _save_local(image_ref: str, *, prefix: str) -> Path:
  81. """Return a local path for *image_ref*, downloading it if it's a URL."""
  82. if image_ref.startswith(("http://", "https://")):
  83. from agent.image_gen_provider import save_url_image
  84. return Path(save_url_image(image_ref, prefix=prefix))
  85. return Path(image_ref)
  86. def _rejected_background(error: str) -> bool:
  87. """True when a provider error is specifically about ``background=transparent`` (a per-model capability; we retry without it)."""
  88. lowered = (error or "").lower()
  89. return "background" in lowered and ("not supported" in lowered or "transparent" in lowered)
  90. def generate(
  91. prompt: str,
  92. *,
  93. n: int = 1,
  94. reference_images: list[Path] | None = None,
  95. provider: SpriteProvider | None = None,
  96. prefix: str = "pet_gen",
  97. aspect_ratio: str = "square",
  98. ) -> list[Path]:
  99. """Generate *n* sprite images and return their local paths.
  100. *reference_images* grounds the output on a base image (required for rows).
  101. *aspect_ratio* ``"landscape"`` (row strips) gives every frame horizontal room so
  102. winged poses needn't shrink. We *ask* for a transparent background but fall
  103. back to opaque on models that reject the flag. Raises :class:`GenerationError`
  104. if nothing usable comes back.
  105. """
  106. sprite = provider or resolve_provider(require_references=bool(reference_images))
  107. if reference_images and not sprite.supports_references:
  108. raise GenerationError(
  109. f"image backend '{sprite.name}' cannot use reference images; "
  110. "configure OpenAI gpt-image-2 or Krea for pet generation"
  111. )
  112. refs = [str(p) for p in (reference_images or [])]
  113. # Providers disagree on the ref kwarg name: our OpenRouter/Nous backends read
  114. # ``reference_images``, OpenAI's gpt-image-2 reads ``reference_image_urls``.
  115. # Send both; each ignores the other.
  116. ref_kwargs = {"reference_images": refs, "reference_image_urls": refs} if refs else {}
  117. def _run(extra: dict) -> tuple[Path | None, str]:
  118. try:
  119. result = sprite.provider.generate(prompt, aspect_ratio=aspect_ratio, **extra, **ref_kwargs)
  120. except Exception as exc:
  121. logger.debug("provider.generate crashed: %s", exc)
  122. return None, str(exc)
  123. if not isinstance(result, dict):
  124. return None, "no result"
  125. if not result.get("success"):
  126. return None, result.get("error", "unknown error")
  127. if not (image_ref := result.get("image")):
  128. return None, "provider returned no image"
  129. try:
  130. return _save_local(str(image_ref), prefix=prefix), ""
  131. except Exception as exc:
  132. return None, f"could not save generated image: {exc}"
  133. out: list[Path] = []
  134. last_error = ""
  135. allow_transparent = True
  136. for _ in range(max(1, n)):
  137. path, err = _run({"background": "transparent"} if allow_transparent else {})
  138. # Model doesn't support the transparent flag → drop it for this and every
  139. # remaining variant (no point re-probing a capability we just disproved).
  140. if path is None and allow_transparent and _rejected_background(err):
  141. allow_transparent = False
  142. path, err = _run({})
  143. if path is not None:
  144. out.append(path)
  145. else:
  146. last_error = err
  147. if not out:
  148. raise GenerationError(last_error or "image generation produced no output")
  149. return out