gateway_media.py 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111
  1. """Editor prompt content -> ``prompt.submit`` text and staged image attachments (ACP gateway bridge)."""
  2. from __future__ import annotations
  3. import asyncio
  4. import logging
  5. import os
  6. from contextlib import suppress
  7. from hermes_cli.gateway_client import GatewayClientError
  8. logger = logging.getLogger(__name__)
  9. async def _fetch_remote_image(url):
  10. """Bytes of an editor's remote image URL, or None when it cannot be fetched.
  11. The in-process server passed such a URL to the model as an image part; admission carries only
  12. staged bytes, so the adapter downloads it through the gateway's SSRF-guarded image fetcher
  13. (pre-flight and per-redirect address checks, the inbound media size cap) and stages the bytes
  14. like any other image (#134889 ACP-16)."""
  15. from gateway.platforms.base import cache_image_from_url
  16. try:
  17. path = await cache_image_from_url(url, ".img", retries=0)
  18. except Exception:
  19. # Boundary: an unsafe address, an HTTP error, a non-image body or a transport failure
  20. # degrades to the URL as text, never a failed prompt.
  21. logger.info("ACP remote image not fetched; sent as text", exc_info=True)
  22. return None
  23. return await asyncio.to_thread(_take, path)
  24. def _take(path):
  25. """The fetched file's bytes; the cache copy is removed (staging writes its own sniffed-type file)."""
  26. try:
  27. with open(path, "rb") as fetched:
  28. return fetched.read()
  29. finally:
  30. with suppress(OSError):
  31. os.unlink(path)
  32. async def stage_user_content(content, home=None):
  33. """Shared-converter output -> ``(text, attachments)`` for ``prompt.submit``.
  34. Text-only prompts stay a plain string. Media parts (image ``data:`` URLs from
  35. image blocks, image resource links and embedded blobs, and remote image URLs once
  36. fetched) are staged as bytes in the profile image cache; the authority commits them
  37. at admission. A remote image URL that cannot be fetched is kept as text so the model
  38. still sees it. Only a remote image awaits (its download); every other part stages inline.
  39. ``home`` is the owner's profile home: a per-task overlay ``HERMES_HOME`` does not share
  40. ``cache/images``, and admission accepts only the owner's cache (never relaxed here)."""
  41. if isinstance(content, str):
  42. return content, []
  43. if home is not None:
  44. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  45. token = set_hermes_home_override(home)
  46. try:
  47. return await stage_user_content(content)
  48. finally:
  49. reset_hermes_home_override(token)
  50. import base64
  51. import binascii
  52. from gateway.platforms.base import cache_image_from_bytes
  53. from gateway.session_ingress_media import _ATTACHMENT_LIMIT, _IMAGE_EXT, sniff_image_mime
  54. texts, attachments = [], []
  55. for part in content:
  56. if part.get('type') == 'text':
  57. texts.append(part['text'])
  58. continue
  59. url = part['image_url']['url']
  60. if len(attachments) >= _ATTACHMENT_LIMIT:
  61. raise GatewayClientError('acp_content_invalid')
  62. if url.startswith('data:'):
  63. try:
  64. raw = base64.b64decode(url.partition(',')[2], validate=True)
  65. except (binascii.Error, ValueError) as exc:
  66. raise GatewayClientError('acp_content_invalid') from exc
  67. else:
  68. raw = await _fetch_remote_image(url) if url.startswith(('http://', 'https://')) else None
  69. if raw is None:
  70. texts.append(f"[Image attached: {url}]")
  71. continue
  72. # The sniffed bytes are authoritative for the staged extension and admitted type; the
  73. # client's declared type (a file suffix, or image/png by default) is never trusted. A
  74. # part admission cannot carry (bmp/svg, over the media cap) degrades to a text note so
  75. # the user's text still runs, as main's in-process server did for oversize images.
  76. mime = sniff_image_mime(raw)
  77. if mime is None:
  78. texts.append("[Image omitted: unsupported image format]")
  79. continue
  80. try:
  81. path = cache_image_from_bytes(raw, _IMAGE_EXT[mime])
  82. except ValueError as exc:
  83. texts.append(f"[Image omitted: {exc}]")
  84. continue
  85. attachments.append({'path': path, 'mime': mime})
  86. return "\n".join(texts), attachments
  87. def submit_fingerprint(session_id, content):
  88. """Identity of one logical prompt as the editor sent it (text, image ``data:`` bytes, remote image
  89. URLs), taken before staging: a retry of the same prompt matches without re-fetching a remote image,
  90. whose second download could fail or return other bytes and so change what is submitted."""
  91. import hashlib
  92. digest = hashlib.sha256()
  93. for part in [content] if isinstance(content, str) else content:
  94. kind, value = ('text', part) if isinstance(part, str) else \
  95. ('text', part['text']) if part.get('type') == 'text' else ('image', part['image_url']['url'])
  96. digest.update(kind.encode() + len(value).to_bytes(8, 'big') + value.encode())
  97. return session_id, digest.hexdigest()