session_api_media.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175
  1. """API image parts become committed media references that outlive the transient turn."""
  2. import base64
  3. import hashlib
  4. import os
  5. from pathlib import Path
  6. _MIME_EXT = {'image/png': '.png', 'image/jpeg': '.jpg', 'image/gif': '.gif', 'image/webp': '.webp'}
  7. # Same per-admission count as ``prompt.submit`` attachments (``session_ingress_media._ATTACHMENT_LIMIT``).
  8. _IMAGE_LIMIT = 10
  9. def _data_url_bytes(url):
  10. """``(mime, bytes)`` for a committable ``data:image/...;base64,`` URL, else ``None``."""
  11. header, _, encoded = url.partition(',')
  12. mime = header[len('data:'):].split(';', 1)[0].strip().lower()
  13. if not url.lower().startswith('data:') or mime not in _MIME_EXT or ';base64' not in header.lower():
  14. return None
  15. try:
  16. return mime, base64.b64decode(encoded, validate=True)
  17. except ValueError:
  18. return None
  19. def _image_urls(content):
  20. for part in content:
  21. if isinstance(part, dict) and part.get('type') == 'image_url':
  22. yield (part.get('image_url') or {}).get('url', '')
  23. def commit_api_images(content):
  24. """Stage every inline ``data:`` image deterministically (same bytes -> same path, so an
  25. exact retry keeps its admission digest) and commit the bytes as immutable media."""
  26. from gateway.platforms.base import get_image_cache_dir
  27. from gateway.platforms.base import get_inbound_media_max_bytes
  28. from gateway.session_ingress_media import capture_native_media, sniff_image_mime
  29. from hermes_state_runtime import RuntimeStoreError
  30. # The bytes decide the committed type: SDKs often label JPEG bytes ``data:image/png`` and
  31. # main forwarded those to the provider; only bytes that are no supported image are refused.
  32. images = [(sniff_image_mime(decoded[1]), decoded[1])
  33. for decoded in map(_data_url_bytes, _image_urls(content)) if decoded is not None]
  34. # Validate the whole batch before one byte lands on disk: a refused request must not
  35. # leave staged or committed bytes behind.
  36. limit = max(0, get_inbound_media_max_bytes())
  37. if (len(images) > _IMAGE_LIMIT or (limit and sum(len(data) for _, data in images) > limit)
  38. or any(mime is None for mime, _ in images)):
  39. raise RuntimeStoreError('invalid_params')
  40. staged = []
  41. for mime, data in images:
  42. path = Path(get_image_cache_dir()).resolve() / ('api_' + hashlib.sha256(data).hexdigest()[:32] + _MIME_EXT[mime])
  43. if not path.exists():
  44. temporary = path.with_name(f'{path.name}.{os.getpid()}.tmp')
  45. temporary.write_bytes(data)
  46. os.replace(temporary, path)
  47. staged.append(path)
  48. return capture_native_media(staged)
  49. def restore_api_images(content, media):
  50. """The transient content keeps its pixels; its text gains the validated committed reference
  51. per inline image (remote URLs keep their address) so the persisted user row records what
  52. was seen, exactly like native image turns."""
  53. from gateway.session_ingress_media import restore_native_media
  54. paths = iter(restore_native_media(media))
  55. hints = []
  56. for url in _image_urls(content):
  57. if _data_url_bytes(url) is not None:
  58. hints.append(f'[Image attached at: {next(paths)}]')
  59. else:
  60. hints.append(f'[Image attached: {url}]')
  61. if not hints:
  62. return content
  63. parts = [dict(part) for part in content]
  64. text = next((part for part in parts if part.get('type') == 'text'), None)
  65. if text is None:
  66. text = {'type': 'text', 'text': 'What do you see in this image?'}
  67. parts.insert(0, text)
  68. text['text'] = text['text'] + '\n\n' + '\n'.join(hints)
  69. return parts
  70. # A settled admission's inline ``data:`` image is replaced by this reference to its committed bytes.
  71. _RETAINED = 'hermes-retained:'
  72. def _compacted(content, media):
  73. """``content`` with every committed inline image reduced to its digest, or ``None`` when
  74. nothing changes. Only bytes this admission committed are compacted (same order as capture)."""
  75. if not isinstance(content, list) or not media:
  76. return None
  77. digests, parts, changed = iter(reference['sha256'] for reference in media), [], False
  78. for part in content:
  79. url = (part.get('image_url') or {}).get('url', '') if isinstance(part, dict) and part.get('type') == 'image_url' else ''
  80. decoded = _data_url_bytes(url) if url else None
  81. digest = next(digests, None) if decoded is not None else None
  82. if digest is not None and hashlib.sha256(decoded[1]).hexdigest() == digest:
  83. part = {**part, 'image_url': {**part['image_url'], 'url': _RETAINED + digest}}
  84. changed = True
  85. parts.append(part)
  86. return parts if changed else None
  87. def compact_settled_api_payloads(db, admission_id=None):
  88. """Drop what a TERMINAL API admission row no longer needs: one row at settlement, or
  89. (``admission_id=None``) the startup sweep for rows a crash left between settlement and this pass.
  90. - The redundant base64 copy of committed images in ``text``: the bytes stay in content-addressed
  91. ``native-inputs`` (held by ``api_turn_v1.media`` as history context).
  92. - The caller-supplied ``api_turn_v1.history`` (every turn carries the whole conversation, inline
  93. images included, so a session's ledger grew quadratically). Execution read it; a terminal row
  94. is read only by exact-retry projections. Responses rows keep it: their terminal replay rebuilds
  95. the ``previous_response_id`` snapshot from it.
  96. ``payload_digest`` is untouched, so an exact retry still matches. Queued/started/unknown rows
  97. are never rewritten: they may still execute."""
  98. import json
  99. from hermes_state_runtime import _json
  100. # LIKE, not instr(): SQLite LIKE is ASCII case-insensitive, as ``_data_url_bytes`` is.
  101. candidates = ("SELECT admission_id FROM session_admissions WHERE status='terminal' AND principal_id='api' "
  102. "AND ((json_type(payload_json, '$.api_turn_v1.media') IS NOT NULL "
  103. "AND payload_json LIKE '%data:image/%') OR (json_type(payload_json, '$.api_turn_v1.history')='array' "
  104. "AND request_id NOT LIKE 'responses:%'))")
  105. if admission_id is None:
  106. # The scan runs on a read snapshot; a write transaction opens only when there is work.
  107. with db._read_ctx() as conn:
  108. ids = [row[0] for row in conn.execute(candidates)]
  109. else:
  110. ids = [admission_id]
  111. def write(conn):
  112. compacted = 0
  113. for row_id in ids:
  114. row = conn.execute(candidates.replace('SELECT admission_id', 'SELECT payload_json, request_id')
  115. + ' AND admission_id=?', (row_id,)).fetchone()
  116. if row is None:
  117. continue
  118. payload = json.loads(row[0])
  119. data = payload['api_turn_v1']
  120. text = _compacted(payload.get('text'), data.get('media'))
  121. drop = isinstance(data.get('history'), list) and not row[1].startswith('responses:')
  122. if text is not None or drop:
  123. payload = {**payload, 'text': payload.get('text') if text is None else text,
  124. 'api_turn_v1': {**data, 'history': None} if drop else data}
  125. conn.execute('UPDATE session_admissions SET payload_json=? WHERE admission_id=?',
  126. (_json(payload), row_id))
  127. compacted += 1
  128. return compacted
  129. return db._execute_write(write) if ids else 0
  130. def rehydrate_api_images(content, media):
  131. """The original request content of a compacted settled row, for terminal replay projections;
  132. an image whose committed bytes are gone becomes a text notice rather than a dangling URL."""
  133. if not isinstance(content, list):
  134. return content
  135. from gateway.session_ingress_media import restore_native_media
  136. from hermes_state_runtime import RuntimeStoreError
  137. by_digest = {reference['sha256']: reference for reference in media or ()}
  138. mimes = {ext: mime for mime, ext in _MIME_EXT.items()}
  139. parts = []
  140. for part in content:
  141. url = (part.get('image_url') or {}).get('url', '') if isinstance(part, dict) and part.get('type') == 'image_url' else ''
  142. reference = by_digest.get(url[len(_RETAINED):]) if url.startswith(_RETAINED) else None
  143. if url.startswith(_RETAINED):
  144. try:
  145. path = Path(restore_native_media([reference])[0]) if reference else None
  146. except RuntimeStoreError:
  147. path = None
  148. if path is None:
  149. part = {'type': 'text', 'text': '[Image no longer retained]'}
  150. else:
  151. data = base64.b64encode(path.read_bytes()).decode()
  152. part = {**part, 'image_url': {**part['image_url'],
  153. 'url': f'data:{mimes.get(path.suffix, "image/png")};base64,{data}'}}
  154. parts.append(part)
  155. return parts