| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175 |
- """API image parts become committed media references that outlive the transient turn."""
- import base64
- import hashlib
- import os
- from pathlib import Path
- _MIME_EXT = {'image/png': '.png', 'image/jpeg': '.jpg', 'image/gif': '.gif', 'image/webp': '.webp'}
- # Same per-admission count as ``prompt.submit`` attachments (``session_ingress_media._ATTACHMENT_LIMIT``).
- _IMAGE_LIMIT = 10
- def _data_url_bytes(url):
- """``(mime, bytes)`` for a committable ``data:image/...;base64,`` URL, else ``None``."""
- header, _, encoded = url.partition(',')
- mime = header[len('data:'):].split(';', 1)[0].strip().lower()
- if not url.lower().startswith('data:') or mime not in _MIME_EXT or ';base64' not in header.lower():
- return None
- try:
- return mime, base64.b64decode(encoded, validate=True)
- except ValueError:
- return None
- def _image_urls(content):
- for part in content:
- if isinstance(part, dict) and part.get('type') == 'image_url':
- yield (part.get('image_url') or {}).get('url', '')
- def commit_api_images(content):
- """Stage every inline ``data:`` image deterministically (same bytes -> same path, so an
- exact retry keeps its admission digest) and commit the bytes as immutable media."""
- from gateway.platforms.base import get_image_cache_dir
- from gateway.platforms.base import get_inbound_media_max_bytes
- from gateway.session_ingress_media import capture_native_media, sniff_image_mime
- from hermes_state_runtime import RuntimeStoreError
- # The bytes decide the committed type: SDKs often label JPEG bytes ``data:image/png`` and
- # main forwarded those to the provider; only bytes that are no supported image are refused.
- images = [(sniff_image_mime(decoded[1]), decoded[1])
- for decoded in map(_data_url_bytes, _image_urls(content)) if decoded is not None]
- # Validate the whole batch before one byte lands on disk: a refused request must not
- # leave staged or committed bytes behind.
- limit = max(0, get_inbound_media_max_bytes())
- if (len(images) > _IMAGE_LIMIT or (limit and sum(len(data) for _, data in images) > limit)
- or any(mime is None for mime, _ in images)):
- raise RuntimeStoreError('invalid_params')
- staged = []
- for mime, data in images:
- path = Path(get_image_cache_dir()).resolve() / ('api_' + hashlib.sha256(data).hexdigest()[:32] + _MIME_EXT[mime])
- if not path.exists():
- temporary = path.with_name(f'{path.name}.{os.getpid()}.tmp')
- temporary.write_bytes(data)
- os.replace(temporary, path)
- staged.append(path)
- return capture_native_media(staged)
- def restore_api_images(content, media):
- """The transient content keeps its pixels; its text gains the validated committed reference
- per inline image (remote URLs keep their address) so the persisted user row records what
- was seen, exactly like native image turns."""
- from gateway.session_ingress_media import restore_native_media
- paths = iter(restore_native_media(media))
- hints = []
- for url in _image_urls(content):
- if _data_url_bytes(url) is not None:
- hints.append(f'[Image attached at: {next(paths)}]')
- else:
- hints.append(f'[Image attached: {url}]')
- if not hints:
- return content
- parts = [dict(part) for part in content]
- text = next((part for part in parts if part.get('type') == 'text'), None)
- if text is None:
- text = {'type': 'text', 'text': 'What do you see in this image?'}
- parts.insert(0, text)
- text['text'] = text['text'] + '\n\n' + '\n'.join(hints)
- return parts
- # A settled admission's inline ``data:`` image is replaced by this reference to its committed bytes.
- _RETAINED = 'hermes-retained:'
- def _compacted(content, media):
- """``content`` with every committed inline image reduced to its digest, or ``None`` when
- nothing changes. Only bytes this admission committed are compacted (same order as capture)."""
- if not isinstance(content, list) or not media:
- return None
- digests, parts, changed = iter(reference['sha256'] for reference in media), [], False
- for part in content:
- url = (part.get('image_url') or {}).get('url', '') if isinstance(part, dict) and part.get('type') == 'image_url' else ''
- decoded = _data_url_bytes(url) if url else None
- digest = next(digests, None) if decoded is not None else None
- if digest is not None and hashlib.sha256(decoded[1]).hexdigest() == digest:
- part = {**part, 'image_url': {**part['image_url'], 'url': _RETAINED + digest}}
- changed = True
- parts.append(part)
- return parts if changed else None
- def compact_settled_api_payloads(db, admission_id=None):
- """Drop what a TERMINAL API admission row no longer needs: one row at settlement, or
- (``admission_id=None``) the startup sweep for rows a crash left between settlement and this pass.
- - The redundant base64 copy of committed images in ``text``: the bytes stay in content-addressed
- ``native-inputs`` (held by ``api_turn_v1.media`` as history context).
- - The caller-supplied ``api_turn_v1.history`` (every turn carries the whole conversation, inline
- images included, so a session's ledger grew quadratically). Execution read it; a terminal row
- is read only by exact-retry projections. Responses rows keep it: their terminal replay rebuilds
- the ``previous_response_id`` snapshot from it.
- ``payload_digest`` is untouched, so an exact retry still matches. Queued/started/unknown rows
- are never rewritten: they may still execute."""
- import json
- from hermes_state_runtime import _json
- # LIKE, not instr(): SQLite LIKE is ASCII case-insensitive, as ``_data_url_bytes`` is.
- candidates = ("SELECT admission_id FROM session_admissions WHERE status='terminal' AND principal_id='api' "
- "AND ((json_type(payload_json, '$.api_turn_v1.media') IS NOT NULL "
- "AND payload_json LIKE '%data:image/%') OR (json_type(payload_json, '$.api_turn_v1.history')='array' "
- "AND request_id NOT LIKE 'responses:%'))")
- if admission_id is None:
- # The scan runs on a read snapshot; a write transaction opens only when there is work.
- with db._read_ctx() as conn:
- ids = [row[0] for row in conn.execute(candidates)]
- else:
- ids = [admission_id]
- def write(conn):
- compacted = 0
- for row_id in ids:
- row = conn.execute(candidates.replace('SELECT admission_id', 'SELECT payload_json, request_id')
- + ' AND admission_id=?', (row_id,)).fetchone()
- if row is None:
- continue
- payload = json.loads(row[0])
- data = payload['api_turn_v1']
- text = _compacted(payload.get('text'), data.get('media'))
- drop = isinstance(data.get('history'), list) and not row[1].startswith('responses:')
- if text is not None or drop:
- payload = {**payload, 'text': payload.get('text') if text is None else text,
- 'api_turn_v1': {**data, 'history': None} if drop else data}
- conn.execute('UPDATE session_admissions SET payload_json=? WHERE admission_id=?',
- (_json(payload), row_id))
- compacted += 1
- return compacted
- return db._execute_write(write) if ids else 0
- def rehydrate_api_images(content, media):
- """The original request content of a compacted settled row, for terminal replay projections;
- an image whose committed bytes are gone becomes a text notice rather than a dangling URL."""
- if not isinstance(content, list):
- return content
- from gateway.session_ingress_media import restore_native_media
- from hermes_state_runtime import RuntimeStoreError
- by_digest = {reference['sha256']: reference for reference in media or ()}
- mimes = {ext: mime for mime, ext in _MIME_EXT.items()}
- parts = []
- for part in content:
- url = (part.get('image_url') or {}).get('url', '') if isinstance(part, dict) and part.get('type') == 'image_url' else ''
- reference = by_digest.get(url[len(_RETAINED):]) if url.startswith(_RETAINED) else None
- if url.startswith(_RETAINED):
- try:
- path = Path(restore_native_media([reference])[0]) if reference else None
- except RuntimeStoreError:
- path = None
- if path is None:
- part = {'type': 'text', 'text': '[Image no longer retained]'}
- else:
- data = base64.b64encode(path.read_bytes()).decode()
- part = {**part, 'image_url': {**part['image_url'],
- 'url': f'data:{mimes.get(path.suffix, "image/png")};base64,{data}'}}
- parts.append(part)
- return parts
|