session_ingress_media.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616
  1. """Immutable local-media references for owner-only native admission.
  2. The existing managed document cache supplies profile routing, delivery eligibility
  3. and upload limits. Its flat age-based cleanup skips this retained subdirectory:
  4. native bytes are released by ``release_admission_media`` once their row is
  5. terminal and no live native input or retained API image context still holds them.
  6. """
  7. from collections import Counter
  8. from contextlib import contextmanager
  9. from contextvars import ContextVar
  10. import hashlib
  11. import json
  12. import logging
  13. import os
  14. from pathlib import Path
  15. import re
  16. import sqlite3
  17. import stat
  18. import tempfile
  19. import threading
  20. from hermes_dir_acl import make_user_dir
  21. from hermes_state_runtime import ADMITTED_REQUEST_CONFLICT, RuntimeStoreError
  22. from utils import fsync_directory
  23. logger = logging.getLogger(__name__)
  24. _API_IMAGE_NAME = re.compile(r'api_[0-9a-f]{32}\.(png|jpg|gif|webp)')
  25. _SHA256_NAME = re.compile(r'[0-9a-f]{64}')
  26. # Retained paths a capture published or reused whose admission has not committed or been refused
  27. # yet: no committed row holds them, so ``release_unheld_media`` must treat them as held. The lock
  28. # orders a capture's lease-then-publish against a release's lease-check-then-unlink, per path.
  29. _leases = Counter()
  30. _lease_lock = threading.Lock()
  31. _lease_scope = ContextVar('native_media_lease_scope', default=None)
  32. class _CaptureScope(list):
  33. def __init__(self, authority):
  34. super().__init__()
  35. self.authority = authority
  36. self.all_paths = []
  37. self.abandoned = False
  38. self.write_started = False
  39. def append(self, path):
  40. super().append(path)
  41. self.all_paths.append(path)
  42. @contextmanager
  43. def capture_lease(authority=None):
  44. """Hold every path captured inside this block (capture through admission commit or refusal)
  45. against a concurrent release: an older turn's settlement or cancellation that names the same
  46. content-addressed bytes would otherwise unlink them before the new admission holds them."""
  47. scope = _CaptureScope(authority)
  48. token = _lease_scope.set(scope)
  49. try:
  50. yield scope
  51. finally:
  52. drop_capture_lease()
  53. _lease_scope.reset(token)
  54. def drop_capture_lease(paths=None, *, scope=None):
  55. """End this scope's lease on *paths* (default: all) early, e.g. before a refused admission
  56. releases its own fresh capture, which its lease would otherwise keep from collection."""
  57. held = scope if scope is not None else _lease_scope.get()
  58. if not held:
  59. return
  60. with _lease_lock:
  61. for path in [path for path in held if paths is None or path in paths]:
  62. held.remove(path)
  63. _leases[path] -= 1
  64. if _leases[path] <= 0:
  65. del _leases[path]
  66. async def abandon_capture(scope):
  67. """Mark *scope*'s capture abandoned and release it, once: no row can own these bytes. A no-op for no
  68. scope, a scope without an authority, or one already abandoned (its release ran or is arranged)."""
  69. if scope is None or scope.authority is None or scope.abandoned:
  70. return
  71. scope.abandoned = True
  72. await release_abandoned_capture(scope)
  73. async def release_abandoned_capture(scope):
  74. """Collect every path published by an abandoned worker, including after its caller left."""
  75. if scope.authority is None:
  76. return
  77. drop_capture_lease(scope=scope)
  78. references = [{'path': path, 'sha256': Path(path).parent.name} for path in set(scope.all_paths)]
  79. if references:
  80. import asyncio
  81. await asyncio.to_thread(release_unheld_media, scope.authority.db, references)
  82. def native_inputs_store():
  83. """The store's path as the profile names it, unresolved. The one value ``make_user_dir`` gets:
  84. ``resolve()`` erases a link planted below the Hermes home (``cache/documents`` -> elsewhere),
  85. and an elevated grant on the resolved path would land outside the home with no home fence."""
  86. from gateway.platforms.base import get_document_cache_dir
  87. return get_document_cache_dir() / 'native-inputs'
  88. def native_inputs_root():
  89. """This profile's content-addressed store of retained native inputs (``<root>/<sha256>/<name>``),
  90. resolved: what the identity checks compare against."""
  91. return native_inputs_store().resolve()
  92. _media_root = native_inputs_root
  93. # Public ``prompt.submit`` attachments: a local client stages image bytes in the
  94. # profile image cache (where messaging adapters stage downloads), the authority
  95. # commits them as immutable bytes at admission so no later mutation or cache
  96. # cleanup of the staging file can change what executes.
  97. _ATTACHMENT_MIMES = frozenset({'image/png', 'image/jpeg', 'image/gif', 'image/webp'})
  98. _ATTACHMENT_LIMIT = 10
  99. _IMAGE_EXT = {'image/png': '.png', 'image/jpeg': '.jpg', 'image/gif': '.gif', 'image/webp': '.webp'}
  100. def sniff_image_mime(data):
  101. """The admission image type the bytes really are (png/jpeg/gif/webp), else ``None``."""
  102. if data.startswith(b'\x89PNG\r\n\x1a\n'):
  103. return 'image/png'
  104. if data.startswith(b'\xff\xd8\xff'):
  105. return 'image/jpeg'
  106. if data[:6] in (b'GIF87a', b'GIF89a'):
  107. return 'image/gif'
  108. if data[:4] == b'RIFF' and data[8:12] == b'WEBP':
  109. return 'image/webp'
  110. return None
  111. def admit_attachments(attachments, *, admitted=None):
  112. """Wire ``attachments: [{path, mime}]`` -> committed payload fields (``{}`` when absent).
  113. ``admitted()`` returns the durable admission already holding this request's identity (or
  114. ``None``): a lost-ACK retry whose disposable staging file is gone reconciles against it."""
  115. if attachments is None:
  116. return {}
  117. if (not isinstance(attachments, list) or not attachments or len(attachments) > _ATTACHMENT_LIMIT
  118. or any(not isinstance(item, dict) or set(item) != {'path', 'mime'}
  119. or not isinstance(item['path'], str) or item['mime'] not in _ATTACHMENT_MIMES
  120. for item in attachments)):
  121. raise RuntimeStoreError('invalid_params')
  122. from gateway.platforms.base import get_image_cache_dir
  123. staging = get_image_cache_dir().resolve()
  124. paths = [Path(item['path']) for item in attachments]
  125. if any(not path.is_absolute() or path.resolve().parent != staging for path in paths):
  126. raise RuntimeStoreError('invalid_params')
  127. mimes = [item['mime'] for item in attachments]
  128. row = admitted() if admitted is not None else None
  129. committed = row['payload'].get('attachments_v1') if row is not None else None
  130. try:
  131. hashes = _attachment_hashes(paths)
  132. except (OSError, ValueError) as exc:
  133. raise RuntimeStoreError('invalid_params') from exc
  134. if None in hashes:
  135. if (committed is None or [Path(r['path']).name for r in committed['media']] != [p.name for p in paths]
  136. or committed['media_types'] != mimes):
  137. raise RuntimeStoreError('invalid_params')
  138. if any(digest is not None and digest != reference['sha256']
  139. for digest, reference in zip(hashes, committed['media'])):
  140. raise RuntimeStoreError('admission_conflict', **ADMITTED_REQUEST_CONFLICT)
  141. retry = True
  142. else:
  143. # Retry identity is the committed bytes, not the disposable staging name.
  144. retry = (committed is not None and committed['media_types'] == mimes
  145. and [r['sha256'] for r in committed['media']] == hashes)
  146. if retry:
  147. # A live row still executes these bytes, so they must verify; a terminal row is
  148. # exact-retry evidence by digest only. ``admit_session_input`` still checks the digest.
  149. if row['status'] != 'terminal':
  150. restore_native_media(committed['media'])
  151. return {'attachments_v1': committed}
  152. return {'attachments_v1': {'media': capture_native_media(paths), 'media_types': mimes}}
  153. def ingress_fingerprint(payload, raw=None):
  154. """SHA-256 of a delivery as received: its prepared payload and the provider payload the hook also
  155. sees (``raw_message``: a webhook's parsed, route-script-transformed body; None on relay, whose
  156. wire fields are all in the payload). Transport headers (delivery id, signature, attempt counters,
  157. timestamps) never reach the hook and are not hashed. Media count by bytes: a redelivery
  158. re-captures them under a fresh staging name (``reconcile_native_retry``)."""
  159. envelope = {**payload['native_text_v1']}
  160. envelope.pop('ingress_sha256', None)
  161. if 'media' in envelope:
  162. envelope['media'] = [[media['sha256'], media['size']] for media in envelope['media']]
  163. received = {**payload, 'native_text_v1': envelope}
  164. if raw is not None:
  165. received = {'payload': received, 'raw_message': raw}
  166. canonical = json.dumps(received, sort_keys=True, separators=(',', ':'), default=repr)
  167. return hashlib.sha256(canonical.encode()).hexdigest()
  168. def reconcile_native_retry(db, *, principal_id, session_id, request_id, payload, raw=None):
  169. """A redelivered native message (lost provider ACK, Relay buffer replay) re-downloads its media
  170. under a fresh disposable staging name, so its captured path differs from the committed one.
  171. Retry identity is the committed bytes, not that name: when this exact identity already holds
  172. the same digests and sizes in the same order, the committed references stand in for the fresh
  173. capture (whose now-unowned copy is collected) and the admission digest decides the rest, so
  174. changed bytes or any other changed field still conflict. A delivery committed in
  175. ``pre_gateway_dispatch``-rewritten form resolves to its committed payload when the resend matches
  176. the fingerprint recorded at ingress (``ingress_fingerprint``); a changed resend, or a row
  177. committed before fingerprints, is compared exactly as above."""
  178. fresh = payload.get('native_text_v1', {}).get('media') or []
  179. with db._read_ctx() as conn:
  180. row = conn.execute('SELECT status, payload_json FROM session_admissions WHERE principal_id=? AND '
  181. 'target_session_id=? AND request_id=?', (principal_id, session_id, request_id)).fetchone()
  182. retained = json.loads(row[1]) if row is not None else {}
  183. recorded = retained.get('native_text_v1', {}).get('ingress_sha256')
  184. if recorded is not None and recorded == ingress_fingerprint(payload, raw):
  185. resolved = retained
  186. else:
  187. committed = retained.get('native_text_v1', {}).get('media')
  188. if not fresh or not committed or [(r['sha256'], r['size']) for r in committed] != [
  189. (r['sha256'], r['size']) for r in fresh]:
  190. return payload
  191. resolved = {**payload, 'native_text_v1': {**payload['native_text_v1'], 'media': committed}}
  192. committed = resolved['native_text_v1'].get('media', [])
  193. if committed and row[0] != 'terminal':
  194. # A live row still executes these bytes, so they must verify (a terminal one is digest evidence).
  195. restore_native_media(committed)
  196. kept = {reference['path'] for reference in committed}
  197. unowned = [reference for reference in fresh if reference['path'] not in kept]
  198. if unowned:
  199. drop_capture_lease({reference['path'] for reference in unowned})
  200. release_unheld_media(db, unowned, retain_history=False)
  201. return resolved
  202. def _attachment_hashes(paths):
  203. hashes = []
  204. for path in paths:
  205. try:
  206. # Staging must not provide a second name for a file outside it.
  207. if path.lstat().st_nlink != 1:
  208. raise RuntimeStoreError('invalid_params')
  209. hashes.append(_sha256(path))
  210. except FileNotFoundError:
  211. # One pruned file cannot skip validation of the remaining batch.
  212. hashes.append(None)
  213. return hashes
  214. def _sha256(path):
  215. with _open_regular(path) as source:
  216. return hashlib.file_digest(source, 'sha256').hexdigest()
  217. def restore_attachments(payload):
  218. """Committed attachment fields -> ``MessageEvent`` media kwargs (``{}`` for text-only rows)."""
  219. data = payload.get('attachments_v1')
  220. if not data:
  221. return {}
  222. return {'media_urls': restore_native_media(data['media']), 'media_types': list(data['media_types'])}
  223. def _open_regular(path):
  224. if not path.is_absolute() or path.is_symlink():
  225. raise ValueError('native media must be a local regular file')
  226. fd = os.open(path, os.O_RDONLY | getattr(os, 'O_NONBLOCK', 0) | getattr(os, 'O_NOFOLLOW', 0))
  227. if not stat.S_ISREG(os.fstat(fd).st_mode):
  228. os.close(fd)
  229. raise ValueError('native media must be a regular file')
  230. return os.fdopen(fd, 'rb')
  231. def capture_native_media(paths):
  232. from gateway.platforms.base import get_inbound_media_max_bytes, validate_inbound_media_size
  233. references = []
  234. limit = max(0, get_inbound_media_max_bytes())
  235. # ``gateway.max_inbound_media_bytes`` bounds the whole admission, not each file: with
  236. # per-file caps alone ten attachments could commit ~1.25 GiB of retained bytes per turn.
  237. total = 0
  238. # Only private snapshots belong exclusively to this batch. Published aliases
  239. # can be reused by another capture before either admission is committed.
  240. staged = []
  241. try:
  242. for value in paths:
  243. _capture_file(Path(value), limit, total, references, staged)
  244. total = sum(reference['size'] for reference in references)
  245. for temporary, reference in zip(staged, references):
  246. target = Path(reference['path'])
  247. root = target.parent.parent
  248. if target.parent.resolve() != target.parent:
  249. raise RuntimeStoreError('invalid_params')
  250. make_user_dir(native_inputs_store() / target.parent.name, mode=0o700, exist_ok=True)
  251. with _lease_lock:
  252. scope = _lease_scope.get()
  253. if scope is not None:
  254. scope.append(str(target))
  255. _leases[str(target)] += 1
  256. if target.exists() or target.is_symlink():
  257. restore_native_media([reference])
  258. os.utime(target) # a fresh capture restarts the retention age (sweep_settled_media)
  259. else:
  260. os.replace(temporary, target)
  261. for directory in (target.parent, root, root.parent, root.parent.parent, root.parent.parent.parent):
  262. fsync_directory(directory)
  263. finally:
  264. for temporary in staged:
  265. temporary.unlink(missing_ok=True)
  266. return references
  267. def _capture_file(path, limit, total, references, staged):
  268. from gateway.platforms.base import validate_inbound_media_size
  269. try:
  270. source = _open_regular(path)
  271. except (OSError, ValueError) as exc:
  272. raise RuntimeStoreError('invalid_params') from exc
  273. with source:
  274. root = _media_root()
  275. if root.resolve() != root:
  276. raise RuntimeStoreError('invalid_params')
  277. make_user_dir(native_inputs_store(), mode=0o700, parents=True, exist_ok=True)
  278. fd, name = tempfile.mkstemp(prefix='.capture-', dir=root)
  279. temporary = Path(name)
  280. staged.append(temporary)
  281. with os.fdopen(fd, 'wb') as output:
  282. digest, size = hashlib.sha256(), 0
  283. while chunk := source.read(1024 * 1024):
  284. size += len(chunk)
  285. try:
  286. validate_inbound_media_size(total + size, max_bytes=limit)
  287. except ValueError as exc:
  288. raise RuntimeStoreError('invalid_params') from exc
  289. digest.update(chunk)
  290. output.write(chunk)
  291. output.flush()
  292. os.fsync(output.fileno())
  293. target = root / digest.hexdigest() / path.name
  294. if target.parent.resolve() != target.parent:
  295. raise RuntimeStoreError('invalid_params')
  296. reference = {'path': str(target), 'sha256': digest.hexdigest(), 'size': size}
  297. if target.exists() or target.is_symlink():
  298. restore_native_media([reference])
  299. references.append(reference)
  300. def validate_media_batch_size(sizes):
  301. """Preflight validated manifest sizes before materializing any batch member."""
  302. from gateway.platforms.base import get_inbound_media_max_bytes, validate_inbound_media_size
  303. try:
  304. validate_inbound_media_size(sum(sizes), max_bytes=max(0, get_inbound_media_max_bytes()))
  305. except ValueError as exc:
  306. raise RuntimeStoreError('invalid_params') from exc
  307. # A hosted room's non-image document is retained under ``native-inputs/<sha256>/<name>`` like any
  308. # other input, but rides in the committed prompt as a path line (``session_hosted_attachments``),
  309. # not as a structured reference. Same literal the hosted payload builders append.
  310. HOSTED_DOCUMENT_LINE = '\n[Shared attachment] file: '
  311. _HOSTED_DOCUMENT_RE = re.compile(re.escape(HOSTED_DOCUMENT_LINE) + r'([^\n]+)\n')
  312. def hosted_document_references(request_id, payload):
  313. """Retained document references a hosted admission's prompt names (``[]`` for other rows).
  314. Only content-addressed names under this profile's retained root count, and never an API
  315. image name: those are held by ``api_turn_v1`` in every status, so a member's prompt text
  316. can name one without gaining deletion authority over it."""
  317. if not str(request_id).startswith('hosted:') or not isinstance(payload.get('text'), str):
  318. return []
  319. root = _media_root()
  320. references = []
  321. for value in _HOSTED_DOCUMENT_RE.findall(payload['text']):
  322. path = Path(value)
  323. if (path.is_absolute() and path.parent.parent == root and _SHA256_NAME.fullmatch(path.parent.name)
  324. and not _API_IMAGE_NAME.fullmatch(path.name)):
  325. references.append({'path': value, 'sha256': path.parent.name})
  326. return references
  327. def admission_media_references(payload):
  328. """Native references eligible as deletion candidates after terminal settlement."""
  329. return list(payload.get('attachments_v1', {}).get('media', ())) + list(
  330. payload.get('native_text_v1', {}).get('media', ()))
  331. def _held_media_paths(conn):
  332. # Project only references, not potentially large inline-image/history payloads; a live hosted
  333. # row's prompt is projected too, because its documents are held by text, not by a reference.
  334. rows = conn.execute("""SELECT status, request_id, json_extract(payload_json,
  335. '$.attachments_v1.media', '$.native_text_v1.media', '$.api_turn_v1.media'),
  336. CASE WHEN status!='terminal' AND request_id LIKE 'hosted:%'
  337. THEN json_extract(payload_json, '$.text') END
  338. FROM session_admissions WHERE status!='terminal'
  339. OR json_type(payload_json, '$.api_turn_v1.media') IS NOT NULL""").fetchall()
  340. held = set()
  341. for status, request_id, encoded, text in rows:
  342. attachments, native, api = json.loads(encoded)
  343. # API images remain canonical history context after the turn completes.
  344. references = list(api or ())
  345. if status != 'terminal':
  346. references.extend(attachments or ())
  347. references.extend(native or ())
  348. references.extend(hosted_document_references(request_id, {'text': text}))
  349. held.update(reference['path'] for reference in references)
  350. return held
  351. def _file_identity(path):
  352. saved = path.stat(follow_symlinks=False)
  353. if not stat.S_ISREG(saved.st_mode) or not saved.st_ino:
  354. raise ValueError('uncertain native file identity')
  355. return saved.st_dev, saved.st_ino
  356. def _held_file_identities(paths, root):
  357. identities = set()
  358. try:
  359. for value in paths:
  360. path = Path(value)
  361. if path.parent.parent != root or path.parent.resolve() != path.parent:
  362. return None
  363. identities.add(_file_identity(path))
  364. except (OSError, ValueError):
  365. return None
  366. return identities
  367. def release_admission_media(db, admission_id):
  368. """Delete eligible terminal native bytes unless another retained input holds them.
  369. Terminal rows are exact-retry evidence by digest only; their bytes are not
  370. replayed. Rows that are not terminal (queued, started, unknown) may still
  371. execute, so any path they reference stays on disk; API image references stay
  372. on disk in every status because they remain history context after settlement,
  373. and are holders only (see ``collect_unheld_api_images``). Different regular
  374. files can collect independently; physical aliases and uncertain stat results
  375. retain conservatively. A shared hardlink can consequently retain an extra old alias.
  376. """
  377. from hermes_state_runtime import get_session_admission
  378. row = get_session_admission(db, admission_id=admission_id)
  379. if row is None or row['status'] != 'terminal':
  380. return 0
  381. api_turn = row['payload'].get('api_turn_v1', {})
  382. if api_turn.get('media') or isinstance(api_turn.get('history'), list):
  383. # Settled: inline base64 of committed images and the client history are redundant row weight.
  384. from gateway.session_api_media import compact_settled_api_payloads
  385. compact_settled_api_payloads(db, admission_id)
  386. # Native and hosted inputs are named in the prompt the transcript keeps ("It is saved at: <path>"),
  387. # so they are history context like an API image: they go once no live row and no transcript row
  388. # names them, never while a follow-up turn can still be told to read them. Session deletion
  389. # collects them through ``retire_media``; ``sweep_settled_media`` bounds the rest at main's 24 h.
  390. released = release_unheld_media(db, admission_media_references(row['payload']))
  391. released += release_unheld_media(db, hosted_document_references(row['request_id'], row['payload']))
  392. return released + sweep_settled_media(db)
  393. _SWEEP_INTERVAL_S = 3600
  394. _last_sweep = {}
  395. def sweep_settled_media(db, *, now=None):
  396. """At most hourly per store: collect settled native/attachment inputs older than
  397. ``MEDIA_CACHE_MAX_AGE_HOURS`` whatever the transcript says, as main's age-based document cache
  398. cleanup did for the staging copy the note used to name. A live admission or an in-flight capture
  399. still holds its bytes; API images and hosted documents keep their own owners."""
  400. import time
  401. from agent.provider_media import MEDIA_CACHE_MAX_AGE_HOURS
  402. now = time.time() if now is None else now
  403. if now - _last_sweep.get(db.db_path, 0) < _SWEEP_INTERVAL_S:
  404. return 0
  405. _last_sweep[db.db_path] = now
  406. with db._read_ctx() as conn:
  407. rows = conn.execute("""SELECT json_extract(payload_json, '$.attachments_v1.media', '$.native_text_v1.media')
  408. FROM session_admissions WHERE status='terminal' AND (json_type(payload_json, '$.attachments_v1.media')
  409. IS NOT NULL OR json_type(payload_json, '$.native_text_v1.media') IS NOT NULL)""").fetchall()
  410. cutoff = now - MEDIA_CACHE_MAX_AGE_HOURS * 3600
  411. stale = {}
  412. for (encoded,) in rows:
  413. attachments, native = json.loads(encoded)
  414. for reference in (attachments or []) + (native or []):
  415. try:
  416. if Path(reference['path']).stat(follow_symlinks=False).st_mtime < cutoff:
  417. stale[reference['path']] = reference
  418. except OSError:
  419. continue
  420. return release_unheld_media(db, list(stale.values()), retain_history=False) if stale else 0
  421. def collect_unheld_api_images(db):
  422. """Collect retained API images no admission holds any more: deleting a chat retires its
  423. admissions and erases their references, so those bytes have no owner left. Only the exact
  424. name ``commit_api_images`` gives (``api_<sha256[:32]><ext>`` under its own digest) is a
  425. candidate; retained hosted documents are held by prompt text, never by a media reference.
  426. Candidates are enumerated from the ACTIVE home's media root while ownership is decided by
  427. ``db``: a sweep whose database belongs to another profile would see every image here as
  428. unowned, so it collects nothing."""
  429. from hermes_constants import get_hermes_home
  430. if Path(db.db_path).resolve().parent != get_hermes_home().resolve():
  431. logger.warning(
  432. 'Skipped retained API image collection: %s is not the active profile store', db.db_path)
  433. return 0
  434. root = _media_root()
  435. try:
  436. return release_unheld_media(db, [
  437. {'path': str(path), 'sha256': path.parent.name} for path in (root.glob('*/api_*') if root.is_dir() else ())
  438. if _API_IMAGE_NAME.fullmatch(path.name) and path.name[4:36] == path.parent.name[:32]])
  439. except sqlite3.DatabaseError:
  440. # Callers run after a committed delete or during startup recovery: neither may fail on it.
  441. # ``DatabaseError`` also covers a corrupt store, not only ``OperationalError`` (locked, limits).
  442. logger.exception('API image collection failed; files stay for the next sweep')
  443. return 0
  444. def _history_mentions(conn, references, after_id=0):
  445. """Candidate paths any transcript row (id > after_id) mentions raw or JSON-escaped, in ONE
  446. messages pass. Rows are prefiltered in SQL on the literal text before each candidate's digest
  447. directory, so only rows that could name a candidate reach Python's exact substring test."""
  448. # Matched by the content-addressed tail ``<sha256>/<name>`` with either separator, so every
  449. # spelling a note can carry (the host path, a sandbox backend's ``to_agent_visible_cache_path``)
  450. # counts as a mention.
  451. needles = {}
  452. for reference in references:
  453. raw = reference['path']
  454. for tail in (reference['sha256'] + sep + Path(raw).name for sep in ('/', '\\')):
  455. needles[tail] = needles[json.dumps(tail)[1:-1]] = raw
  456. prefixes = sorted({reference['sha256'] for reference in references})
  457. # The digests are ONE bound JSON array, never one ``instr`` term each: the statement's size
  458. # must not grow with the candidate count (SQLite's expression-depth limit fails at ~1000 terms).
  459. query = ('WITH digests(value) AS MATERIALIZED (SELECT value FROM json_each(?)) SELECT content FROM messages '
  460. 'WHERE EXISTS (SELECT 1 FROM digests WHERE instr(content, digests.value))' + (' AND id>?' if after_id else ''))
  461. mentioned = set()
  462. for (content,) in conn.execute(query, (json.dumps(prefixes), *((after_id,) if after_id else ()))):
  463. text = content.decode('utf-8', 'replace') if isinstance(content, bytes) else str(content)
  464. mentioned.update(raw for needle, raw in needles.items() if needle in text)
  465. return mentioned
  466. def _collectable(references, root):
  467. return [reference for reference in references if Path(reference['path']).parent.parent == root
  468. and Path(reference['path']).parent.name == reference['sha256'] and Path(reference['path']).exists()]
  469. def release_unheld_media(db, references, *, retain_history=True):
  470. """Delete unowned references, preserving accepted admission and retained transcript owners.
  471. The history owner check is one read-snapshot pass over messages, outside the write lock; under
  472. the lock only rows inserted after that snapshot (an indexed id range) are re-checked. A new
  473. transcript mention of a retained image comes from an admission holding it (re-checked under the
  474. lock) or from a copied row (branch/compress insert, caught by the range)."""
  475. root = _media_root()
  476. references = _collectable(references or (), root)
  477. if not references:
  478. return 0
  479. seen, last_id = set(), 0
  480. if retain_history:
  481. with db._read_ctx() as conn:
  482. conn.execute('BEGIN') # one snapshot: the id watermark and the scan agree
  483. try:
  484. last_id = conn.execute('SELECT coalesce(max(id), 0) FROM messages').fetchone()[0]
  485. seen = _history_mentions(conn, references)
  486. finally:
  487. if conn.in_transaction:
  488. conn.execute('ROLLBACK')
  489. references = [reference for reference in references if reference['path'] not in seen]
  490. if not references:
  491. return 0
  492. def collect(conn):
  493. held = _held_media_paths(conn)
  494. identities = _held_file_identities(held, root)
  495. if identities is None:
  496. return 0
  497. if retain_history:
  498. held |= _history_mentions(conn, references, after_id=last_id)
  499. released = 0
  500. for reference in references:
  501. path = Path(reference['path'])
  502. if reference['path'] in held:
  503. continue
  504. try:
  505. with _lease_lock:
  506. if (reference['path'] in _leases or path.parent.resolve() != path.parent
  507. or _file_identity(path) in identities):
  508. continue
  509. path.unlink()
  510. released += 1
  511. path.parent.rmdir()
  512. except (OSError, ValueError):
  513. continue
  514. return released
  515. return db._execute_write(collect)
  516. def restore_native_media(references):
  517. if not references:
  518. return []
  519. root = _media_root()
  520. paths = []
  521. try:
  522. for reference in references:
  523. if set(reference) != {'path', 'sha256', 'size'}:
  524. raise ValueError('invalid media reference')
  525. path = Path(reference['path'])
  526. if (path.parent.parent != root or path.resolve() != path
  527. or path.parent.name != reference['sha256']):
  528. raise ValueError('foreign media reference')
  529. with _open_regular(path) as source:
  530. if os.fstat(source.fileno()).st_size != reference['size']:
  531. raise ValueError('changed media size')
  532. if hashlib.file_digest(source, 'sha256').hexdigest() != reference['sha256']:
  533. raise ValueError('changed media bytes')
  534. paths.append(str(path))
  535. except (OSError, ValueError, TypeError, KeyError) as exc:
  536. raise RuntimeStoreError('storage_unavailable') from exc
  537. return paths