hermes_state_media.py 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869
  1. """Durable media retirement candidates survive transcript deletion and failed cleanup."""
  2. import hashlib
  3. import json
  4. from hermes_state_keys import prefix_range
  5. PREFIX = 'gateway.media.retired.v1.'
  6. SPILLOVER_PREFIX = 'gateway.spillover.retired.v1.'
  7. def retire_media(conn, payload, request_id=''):
  8. from gateway.session_ingress_media import admission_media_references, hosted_document_references
  9. references = [*admission_media_references(payload), *hosted_document_references(request_id, payload),
  10. *payload.get('api_turn_v1', {}).get('media', [])]
  11. for reference in references:
  12. key = PREFIX + hashlib.sha256(reference['path'].encode()).hexdigest()
  13. conn.execute('INSERT OR IGNORE INTO state_meta(key,value) VALUES(?,?)',
  14. (key, json.dumps(reference)))
  15. def retire_spillover(conn, session_ids):
  16. """Record the spillover archives the doomed sessions' messages point at, inside the delete
  17. transaction (the pointers vanish with the rows), as durable candidates (#126351)."""
  18. from hermes_state_common import _id_chunks, _placeholders
  19. from tools.tool_result_storage import SPILLOVER_SQL_FILTER, spillover_names_in_text
  20. for chunk in _id_chunks(list(session_ids)):
  21. for content, api_content in conn.execute(
  22. f"SELECT content, api_content FROM messages WHERE session_id IN ({_placeholders(chunk)})"
  23. f" AND {SPILLOVER_SQL_FILTER}", chunk).fetchall():
  24. conn.executemany('INSERT OR IGNORE INTO state_meta(key,value) VALUES(?,?)',
  25. [(SPILLOVER_PREFIX + n, json.dumps(n)) for n in spillover_names_in_text(content, api_content)])
  26. def collect_retired_spillover(db):
  27. """Unlink retired archives no surviving message references (a branch/compression sibling sharing
  28. the pointer keeps it until it dies too). Unknown reference set: keep files and candidates."""
  29. from tools.tool_result_storage import referenced_spillover_names
  30. with db._read_ctx() as conn:
  31. rows = conn.execute('SELECT key,value FROM state_meta WHERE key>=? AND key<?',
  32. prefix_range(SPILLOVER_PREFIX)).fetchall()
  33. spill_dir = db.db_path.parent / 'cache' / 'spillover'
  34. if not rows or (referenced := referenced_spillover_names(spill_dir)) is None:
  35. return
  36. done = []
  37. for key, raw in rows:
  38. try:
  39. if (name := json.loads(raw)) not in referenced:
  40. (spill_dir / name).unlink(missing_ok=True)
  41. except OSError:
  42. continue # stays a candidate for the next delete/prune pass
  43. done.append((key,))
  44. db._execute_write(lambda conn: conn.executemany('DELETE FROM state_meta WHERE key=?', done))
  45. def collect_retired_media(db):
  46. """Release retired candidates no admission or retained transcript row owns; drop collected keys.
  47. Synchronous SQLite plus one messages pass: async callers must run it off the event loop."""
  48. from gateway.session_ingress_media import release_unheld_media
  49. collect_retired_spillover(db)
  50. with db._read_ctx() as conn:
  51. # Primary-key range, not LIKE: the common no-candidate case costs one index probe.
  52. rows = conn.execute('SELECT key,value FROM state_meta WHERE key>=? AND key<?',
  53. prefix_range(PREFIX)).fetchall()
  54. if rows:
  55. release_unheld_media(db, [json.loads(row['value']) for row in rows])
  56. from pathlib import Path
  57. gone = [(row['key'],) for row in rows if not Path(json.loads(row['value'])['path']).exists()]
  58. if gone:
  59. db._execute_write(lambda conn: conn.executemany('DELETE FROM state_meta WHERE key=?', gone))