hermes_state_runtime_workers.py 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112
  1. """Worker exclusion follows the logical conversation and its physical transcript lineage."""
  2. import json
  3. def runtime_lineage(conn, session_id):
  4. from hermes_state_local import POLICY_PREFIX
  5. from hermes_state_local_lineage import local_physical_target
  6. from hermes_state_sessions import _expand_compression_lineage_ids
  7. from hermes_state_compression import _CHAIN_CAP
  8. from hermes_state_runtime import RuntimeStoreError
  9. targets = {session_id, local_physical_target(conn, session_id)}
  10. row = conn.execute('SELECT chat_id FROM sessions WHERE id=?', (session_id,)).fetchone()
  11. if row and row['chat_id']:
  12. saved = conn.execute('SELECT value FROM state_meta WHERE key=?', (POLICY_PREFIX + row['chat_id'],)).fetchone()
  13. if saved is not None and session_id in json.loads(saved[0]).get('lineage', [row['chat_id']]):
  14. targets.update({row['chat_id'], local_physical_target(conn, row['chat_id'])})
  15. frontier = list(targets)
  16. for _ in range(_CHAIN_CAP):
  17. frontier = [sid for sid in _expand_compression_lineage_ids(conn, frontier) if sid not in targets]
  18. if not frontier:
  19. return targets
  20. targets.update(frontier)
  21. raise RuntimeStoreError('storage_unavailable')
  22. def worker_states(conn, session_id):
  23. # One indexed probe for the whole lineage (json_each: no bound-parameter limit on long chains).
  24. return {row[0] for row in conn.execute(
  25. "SELECT DISTINCT status FROM worker_executions WHERE session_id IN (SELECT value FROM json_each(?)) "
  26. "AND status IN ('registered','running','unknown')", (json.dumps(sorted(runtime_lineage(conn, session_id))),))}
  27. def discard_orphan_workers_on_reset(conn, session_ids):
  28. """An explicit reset revokes unlinked unknown compute; it never replays its side effects.
  29. Admission-linked work still requires its own Discard receipt. A running/registered worker
  30. remains a hard reset refusal, and transaction rollback preserves all fences on refusal.
  31. """
  32. for sid in session_ids:
  33. orphans = conn.execute("SELECT execution_id FROM worker_executions WHERE session_id=? AND status='unknown' "
  34. "AND execution_id NOT LIKE 'admission-worker:%'", (sid,)).fetchall()
  35. for (execution_id,) in orphans:
  36. conn.execute("UPDATE worker_executions SET status='terminal' WHERE execution_id=?", (execution_id,))
  37. compact_terminal_receipts(conn, execution_id)
  38. RETIRED_RECEIPT = {'retired': True}
  39. # A delegated child's execution id is its parent's plus this and a random suffix, so every
  40. # descendant of an execution is one prefix probe away (gateway/session_worker_child.py).
  41. CHILD_SEPARATOR = '/child:'
  42. def retire_child_executions(conn, execution_id):
  43. """A child never outlives the execution that reserved it: every live descendant goes terminal
  44. with it, its read receipts shrink and its open session row is ended. A late child write is
  45. then refused ``stale_generation`` like any write after retirement."""
  46. _retire_children(conn, conn.execute("SELECT execution_id,session_id FROM worker_executions "
  47. "WHERE execution_id>=? AND execution_id<? AND status!='terminal'",
  48. descendant_range(execution_id)).fetchall())
  49. def descendant_range(execution_id):
  50. """``(low, high)`` bounds every descendant id of ``execution_id`` on the primary key index
  51. (';' sorts right after the separator's ':'), so a no-child finish costs one index probe."""
  52. return execution_id + CHILD_SEPARATOR, execution_id + CHILD_SEPARATOR[:-1] + ';'
  53. def retire_stale_children(conn, epoch):
  54. """Owner restart: a child dies with its worker, which dies with its owner, so it is never
  55. adoptable; only its root execution can be recovered as ``unknown``."""
  56. _retire_children(conn, conn.execute("SELECT execution_id,session_id FROM worker_executions WHERE kind='child' "
  57. "AND status!='terminal' AND owner_epoch!=?", (epoch,)).fetchall())
  58. def _retire_children(conn, rows):
  59. for child, _session_id in rows:
  60. conn.execute("UPDATE worker_executions SET status='terminal' WHERE execution_id=?", (child,))
  61. _compact(conn, child)
  62. _end_child_row(conn, child)
  63. def _end_child_row(conn, execution_id):
  64. import time
  65. conn.execute("UPDATE sessions SET ended_at=?,end_reason='agent_close' WHERE ended_at IS NULL "
  66. "AND id=(SELECT session_id FROM worker_executions WHERE execution_id=?)", (time.time(), execution_id))
  67. def compact_terminal_receipts(conn, execution_id):
  68. """Every terminal transition calls this, so it also retires the execution's delegated children.
  69. Once an execution is terminal its read receipts are dead weight: a ``compression.history``
  70. read stores the whole transcript and a context read a full session row, so keeping them leaves
  71. one full-history copy per managed turn and ``state.db`` grows quadratically. They shrink to a
  72. marker in the closing transaction (an exact late replay is refused ``stale_generation``, as
  73. after retirement); ``payload_digest`` stays, so a late duplicate with other content still
  74. conflicts. Mutation acknowledgements (counts, row annotations, assignments) stay replayable."""
  75. retire_child_executions(conn, execution_id)
  76. _compact(conn, execution_id)
  77. if CHILD_SEPARATOR in execution_id:
  78. # A child that finished itself (an early close: failed constructor, refused sibling) never
  79. # reaches _retire_children, which only selects live children: its row ends here.
  80. _end_child_row(conn, execution_id)
  81. def _compact(conn, execution_id):
  82. conn.execute("UPDATE worker_receipts SET result_json=? WHERE execution_id=? AND json_valid(result_json) "
  83. "AND (json_type(result_json, '$.messages') IS NOT NULL "
  84. "OR json_type(result_json, '$.session') IS NOT NULL)",
  85. (json.dumps(RETIRED_RECEIPT), execution_id))
  86. def retired_receipt(result_json):
  87. return json.loads(result_json) == RETIRED_RECEIPT