session_schema_history.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276
  1. """Physical column history of the state.db tables the page-level salvage lane maps.
  2. ``_reconcile_columns()`` (hermes_state_schema.py) upgrades a live store with
  3. ``ALTER TABLE ADD COLUMN``, which APPENDS each newly declared column. A store
  4. therefore keeps its columns in the order they were *added*, which diverges
  5. from the order ``SCHEMA_SQL`` declares them whenever a later release inserts a
  6. column mid-definition (#101409). SQLite never rewrites existing rows on ADD
  7. COLUMN either, so a row written when the table had ``k`` columns carries
  8. exactly the first ``k`` physical names.
  9. A ``lost_and_found`` record carries no schema, so the salvage lane has to
  10. recognise the physical layout that produced it. This module is the single
  11. source of truth for that: the declared order of every table as first shipped
  12. (``base``) and, per release that changed it, the edits made to the
  13. declaration (``events``). Replaying the events reproduces every historical
  14. declared order, and every physical layout a real store can have is some
  15. prefix chain over those snapshots (created at snapshot ``i``, then each later
  16. upgrade appending the columns it had not seen yet).
  17. ``tests/hermes_cli/test_session_schema_history.py`` asserts the replay ends
  18. at the current ``SCHEMA_SQL``. When that test fails you added or moved a
  19. column: append an event at the END, labelled with the next sequence number,
  20. describing the edit (``("+", column, after)`` / ``("-", column)``) — do NOT
  21. rewrite or reorder older events, real stores were shaped by them.
  22. """
  23. from __future__ import annotations
  24. from dataclasses import dataclass
  25. from typing import Callable, Iterator, Optional, Sequence
  26. Edit = tuple # ("+", column, previous_column_or_None) | ("-", column)
  27. @dataclass(frozen=True)
  28. class _TableHistory:
  29. base: tuple[str, ...]
  30. # (label, edits) in replay order. Labels are "<seq> <commit time UTC>
  31. # <sha>" of the main commit that shipped the edit; the sequence number
  32. # is what the ordering test checks, so an event appended out of order
  33. # (or inserted mid-list "to keep dates sorted") fails loudly.
  34. events: tuple[tuple[str, tuple[Edit, ...]], ...]
  35. def _apply(columns: list[str], edits: Sequence[Edit]) -> None:
  36. for edit in edits:
  37. if edit[0] == "-":
  38. columns.remove(edit[1])
  39. continue
  40. _, column, after = edit
  41. columns.insert(0 if after is None else columns.index(after) + 1, column)
  42. def declared_snapshots(table: str) -> list[tuple[str, ...]]:
  43. """Every declared column order the table has shipped with, oldest first."""
  44. history = SCHEMA_HISTORY[table]
  45. columns = list(history.base)
  46. snapshots = [tuple(columns)]
  47. for _label, edits in history.events:
  48. _apply(columns, edits)
  49. snapshots.append(tuple(columns))
  50. return snapshots
  51. def current_declared_columns(table: str) -> tuple[str, ...]:
  52. return declared_snapshots(table)[-1]
  53. def reachable_physical_layouts(
  54. table: str,
  55. accept: Optional[Callable[[tuple[str, ...], int], bool]] = None,
  56. ) -> Iterator[tuple[str, ...]]:
  57. """Yield every physical layout an upgraded store of ``table`` can have.
  58. A store created at snapshot ``i`` and later opened by releases
  59. ``j1 < j2 < ...`` appends, at each step, the snapshot's columns it does
  60. not have yet. Columns declared and later removed stay physically present
  61. (``_reconcile_columns`` never drops), so they appear in layouts too.
  62. ``accept(layout, first_new_index)`` lets the caller prune: it is called
  63. for every new layout with the index where it starts to differ from its
  64. parent, and a False return stops extending that branch. The graph is
  65. large without pruning (~230k sessions layouts) but salvage always
  66. prunes against real cells, which keeps it in the low thousands.
  67. """
  68. snapshots = declared_snapshots(table)
  69. seen: dict[tuple[str, ...], int] = {}
  70. frontier: list[tuple[tuple[str, ...], int]] = []
  71. for index, snapshot in enumerate(snapshots):
  72. if snapshot in seen:
  73. continue
  74. if accept is not None and not accept(snapshot, 0):
  75. continue
  76. seen[snapshot] = index
  77. frontier.append((snapshot, index))
  78. yield snapshot
  79. while frontier:
  80. next_frontier: list[tuple[tuple[str, ...], int]] = []
  81. for layout, index in frontier:
  82. have = set(layout)
  83. for later in range(index + 1, len(snapshots)):
  84. added = tuple(c for c in snapshots[later] if c not in have)
  85. if not added:
  86. continue
  87. candidate = layout + added
  88. if candidate in seen and seen[candidate] <= later:
  89. continue
  90. if accept is not None and not accept(candidate, len(layout)):
  91. continue
  92. seen[candidate] = later
  93. next_frontier.append((candidate, later))
  94. yield candidate
  95. frontier = next_frontier
  96. SCHEMA_HISTORY: dict[str, _TableHistory] = {
  97. "sessions": _TableHistory(
  98. base=(
  99. 'id', 'source', 'user_id', 'model', 'model_config',
  100. 'system_prompt', 'parent_session_id', 'started_at', 'ended_at',
  101. 'end_reason', 'message_count', 'tool_call_count', 'input_tokens',
  102. 'output_tokens',
  103. ),
  104. events=(
  105. ('01 2026-03-08T23:32Z c5e8166c8b', (('+', 'title', 'output_tokens'),)),
  106. ('02 2026-03-17T10:44Z d417ba2a48', (
  107. ('+', 'cache_read_tokens', 'output_tokens'),
  108. ('+', 'cache_write_tokens', 'cache_read_tokens'),
  109. ('+', 'reasoning_tokens', 'cache_write_tokens'),
  110. ('+', 'billing_provider', 'reasoning_tokens'),
  111. ('+', 'billing_base_url', 'billing_provider'),
  112. ('+', 'billing_mode', 'billing_base_url'),
  113. ('+', 'estimated_cost_usd', 'billing_mode'),
  114. ('+', 'actual_cost_usd', 'estimated_cost_usd'),
  115. ('+', 'cost_status', 'actual_cost_usd'),
  116. ('+', 'cost_source', 'cost_status'),
  117. ('+', 'pricing_version', 'cost_source'),
  118. )),
  119. ('03 2026-04-22T12:51Z 5fb143169b', (('+', 'api_call_count', 'title'),)),
  120. ('04 2026-05-10T20:06Z 878611a79d', (
  121. ('+', 'handoff_pending', 'api_call_count'),
  122. ('+', 'handoff_platform', 'handoff_pending'),
  123. )),
  124. ('05 2026-05-10T20:06Z 00ce5f04d9', (
  125. ('-', 'handoff_pending'),
  126. ('+', 'handoff_state', 'api_call_count'),
  127. ('+', 'handoff_error', 'handoff_platform'),
  128. )),
  129. ('06 2026-05-31T22:46Z 51c68d4ab1', (('+', 'cwd', 'reasoning_tokens'),)),
  130. ('07 2026-06-01T08:22Z 3e59be0c41', (('+', 'rewind_count', 'handoff_error'),)),
  131. ('08 2026-06-02T01:41Z 85b65e29f0', (('+', 'archived', 'rewind_count'),)),
  132. ('09 2026-06-25T22:49Z ffa3d3c811', (
  133. ('+', 'git_branch', 'cwd'),
  134. ('+', 'git_repo_root', 'git_branch'),
  135. )),
  136. ('10 2026-06-28T22:10Z 86e64900b9', (
  137. ('+', 'session_key', 'user_id'),
  138. ('+', 'chat_id', 'session_key'),
  139. ('+', 'chat_type', 'chat_id'),
  140. ('+', 'thread_id', 'chat_type'),
  141. )),
  142. ('11 2026-06-30T08:06Z f2ccb2859f', (
  143. ('+', 'compression_failure_cooldown_until', 'handoff_error'),
  144. ('+', 'compression_failure_error', 'compression_failure_cooldown_until'),
  145. )),
  146. ('12 2026-07-05T21:01Z 747386ecfa', (
  147. ('+', 'display_name', 'thread_id'),
  148. ('+', 'origin_json', 'display_name'),
  149. ('+', 'expiry_finalized', 'origin_json'),
  150. )),
  151. ('13 2026-07-13T20:49Z af7dceaf77', (
  152. ('+', 'compression_fallback_streak', 'compression_failure_error'),
  153. )),
  154. ('14 2026-07-15T16:50Z e8b7ce8c19', (('+', 'profile_name', 'compression_fallback_streak'),)),
  155. ('15 2026-07-23T15:08Z ec5835ab8b', (
  156. ('+', 'compression_ineffective_count', 'compression_fallback_streak'),
  157. )),
  158. ('16 2026-07-24T16:15Z 951d606730', (('+', 'pinned', 'archived'),)),
  159. ('17 2026-07-27T19:14Z cfb206fe2e', (
  160. ('+', 'last_activity_at', 'title'),
  161. ('+', 'last_activity_description', 'last_activity_at'),
  162. ('+', 'last_activity_provenance', 'last_activity_description'),
  163. )),
  164. ('18 2026-07-27T19:22Z 5646fed97e', (
  165. ('-', 'last_activity_at'),
  166. ('-', 'last_activity_description'),
  167. ('-', 'last_activity_provenance'),
  168. )),
  169. ('19 2026-08-02T23:16Z c2088efe9e', (
  170. ('+', 'last_activity_at', 'title'),
  171. ('+', 'last_activity_description', 'last_activity_at'),
  172. ('+', 'last_activity_provenance', 'last_activity_description'),
  173. )),
  174. ('20 2026-08-03T15:07Z 7d066c3c56', (('+', 'system_prompt_hash', 'system_prompt'),)),
  175. ('21 2026-08-04T19:20Z d98287fe3c', (('+', 'last_read_at', 'pinned'),)),
  176. ('22 2026-08-08T22:17Z fe42097865', (('+', 'title_source', 'title'),)),
  177. ('23 2026-08-15T07:31Z fbaea9bddc', (('+', 'hidden', 'pinned'),)),
  178. ('24 2026-08-15T07:33Z e89532d97e', (('+', 'git_metadata_generation', 'git_repo_root'),)),
  179. ('25 2026-09-02T11:14Z 238b6c1ab9', (
  180. ('+', 'compression_recovery_deadline', 'compression_ineffective_count'),
  181. )),
  182. ('26 2026-09-02T14:22Z 8e4366d358', (('+', 'tool_names', 'last_read_at'),)),
  183. ('27 2026-09-19T00:10Z 922a0c3c87', (('+', 'transport_profile', 'profile_name'),)),
  184. ('28 2026-09-25T23:25Z 2941aadffa', (('+', 'compression_overload_streak', 'compression_recovery_deadline'),)),
  185. ('29 2026-09-27T00:29Z d75f29934b', (('+', 'created_source', 'source'),)),
  186. ('30 2026-09-28T00:00Z #117713', (('+', 'auto_archived', 'archived'),)),
  187. # Runtime-authority columns (unified gateway runtime, PR #106742): appended after main's
  188. # 27-30 because they ship after them on main.
  189. ('31 2026-09-08T18:45Z 0695b12b4c', (
  190. ('+', 'runtime_revision', 'id'),
  191. ('+', 'runtime_generation', 'runtime_revision'),
  192. )),
  193. # GW-18 (package 10, #140968): write-side revision a transcript-derived cache keys on.
  194. ('32 2026-10-18T10:11Z 1fc1999769', (('+', 'transcript_revision', 'tool_names'),)),
  195. ),
  196. ),
  197. "messages": _TableHistory(
  198. base=(
  199. 'id', 'session_id', 'role', 'content', 'tool_call_id',
  200. 'tool_calls', 'tool_name', 'timestamp', 'token_count',
  201. ),
  202. events=(
  203. ('01 2026-02-21T08:05Z b33ed9176f', (('+', 'finish_reason', 'token_count'),)),
  204. ('02 2026-03-25T16:47Z 42fec19151', (
  205. ('+', 'reasoning', 'finish_reason'),
  206. ('+', 'reasoning_details', 'reasoning'),
  207. ('+', 'codex_reasoning_items', 'reasoning_details'),
  208. )),
  209. ('03 2026-04-22T11:31Z a7d78d3bfd', (('+', 'reasoning_content', 'reasoning'),)),
  210. ('04 2026-04-26T01:22Z 81e01f6ee9', (('+', 'codex_message_items', 'codex_reasoning_items'),)),
  211. ('05 2026-05-20T20:00Z 31a0100104', (('+', 'platform_message_id', 'codex_message_items'),)),
  212. ('06 2026-05-23T08:33Z 4a91e36495', (('+', 'observed', 'platform_message_id'),)),
  213. ('07 2026-06-01T08:22Z 3e59be0c41', (('+', 'active', 'observed'),)),
  214. ('08 2026-06-11T03:45Z aaccaada28', (('+', 'anthropic_content_blocks', 'reasoning_details'),)),
  215. ('09 2026-06-11T03:45Z efcbbde48c', (('-', 'anthropic_content_blocks'),)),
  216. ('10 2026-06-20T17:57Z 854d75723f', (('+', 'compacted', 'active'),)),
  217. ('11 2026-07-11T12:41Z a0a6cd80f5', (('+', 'effect_disposition', 'tool_name'),)),
  218. ('12 2026-07-19T02:55Z 7b3dcee928', (('+', 'api_content', 'compacted'),)),
  219. ('13 2026-07-23T18:46Z a4bc1ca502', (
  220. ('+', 'display_kind', 'api_content'),
  221. ('+', 'display_metadata', 'display_kind'),
  222. )),
  223. ('14 2026-08-25T10:55Z 1104ffe0b9', (('+', '_compressed_summary', 'observed'),)),
  224. ('15 2026-09-09T17:05Z 1c6683e8e0', (
  225. ('+', 'display_identity', 'display_metadata'),
  226. ('+', 'display_order', 'display_identity'),
  227. )),
  228. ('16 2026-09-28T08:42Z eed37d63ce', (
  229. ('+', 'message_uid', 'display_order'),
  230. ('+', 'absorbed_message_uids', 'message_uid'),
  231. ('+', 'tool_call_uids', 'absorbed_message_uids'),
  232. ('+', 'tool_call_uid', 'tool_call_uids'),
  233. )),
  234. ),
  235. ),
  236. "session_model_usage": _TableHistory(
  237. base=(
  238. 'session_id', 'model', 'billing_provider', 'billing_base_url',
  239. 'api_call_count', 'input_tokens', 'output_tokens',
  240. 'cache_read_tokens', 'cache_write_tokens', 'reasoning_tokens',
  241. 'estimated_cost_usd', 'first_seen', 'last_seen',
  242. ),
  243. events=(
  244. ('01 2026-07-11T12:59Z 0d63c23f36', (
  245. ('+', 'billing_mode', 'billing_base_url'),
  246. ('+', 'actual_cost_usd', 'estimated_cost_usd'),
  247. ('+', 'cost_status', 'actual_cost_usd'),
  248. ('+', 'cost_source', 'cost_status'),
  249. )),
  250. ('02 2026-07-16T11:23Z eb6aa03609', (('+', 'task', 'billing_mode'),)),
  251. ),
  252. ),
  253. }