shared_metrics.py 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792
  1. """Durable aggregation and local export for Hermes shared metrics."""
  2. from __future__ import annotations
  3. import atexit
  4. import json
  5. import logging
  6. import sqlite3
  7. import threading
  8. import time
  9. import uuid
  10. from collections.abc import Callable, Iterator
  11. from contextlib import contextmanager
  12. from datetime import datetime, timedelta, timezone, UTC
  13. from pathlib import Path
  14. from typing import Any
  15. from hermes_cli.sqlite_util import add_column_if_missing, write_txn
  16. from hermes_constants import get_hermes_home
  17. from utils import atomic_json_write
  18. from .shared_metrics_contract import (
  19. CLIENT_ACTIVE_METRIC,
  20. COUNTER_METRICS,
  21. FEATURE_ADOPTION_METRIC,
  22. INSTALL_SNAPSHOT_METRIC,
  23. MILESTONE_METRIC,
  24. MODEL_ROUTE_METRIC,
  25. SUM_METRICS,
  26. client_resource_is_valid,
  27. counter_dimensions_are_valid,
  28. )
  29. from .shared_metrics_drift import conform_counter_dimensions, counter_dimensions_shape_is_known
  30. _PACKAGE_SCHEMA_VERSION = "hermes.shared_metrics.v4"
  31. _STORE_SCHEMA_VERSION = "2"
  32. _BUSY_TIMEOUT_MS = 250
  33. _SCHEMA_BUSY_TIMEOUT_MS = 5_000
  34. # Writes a busy store could not take within _BUSY_TIMEOUT_MS, kept in memory for the next write.
  35. _DEFERRED_WRITES_MAX = 1_024
  36. _LOCAL_HISTORY_RETENTION_DAYS = 30
  37. _ACTIVE_INSTALL_STATE_KEY = "client_active_recorded_at"
  38. _INSTALL_SNAPSHOT_STATE_KEY = "install_snapshot_recorded_at"
  39. _MILESTONE_STATE_PREFIX = "milestone:"
  40. _FEATURE_STATE_PREFIX = "feature:"
  41. _DAILY_ONCE_STATE_PREFIX = "daily_once:"
  42. _ACTIVE_INSTALL_INTERVAL = timedelta(hours=24)
  43. # Column order of the client resource in every counter_aggregates statement.
  44. _RESOURCE_COLUMNS = ("hermes_version", "os_family", "architecture", "install_method")
  45. _CREATE_TELEMETRY_STATE_SQL = """
  46. CREATE TABLE IF NOT EXISTS telemetry_state (
  47. key TEXT PRIMARY KEY,
  48. value TEXT NOT NULL
  49. )
  50. """
  51. _CREATE_COUNTER_AGGREGATES_SQL = """
  52. CREATE TABLE IF NOT EXISTS counter_aggregates (
  53. period_start TEXT NOT NULL,
  54. metric_name TEXT NOT NULL,
  55. hermes_version TEXT NOT NULL,
  56. os_family TEXT NOT NULL,
  57. architecture TEXT NOT NULL,
  58. install_method TEXT NOT NULL,
  59. dimensions_json TEXT NOT NULL,
  60. value INTEGER NOT NULL,
  61. packaged_value INTEGER NOT NULL DEFAULT 0,
  62. PRIMARY KEY (
  63. period_start, metric_name, hermes_version, os_family, architecture,
  64. install_method, dimensions_json
  65. )
  66. )
  67. """
  68. _CREATE_PACKAGE_OUTBOX_SQL = """
  69. CREATE TABLE IF NOT EXISTS package_outbox (
  70. package_id TEXT PRIMARY KEY,
  71. period_start TEXT NOT NULL,
  72. period_end TEXT NOT NULL,
  73. payload_json TEXT NOT NULL,
  74. created_at TEXT NOT NULL,
  75. exported_at TEXT
  76. )
  77. """
  78. # ``send_consent_windows`` records consent as explicit intervals rather than a moving
  79. # day-stamp: opened when send consent is observed, heartbeat-confirmed on every later
  80. # observation, closed at the LAST CONFIRMED moment (never "now") on withdrawal. Additive
  81. # like the send columns; old readers never touch these tables.
  82. _CREATE_CONSENT_TABLES_SQL = (
  83. """
  84. CREATE TABLE IF NOT EXISTS send_consent_windows (
  85. opened_at TEXT NOT NULL,
  86. last_confirmed_at TEXT NOT NULL,
  87. closed_at TEXT
  88. )
  89. """,
  90. """
  91. CREATE TABLE IF NOT EXISTS consent_marks (
  92. name TEXT PRIMARY KEY CHECK (name IN ('obs', 'data')),
  93. stamp TEXT NOT NULL
  94. )
  95. """,
  96. )
  97. # v1 -> v2: the resource columns were added to the counter primary key.
  98. _MIGRATE_V1_SQL = (
  99. "ALTER TABLE counter_aggregates RENAME TO counter_aggregates_v1",
  100. _CREATE_COUNTER_AGGREGATES_SQL,
  101. """
  102. INSERT INTO counter_aggregates(
  103. period_start, metric_name, hermes_version, os_family, architecture,
  104. install_method, dimensions_json, value, packaged_value
  105. )
  106. SELECT
  107. period_start, metric_name, hermes_version, 'unknown', 'unknown', 'unknown',
  108. dimensions_json, value, packaged_value
  109. FROM counter_aggregates_v1
  110. """,
  111. "DROP TABLE counter_aggregates_v1",
  112. )
  113. _PENDING_PERIOD_COUNT_SQL = """
  114. SELECT COUNT(*) AS period_count FROM (
  115. SELECT period_start, hermes_version, os_family, architecture, install_method
  116. FROM counter_aggregates WHERE value > packaged_value
  117. GROUP BY period_start, hermes_version, os_family, architecture, install_method
  118. )
  119. """
  120. _INCREMENT_COUNTER_SQL = """
  121. INSERT INTO counter_aggregates(
  122. period_start, metric_name, hermes_version, os_family, architecture,
  123. install_method, dimensions_json, value, packaged_value
  124. ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)
  125. ON CONFLICT(
  126. period_start, metric_name, hermes_version, os_family, architecture,
  127. install_method, dimensions_json
  128. )
  129. DO UPDATE SET value = value + excluded.value
  130. """
  131. # (column, declaration) added to package_outbox for transmission bookkeeping.
  132. _SEND_COLUMNS = (
  133. # When the 202 was received. NULL = never acknowledged.
  134. ("sent_at", "TEXT"),
  135. # NULL/'pending' = eligible, 'sent' = done, 'rejected' = permanent 400.
  136. ("send_state", "TEXT"),
  137. ("send_attempts", "INTEGER NOT NULL DEFAULT 0"),
  138. # Earliest next attempt; enforces backoff across process restarts.
  139. ("next_attempt_at", "TEXT"),
  140. ("last_error", "TEXT"),
  141. # The install_id actually transmitted, frozen on the first attempt so retries stay
  142. # byte-identical; the body is recomputed from payload_json.
  143. ("sent_install_id", "TEXT"),
  144. # Rewritten on every claim. Settlement and the pre-POST revalidation are compare-and-set
  145. # on it, so a claimant whose lease lapsed loses authority the moment another process
  146. # reclaims (else a suspended sender double-POSTs).
  147. ("claim_token", "TEXT"),
  148. )
  149. logger = logging.getLogger(__name__)
  150. # Every store in this process that ever deferred a write, the per-profile runtimes' stores and the
  151. # ones no runtime owns alike (the root profile's host-level active-profiles accumulator, the Desktop
  152. # store). ``os._exit`` skips each store's atexit drain, so a hard exit drains this set instead.
  153. _DEFERRING_STORES: list[SharedMetricsStore] = []
  154. _DEFERRING_STORES_LOCK = threading.Lock()
  155. def drain_deferring_stores(deadline: float) -> None:
  156. """Write every store's deferred rows before an ``os._exit``, waiting on a busy store no later
  157. than ``deadline`` (``time.monotonic``). Each deferred write runs against the store that deferred
  158. it, so one profile's rows never land in another's database."""
  159. with _DEFERRING_STORES_LOCK:
  160. stores = list(_DEFERRING_STORES)
  161. for store in stores:
  162. remaining_ms = int((deadline - time.monotonic()) * 1000)
  163. if remaining_ms <= 0:
  164. return
  165. store.drain_deferred(busy_timeout_ms=remaining_ms)
  166. def _utc_now() -> datetime:
  167. return datetime.now(UTC)
  168. def _isoformat(value: datetime) -> str:
  169. return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
  170. def _compact_json(value: Any) -> str:
  171. return json.dumps(value, sort_keys=True, separators=(",", ":"))
  172. def _row_resource(row: sqlite3.Row) -> dict[str, str]:
  173. return {column: row[column] for column in _RESOURCE_COLUMNS}
  174. def _ensure_private(path: Path, mode: int) -> None:
  175. if mode == 0o700:
  176. path.mkdir(parents=True, exist_ok=True, mode=mode)
  177. else:
  178. path.touch(mode=mode, exist_ok=True)
  179. try:
  180. path.chmod(mode)
  181. except OSError:
  182. pass
  183. class SharedMetricsStore:
  184. """Persist allowlisted counters and export immutable delta packages."""
  185. def __init__(
  186. self, database_path: Path | None = None, outbox_directory: Path | None = None
  187. ) -> None:
  188. root = get_hermes_home() / "telemetry" / "shared_metrics"
  189. self.database_path = database_path or root / "metrics.sqlite3"
  190. self.outbox_directory = outbox_directory or root / "outbox"
  191. _ensure_private(self.database_path.parent, 0o700)
  192. _ensure_private(self.outbox_directory, 0o700)
  193. _ensure_private(self.database_path, 0o600)
  194. self._ensure_schema()
  195. self._deferred: list[Callable[[sqlite3.Connection], None]] = []
  196. self._deferred_lock = threading.Lock()
  197. self._drain_at_exit = False
  198. def record_model_call(self, dimensions: dict[str, str], resource: dict[str, str]) -> None:
  199. """Increment the terminal model-call counter for the current UTC day."""
  200. self.record_counter(MODEL_ROUTE_METRIC, dimensions, resource)
  201. def record_client_active(self, resource: dict[str, str]) -> bool:
  202. """Record this install at most once in any rolling 24-hour window."""
  203. return self._record_once_per_interval(
  204. _ACTIVE_INSTALL_STATE_KEY, CLIENT_ACTIVE_METRIC, {}, resource
  205. )
  206. def record_install_snapshot(self, dimensions: dict[str, str], resource: dict[str, str]) -> bool:
  207. """Record the configuration snapshot at most once in any rolling 24-hour window."""
  208. return self._record_once_per_interval(
  209. _INSTALL_SNAPSHOT_STATE_KEY, INSTALL_SNAPSHOT_METRIC, dimensions, resource
  210. )
  211. def install_snapshot_due(self) -> bool:
  212. """Cheap read so the producer skips collecting a snapshot the latch would drop."""
  213. with self._connection() as connection:
  214. row = connection.execute(
  215. "SELECT value FROM telemetry_state WHERE key = ?", (_INSTALL_SNAPSHOT_STATE_KEY,)
  216. ).fetchone()
  217. last = self._parse_state_timestamp(row["value"]) if row is not None else None
  218. now = _utc_now()
  219. return last is None or last > now or now >= last + _ACTIVE_INSTALL_INTERVAL
  220. def _record_once_per_interval(
  221. self, state_key: str, metric_name: str, dimensions: dict[str, str],
  222. resource: dict[str, str],
  223. ) -> bool:
  224. """Transactional compare-and-set latch shared by concurrent Hermes processes."""
  225. self._validate_counter(metric_name, dimensions, resource)
  226. now = _utc_now()
  227. with self._write() as connection:
  228. row = connection.execute(
  229. "SELECT value FROM telemetry_state WHERE key = ?", (state_key,)
  230. ).fetchone()
  231. last_recorded = self._parse_state_timestamp(row["value"]) if row is not None else None
  232. if last_recorded is not None and last_recorded > now:
  233. # A wall-clock correction must not suppress activity until the stale future
  234. # timestamp plus another full interval.
  235. connection.execute(
  236. "UPDATE telemetry_state SET value = ? WHERE key = ?",
  237. (_isoformat(now), state_key),
  238. )
  239. return False
  240. if last_recorded is not None and now < last_recorded + _ACTIVE_INSTALL_INTERVAL:
  241. return False
  242. self._install_id(connection)
  243. self._record_counter_in_transaction(
  244. connection, metric_name, dimensions, resource, now.date().isoformat()
  245. )
  246. connection.execute(
  247. "INSERT INTO telemetry_state(key, value) VALUES (?, ?)"
  248. " ON CONFLICT(key) DO UPDATE SET value = excluded.value",
  249. (state_key, _isoformat(now)),
  250. )
  251. return True
  252. def record_counter(
  253. self, metric_name: str, dimensions: dict[str, str], resource: dict[str, str],
  254. amount: int = 1,
  255. ) -> None:
  256. """Add ``amount`` (1 for event counters, a token count for sums) for the current UTC day."""
  257. self._validate_counter(metric_name, dimensions, resource)
  258. if isinstance(amount, bool) or not isinstance(amount, int) or amount < 1 or (
  259. amount != 1 and metric_name not in SUM_METRICS
  260. ):
  261. raise ValueError(f"Unsupported amount for shared metric: {metric_name}")
  262. period_start = _utc_now().date().isoformat()
  263. self._write_or_defer(lambda connection: self._record_counter_in_transaction(
  264. connection, metric_name, dimensions, resource, period_start, amount
  265. ))
  266. def update_rollup_state(
  267. self, state_key: str,
  268. update: Callable[[dict[str, Any] | None], tuple[dict[str, Any], list[tuple[str, dict, dict, str]]]],
  269. ) -> None:
  270. """Read-modify-write one JSON rollup state and record the ``(metric, dimensions, resource,
  271. period_start)`` rows ``update`` closes, in one write transaction: every process of the profile
  272. serializes here, so a closed period is emitted exactly once. Rows the contract rejects are
  273. dropped (never retried) so one bad row cannot wedge the rollup. A busy store defers the update
  274. to the next write (it then runs with that write's clock)."""
  275. self._write_or_defer(lambda connection: self._update_rollup_state(connection, state_key, update))
  276. def _update_rollup_state(
  277. self, connection: sqlite3.Connection, state_key: str,
  278. update: Callable[[dict[str, Any] | None], tuple[dict[str, Any], list[tuple[str, dict, dict, str]]]],
  279. ) -> None:
  280. row = connection.execute(
  281. "SELECT value FROM telemetry_state WHERE key = ?", (state_key,)
  282. ).fetchone()
  283. try:
  284. state = json.loads(row["value"]) if row is not None else None
  285. except (TypeError, ValueError):
  286. state = None
  287. new_state, rows = update(state if isinstance(state, dict) else None)
  288. valid = [
  289. r for r in rows
  290. if counter_dimensions_are_valid(r[0], r[1]) and client_resource_is_valid(r[2])
  291. ]
  292. if valid:
  293. self._install_id(connection)
  294. for metric_name, dimensions, resource, period_start in valid:
  295. self._record_counter_in_transaction(connection, metric_name, dimensions, resource, period_start)
  296. connection.execute(
  297. "INSERT INTO telemetry_state(key, value) VALUES (?, ?)"
  298. " ON CONFLICT(key) DO UPDATE SET value = excluded.value",
  299. (state_key, _compact_json(new_state)),
  300. )
  301. def recorded_milestones(self) -> frozenset[str]:
  302. return self._latched(_MILESTONE_STATE_PREFIX)
  303. def record_milestone(self, milestone: str, install_age_bucket: str, resource: dict[str, str]) -> bool:
  304. """Record an install milestone at most once, ever (compare-and-set across processes)."""
  305. dimensions = {"install_age_bucket": install_age_bucket, "milestone": milestone}
  306. return self._record_once_ever(f"{_MILESTONE_STATE_PREFIX}{milestone}", MILESTONE_METRIC, dimensions, resource)
  307. # ---- v5 signals ----
  308. def recorded_features(self) -> frozenset[str]:
  309. return self._latched(_FEATURE_STATE_PREFIX)
  310. def record_feature_adoption(self, feature: str, days_bucket: str, resource: dict[str, str]) -> bool:
  311. """Record a feature's first use at most once per install, ever (same latch as milestones)."""
  312. dimensions = {"days_since_install_bucket": days_bucket, "feature": feature}
  313. return self._record_once_ever(f"{_FEATURE_STATE_PREFIX}{feature}", FEATURE_ADOPTION_METRIC, dimensions, resource)
  314. def record_counter_once_per_day(self, metric_name: str, dimensions: dict[str, str], resource: dict[str, str]) -> bool:
  315. """Count at most once per UTC day per dimension set, whatever the surface (``surface`` is
  316. attribution of the first one that day). One state row per dimension set, overwritten daily."""
  317. self._validate_counter(metric_name, dimensions, resource)
  318. now = _utc_now()
  319. today = now.date().isoformat()
  320. key = f"{_DAILY_ONCE_STATE_PREFIX}{metric_name}:" + _compact_json(
  321. {k: v for k, v in dimensions.items() if k != "surface"})
  322. with self._write() as connection:
  323. changed = connection.execute(
  324. "INSERT INTO telemetry_state(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE"
  325. " SET value = excluded.value WHERE telemetry_state.value != excluded.value",
  326. (key, today),
  327. ).rowcount
  328. if not changed:
  329. return False
  330. self._install_id(connection)
  331. self._record_counter_in_transaction(connection, metric_name, dimensions, resource, today)
  332. return True
  333. # ---- end v5 signals ----
  334. def _latched(self, prefix: str) -> frozenset[str]:
  335. with self._connection() as connection:
  336. rows = connection.execute(
  337. "SELECT key FROM telemetry_state WHERE key LIKE ?", (f"{prefix}%",)
  338. ).fetchall()
  339. return frozenset(row["key"][len(prefix):] for row in rows)
  340. def _record_once_ever(
  341. self, state_key: str, metric_name: str, dimensions: dict[str, str], resource: dict[str, str],
  342. ) -> bool:
  343. self._validate_counter(metric_name, dimensions, resource)
  344. now = _utc_now()
  345. with self._write() as connection:
  346. inserted = connection.execute(
  347. "INSERT INTO telemetry_state(key, value) VALUES (?, ?) ON CONFLICT(key) DO NOTHING",
  348. (state_key, _isoformat(now)),
  349. ).rowcount
  350. if not inserted:
  351. return False
  352. self._install_id(connection)
  353. self._record_counter_in_transaction(
  354. connection, metric_name, dimensions, resource, now.date().isoformat()
  355. )
  356. return True
  357. @staticmethod
  358. def _validate_counter(
  359. metric_name: str, dimensions: dict[str, str], resource: dict[str, str]
  360. ) -> None:
  361. if metric_name not in COUNTER_METRICS:
  362. raise ValueError(f"Unsupported shared metric: {metric_name}")
  363. if not counter_dimensions_are_valid(metric_name, dimensions):
  364. raise ValueError(f"Unsupported dimensions for shared metric: {metric_name}")
  365. if not client_resource_is_valid(resource):
  366. raise ValueError("Unsupported shared-metrics client resource")
  367. @staticmethod
  368. def _record_counter_in_transaction(
  369. connection: sqlite3.Connection,
  370. metric_name: str,
  371. dimensions: dict[str, str],
  372. resource: dict[str, str],
  373. period_start: str,
  374. amount: int = 1,
  375. ) -> None:
  376. connection.execute(
  377. _INCREMENT_COUNTER_SQL,
  378. (
  379. period_start, metric_name, *(resource[column] for column in _RESOURCE_COLUMNS),
  380. _compact_json(dimensions), amount,
  381. ),
  382. )
  383. def create_and_export_package(self) -> list[Path]:
  384. """Commit every pending delta package (one per period), then export the outbox."""
  385. with self._connection() as connection:
  386. row = connection.execute(_PENDING_PERIOD_COUNT_SQL).fetchone()
  387. for _ in range(int(row["period_count"]) if row is not None else 0):
  388. if self._create_package() is None:
  389. break
  390. return self._export_and_prune()
  391. def _write_or_defer(self, write: Callable[[sqlite3.Connection], None]) -> None:
  392. """Run ``write`` (after any deferred ones) in one write transaction. A store another writer
  393. holds past the short busy timeout keeps them in memory for the next write or process exit
  394. instead of losing them or stalling the caller."""
  395. with self._deferred_lock:
  396. pending, self._deferred = [*self._deferred, write], []
  397. try:
  398. with self._write() as connection:
  399. for item in pending:
  400. item(connection)
  401. except sqlite3.OperationalError as exc:
  402. busy = "locked" in str(exc) or "busy" in str(exc)
  403. with self._deferred_lock:
  404. # A failure that is not contention drops only this write: the earlier ones stay queued.
  405. self._deferred = [*(pending if busy else pending[:-1]), *self._deferred][-_DEFERRED_WRITES_MAX:]
  406. register, self._drain_at_exit = not self._drain_at_exit, True
  407. if register:
  408. atexit.register(self.drain_deferred)
  409. with _DEFERRING_STORES_LOCK:
  410. _DEFERRING_STORES.append(self)
  411. if not busy:
  412. raise
  413. def drain_deferred(self, busy_timeout_ms: int = _SCHEMA_BUSY_TIMEOUT_MS) -> None:
  414. """Write the deferred writes (process exit / export waits longer than the user's path does)."""
  415. with self._deferred_lock:
  416. pending, self._deferred = self._deferred, []
  417. if not pending:
  418. return
  419. try:
  420. with self._write(busy_timeout_ms=busy_timeout_ms) as connection:
  421. for item in pending:
  422. item(connection)
  423. except Exception as exc:
  424. if isinstance(exc, sqlite3.OperationalError) and ("locked" in str(exc) or "busy" in str(exc)):
  425. with self._deferred_lock: # still contended: keep them for the next write or exit drain
  426. self._deferred = [*pending, *self._deferred][-_DEFERRED_WRITES_MAX:]
  427. return
  428. logger.warning("Unable to write %d deferred shared-metrics updates", len(pending), exc_info=True)
  429. def create_and_export_package_if_due(self) -> list[Path]:
  430. """Create pending packages at most once per UTC day, then export them."""
  431. self.drain_deferred(_BUSY_TIMEOUT_MS)
  432. self._create_pending_packages_if_due()
  433. return self._export_and_prune()
  434. def _export_and_prune(self) -> list[Path]:
  435. exported = self._export_pending_packages()
  436. try:
  437. self._prune_expired_history()
  438. except Exception:
  439. logger.warning("Unable to prune expired shared-metrics history", exc_info=True)
  440. return exported
  441. def counter_snapshot(self) -> list[dict[str, Any]]:
  442. """Return cumulative counters for focused tests and local inspection."""
  443. with self._connection() as connection:
  444. rows = connection.execute(
  445. """
  446. SELECT period_start, metric_name, hermes_version, os_family, architecture,
  447. install_method, dimensions_json, value, packaged_value
  448. FROM counter_aggregates
  449. ORDER BY period_start, hermes_version, os_family, architecture, install_method,
  450. metric_name, dimensions_json
  451. """
  452. ).fetchall()
  453. return [
  454. {
  455. "period_start": row["period_start"], "metric_name": row["metric_name"],
  456. "resource": _row_resource(row), "dimensions": json.loads(row["dimensions_json"]),
  457. "value": row["value"], "packaged_value": row["packaged_value"],
  458. }
  459. for row in rows
  460. ]
  461. @contextmanager
  462. def _connection(
  463. self, *, busy_timeout_ms: int = _BUSY_TIMEOUT_MS
  464. ) -> Iterator[sqlite3.Connection]:
  465. connection = sqlite3.connect(self.database_path, timeout=busy_timeout_ms / 1000)
  466. try:
  467. connection.row_factory = sqlite3.Row
  468. connection.execute(f"PRAGMA busy_timeout={busy_timeout_ms}")
  469. with connection:
  470. yield connection
  471. finally:
  472. connection.close()
  473. @contextmanager
  474. def _write(self, *, busy_timeout_ms: int = _BUSY_TIMEOUT_MS) -> Iterator[sqlite3.Connection]:
  475. """A connection inside one write transaction."""
  476. with self._connection(busy_timeout_ms=busy_timeout_ms) as connection:
  477. with write_txn(connection):
  478. yield connection
  479. def _ensure_schema(self) -> None:
  480. # Serialize first-run creation and upgrades across Hermes processes.
  481. with self._write(busy_timeout_ms=_SCHEMA_BUSY_TIMEOUT_MS) as connection:
  482. connection.execute(_CREATE_TELEMETRY_STATE_SQL)
  483. schema_row = connection.execute(
  484. "SELECT value FROM telemetry_state WHERE key = 'schema_version'"
  485. ).fetchone()
  486. schema_version = str(schema_row["value"]) if schema_row is not None else None
  487. if schema_version == "1":
  488. for statement in _MIGRATE_V1_SQL:
  489. connection.execute(statement)
  490. schema_version = _STORE_SCHEMA_VERSION
  491. if schema_version is not None and schema_version != _STORE_SCHEMA_VERSION:
  492. raise RuntimeError(
  493. f"Unsupported shared-metrics store schema version: {schema_version}"
  494. )
  495. connection.execute(_CREATE_COUNTER_AGGREGATES_SQL)
  496. connection.execute(_CREATE_PACKAGE_OUTBOX_SQL)
  497. # Send bookkeeping columns are ADDITIVE and nullable; the schema version is
  498. # deliberately NOT bumped because the check above raises on unknown versions,
  499. # so a bump would hard-fail an older Hermes (second profile, rollback) sharing
  500. # the database. Old readers select named columns, never ``SELECT *``.
  501. existing = {
  502. str(row["name"])
  503. for row in connection.execute("PRAGMA table_info(package_outbox)")
  504. }
  505. for column, declaration in _SEND_COLUMNS:
  506. if column not in existing:
  507. add_column_if_missing(connection, "package_outbox", column, f"{column} {declaration}")
  508. for statement in _CREATE_CONSENT_TABLES_SQL:
  509. connection.execute(statement)
  510. connection.execute(
  511. "INSERT INTO telemetry_state(key, value) VALUES ('schema_version', ?)"
  512. " ON CONFLICT(key) DO UPDATE SET value = excluded.value",
  513. (_STORE_SCHEMA_VERSION,),
  514. )
  515. def _install_id(self, connection: sqlite3.Connection) -> str:
  516. query = "SELECT value FROM telemetry_state WHERE key = 'install_id'"
  517. row = connection.execute(query).fetchone()
  518. if row is None:
  519. connection.execute(
  520. "INSERT OR IGNORE INTO telemetry_state(key, value) VALUES ('install_id', ?)",
  521. (str(uuid.uuid4()),),
  522. )
  523. row = connection.execute(query).fetchone()
  524. if row is None:
  525. raise RuntimeError("Unable to create the shared-metrics install identity")
  526. return str(row["value"])
  527. @staticmethod
  528. def _parse_state_timestamp(value: Any) -> datetime | None:
  529. try:
  530. parsed = datetime.fromisoformat(str(value))
  531. except (TypeError, ValueError):
  532. return None
  533. return None if parsed.tzinfo is None else parsed.astimezone(UTC)
  534. def _create_pending_packages_if_due(self) -> None:
  535. now = _utc_now()
  536. with self._write() as connection:
  537. # Gate on the committed package, not its file write, so a failed
  538. # outbox export can be retried without packaging deltas twice.
  539. package_created_today = connection.execute(
  540. "SELECT 1 FROM package_outbox WHERE substr(created_at, 1, 10) >= ? LIMIT 1",
  541. (now.date().isoformat(),),
  542. ).fetchone()
  543. if package_created_today is None:
  544. while self._create_package_in_transaction(connection, now) is not None:
  545. pass
  546. def _create_package(self) -> dict[str, Any] | None:
  547. with self._write() as connection:
  548. return self._create_package_in_transaction(connection, _utc_now())
  549. def _create_package_in_transaction(
  550. self, connection: sqlite3.Connection, now: datetime
  551. ) -> dict[str, Any] | None:
  552. period_row = connection.execute(
  553. """
  554. SELECT period_start, hermes_version, os_family, architecture, install_method
  555. FROM counter_aggregates WHERE value > packaged_value
  556. ORDER BY period_start, hermes_version, os_family, architecture, install_method
  557. LIMIT 1
  558. """
  559. ).fetchone()
  560. if period_row is None or not period_row["period_start"]:
  561. return None
  562. period_value = period_row["period_start"]
  563. resource = _row_resource(period_row)
  564. resource_values = tuple(resource[column] for column in _RESOURCE_COLUMNS)
  565. rows = connection.execute(
  566. """
  567. SELECT metric_name, dimensions_json, value, packaged_value FROM counter_aggregates
  568. WHERE period_start = ? AND hermes_version = ? AND os_family = ?
  569. AND architecture = ? AND install_method = ? AND value > packaged_value
  570. ORDER BY metric_name, dimensions_json
  571. """,
  572. (period_value, *resource_values),
  573. ).fetchall()
  574. period_start = datetime.fromisoformat(str(period_value)).replace(tzinfo=UTC)
  575. if not client_resource_is_valid(resource):
  576. raise ValueError("Unsupported shared-metrics client resource")
  577. metrics = self._package_metrics(rows)
  578. if not metrics: # every row was a retired value with no bucket: settle them, try the next period
  579. self._mark_packaged(connection, rows, period_value, resource_values)
  580. return self._create_package_in_transaction(connection, now)
  581. payload = {
  582. "schema_version": _PACKAGE_SCHEMA_VERSION,
  583. "package_id": str(uuid.uuid4()),
  584. "install_id": self._install_id(connection),
  585. "period_start": _isoformat(period_start),
  586. "period_end": _isoformat(period_start + timedelta(days=1)),
  587. "generated_at": _isoformat(now),
  588. "resource": resource,
  589. "metrics": metrics,
  590. }
  591. connection.execute(
  592. "INSERT INTO package_outbox( package_id, period_start, period_end, payload_json,"
  593. " created_at ) VALUES (?, ?, ?, ?, ?)",
  594. (
  595. payload["package_id"], payload["period_start"], payload["period_end"],
  596. _compact_json(payload), payload["generated_at"],
  597. ),
  598. )
  599. # Advance the data high-water mark. This is the ONLY writer of the 'data' mark:
  600. # it clamps consent-window opens so a rolled-back clock can never open a window
  601. # underneath packages that already exist.
  602. connection.execute(
  603. "INSERT INTO consent_marks(name, stamp) VALUES ('data', ?)"
  604. " ON CONFLICT(name) DO UPDATE SET stamp = MAX(stamp, excluded.stamp)",
  605. (payload["period_end"],),
  606. )
  607. self._mark_packaged(connection, rows, period_value, resource_values)
  608. return payload
  609. @staticmethod
  610. def _mark_packaged(
  611. connection: sqlite3.Connection, rows: list[sqlite3.Row], period_value: str, resource_values: tuple,
  612. ) -> None:
  613. for row in rows:
  614. connection.execute(
  615. """
  616. UPDATE counter_aggregates SET packaged_value = value
  617. WHERE period_start = ? AND metric_name = ? AND hermes_version = ?
  618. AND os_family = ? AND architecture = ? AND install_method = ?
  619. AND dimensions_json = ?
  620. """,
  621. (period_value, row["metric_name"], *resource_values, row["dimensions_json"]),
  622. )
  623. @classmethod
  624. def _package_metrics(cls, rows: list[sqlite3.Row]) -> list[dict[str, Any]]:
  625. """Package rows, merging rows a re-bucket folded onto the same dimensions into one metric."""
  626. merged: dict[tuple[str, str], dict[str, Any]] = {}
  627. for row in rows:
  628. metric = cls._package_metric(row)
  629. if metric is None:
  630. continue
  631. key = (metric["name"], _compact_json(metric["dimensions"]))
  632. if key in merged:
  633. merged[key]["value"] += metric["value"]
  634. else:
  635. merged[key] = metric
  636. return list(merged.values())
  637. @staticmethod
  638. def _package_metric(row: sqlite3.Row) -> dict[str, Any] | None:
  639. metric_name = str(row["metric_name"])
  640. dimensions = json.loads(row["dimensions_json"])
  641. # An unknown field set is tamper evidence and fails closed (must-not-be-exported). A known
  642. # shape whose values fell out of the contract was recorded by an earlier build or catalog
  643. # (#133017): re-bucket it inside the closed enum, or leave the row out when its field has no
  644. # bucket, rather than abort the whole period's package. Either way no stale value leaves.
  645. if not isinstance(dimensions, dict) or not counter_dimensions_shape_is_known(metric_name, dimensions):
  646. raise ValueError(f"Unsupported dimensions for shared metric: {metric_name}")
  647. if not counter_dimensions_are_valid(metric_name, dimensions):
  648. conformed = conform_counter_dimensions(metric_name, dimensions)
  649. if conformed is None:
  650. logger.info("Shared-metric %s row holds a retired value with no bucket; not packaged", metric_name)
  651. return None
  652. logger.info(
  653. "Shared-metric %s row drifted out of the contract since it was recorded; "
  654. "packaged with bucketed dimensions %s", metric_name, _compact_json(conformed),
  655. )
  656. dimensions = conformed
  657. return {
  658. "name": metric_name, "type": "counter", "dimensions": dimensions,
  659. "value": row["value"] - row["packaged_value"],
  660. }
  661. def _export_pending_packages(self) -> list[Path]:
  662. with self._connection() as connection:
  663. rows = connection.execute(
  664. "SELECT package_id, payload_json FROM package_outbox"
  665. " WHERE exported_at IS NULL ORDER BY created_at, package_id"
  666. ).fetchall()
  667. exported: list[Path] = []
  668. for row in rows:
  669. package_id = str(row["package_id"])
  670. path = self.outbox_directory / f"{package_id}.json"
  671. atomic_json_write(
  672. path, json.loads(row["payload_json"]), indent=None, separators=(",", ":"),
  673. sort_keys=True, mode=0o600,
  674. )
  675. with self._connection() as connection:
  676. connection.execute(
  677. "UPDATE package_outbox SET exported_at = ?"
  678. " WHERE package_id = ? AND exported_at IS NULL",
  679. (_isoformat(_utc_now()), package_id),
  680. )
  681. exported.append(path)
  682. return exported
  683. def _prune_expired_history(self, *, now: datetime | None = None) -> None:
  684. """Remove exported local history after the bounded retention window."""
  685. cutoff = (now or _utc_now()) - timedelta(days=_LOCAL_HISTORY_RETENTION_DAYS)
  686. cutoff_timestamp = _isoformat(cutoff)
  687. with self._connection() as connection:
  688. rows = connection.execute(
  689. "SELECT package_id FROM package_outbox"
  690. " WHERE exported_at IS NOT NULL AND exported_at < ?"
  691. " ORDER BY exported_at, package_id",
  692. (cutoff_timestamp,),
  693. ).fetchall()
  694. removable_package_ids: list[str] = []
  695. for row in rows:
  696. package_id = str(row["package_id"])
  697. try:
  698. (self.outbox_directory / f"{package_id}.json").unlink(missing_ok=True)
  699. removable_package_ids.append(package_id)
  700. except OSError:
  701. logger.warning(
  702. "Unable to prune expired shared-metrics package %s", package_id, exc_info=True
  703. )
  704. with self._write() as connection:
  705. # A package the ingest accepted or refused is never sent again, and its outbox file is
  706. # the local history copy, so the database drops its duplicate of the body.
  707. connection.execute(
  708. "UPDATE package_outbox SET payload_json = ''"
  709. " WHERE exported_at IS NOT NULL AND send_state IN ('sent', 'rejected')"
  710. " AND payload_json != ''"
  711. )
  712. for package_id in removable_package_ids:
  713. connection.execute(
  714. "DELETE FROM package_outbox"
  715. " WHERE package_id = ? AND exported_at IS NOT NULL AND exported_at < ?",
  716. (package_id, cutoff_timestamp),
  717. )
  718. connection.execute(
  719. """
  720. DELETE FROM counter_aggregates
  721. WHERE period_start < ? AND value = packaged_value AND NOT EXISTS (
  722. SELECT 1 FROM package_outbox WHERE exported_at IS NULL
  723. AND substr(package_outbox.period_start, 1, 10)
  724. = counter_aggregates.period_start
  725. )
  726. """,
  727. (cutoff.date().isoformat(),),
  728. )