shared_metrics_sender.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526
  1. """Transmit exported shared-metrics packages to the Nous telemetry service.
  2. Sender side of the ingest contract (telemetry repo ``CONTRACT.md``): ``202`` durably stored,
  3. mark sent; ``400`` permanently malformed, never retry; ``429`` keep, retry after
  4. ``Retry-After``; ``5xx`` / timeout / connection error keep, retry with backoff.
  5. """
  6. from __future__ import annotations
  7. import gzip
  8. import json
  9. import logging
  10. import random
  11. import sqlite3
  12. import time
  13. import urllib.error
  14. import urllib.request
  15. import uuid
  16. from contextlib import contextmanager
  17. from dataclasses import dataclass
  18. from datetime import datetime, timedelta, timezone, UTC
  19. from agent.retry_utils import parse_retry_after_seconds
  20. from hermes_cli.sqlite_util import write_txn
  21. from .shared_metrics import _isoformat, _utc_now
  22. logger = logging.getLogger(__name__)
  23. #: Contract recommends timing out at 30s and treating a timeout as retryable.
  24. REQUEST_TIMEOUT_SECONDS = 30
  25. #: In-process attempts per package per pass, then the package waits for a
  26. #: later pass. Backoff is 1s/5s/25s with full jitter.
  27. MAX_ATTEMPTS = 3
  28. _BACKOFF_BASE_SECONDS = 1
  29. _BACKOFF_FACTOR = 5
  30. #: Contract recommends gzip above roughly this size.
  31. GZIP_THRESHOLD_BYTES = 4096
  32. #: Packages per pass. Bounds work on an interactive hook even after an outage.
  33. MAX_PACKAGES_PER_PASS = 20
  34. #: Claim lease, written INTO THE FUTURE as next_attempt_at so no other process selects
  35. #: the row meanwhile. Must exceed one package's worst case (3x30s timeouts + 1s+5s
  36. #: backoff, ~96s) — hence packages are claimed one at a time right before sending; a
  37. #: batch of 20 under one lease can run ~1900s and get re-sent by another process.
  38. _CLAIM_LEASE_SECONDS = 300
  39. #: Floor applied after a pass fails to deliver, so a hard-down service is not
  40. #: retried on every task completion.
  41. _FAILURE_BACKOFF_SECONDS = 15 * 60
  42. #: Permanent statuses per the ingest contract, deliberately narrow: 400 never validates,
  43. #: 413 (over the 1 MiB cap) cannot shrink on retry. Everything else — including 403 from
  44. #: the origin guard and 404 from a bad path — is deployment/edge misconfiguration that
  45. #: resolves without the package changing, so it is retried.
  46. _PERMANENT_STATUSES = frozenset({400, 413})
  47. #: Attempts after which a package is abandoned; otherwise a poisoned row is retried until
  48. #: 30-day retention deletes it (~160 requests) and pins the head of the queue.
  49. MAX_SEND_ATTEMPTS = 25
  50. #: Max distance one reconcile can advance the 'obs' mark. Never binds for honest
  51. #: heartbeats (hours apart); bounds FORWARD clock poison, where one glitched sample (NTP
  52. #: flap reading 2099) would drag every window open and confirmation horizon decades ahead.
  53. MAX_OBS_ADVANCE_SECONDS = 30 * 24 * 3600
  54. def _parse_stamp(value: str) -> datetime:
  55. """Parse a stamp this module itself wrote (Z-suffixed ISO-8601, UTC)."""
  56. return datetime.fromisoformat(value).astimezone(UTC)
  57. @dataclass
  58. class SendOutcome:
  59. """What one pass did. Returned for tests and diagnostics."""
  60. sent: int = 0
  61. rejected: int = 0
  62. deferred: int = 0
  63. @dataclass(slots=True)
  64. class _Response:
  65. status: int
  66. retry_after: str | None
  67. body: str
  68. def _post(endpoint: str, payload: bytes, *, timeout: int) -> _Response:
  69. """POST one package. Raises on transport failure; never on HTTP status."""
  70. headers = {
  71. "Content-Type": "application/json",
  72. "User-Agent": "hermes-agent-shared-metrics/1",
  73. }
  74. body = payload
  75. if len(payload) > GZIP_THRESHOLD_BYTES:
  76. # mtime=0 keeps two sends of one package byte-identical on the wire too.
  77. body = gzip.compress(payload, mtime=0)
  78. headers["Content-Encoding"] = "gzip"
  79. request = urllib.request.Request(endpoint, data=body, headers=headers, method="POST")
  80. try:
  81. with urllib.request.urlopen(request, timeout=timeout) as response:
  82. return _Response(
  83. response.status,
  84. response.headers.get("Retry-After"),
  85. response.read(2048).decode("utf-8", "replace"),
  86. )
  87. except urllib.error.HTTPError as exc:
  88. # An HTTP error status is a normal contract outcome, not a failure.
  89. return _Response(
  90. exc.code,
  91. exc.headers.get("Retry-After") if exc.headers else None,
  92. exc.read(2048).decode("utf-8", "replace") if exc.fp else "",
  93. )
  94. def _retry_after_seconds(value: str | None, default: int) -> int:
  95. seconds = parse_retry_after_seconds(value)
  96. if seconds is None:
  97. return default
  98. # Clamp so a bogus value cannot park a package for years, and never go below one second.
  99. return max(1, min(int(seconds), 86_400))
  100. def reconcile_send_consent(
  101. connection: sqlite3.Connection,
  102. send_enabled: bool,
  103. *,
  104. now: datetime | None = None,
  105. ) -> None:
  106. """Reconcile the consent-window table with the observed config state.
  107. THE ONLY writer of consent state; must run inside a write transaction. Idempotent in
  108. (config, now, store): call it from anywhere, any number of times, in any order.
  109. """
  110. stamp = _isoformat(now or _utc_now())
  111. raw_stamp = stamp # pre-cap observation time, used to clamp closes
  112. previous_obs = connection.execute(
  113. "SELECT stamp FROM consent_marks WHERE name = 'obs'"
  114. ).fetchone()
  115. if previous_obs is not None:
  116. ceiling = _isoformat(
  117. _parse_stamp(str(previous_obs[0])) + timedelta(seconds=MAX_OBS_ADVANCE_SECONDS)
  118. )
  119. stamp = min(stamp, ceiling)
  120. connection.execute(
  121. """
  122. INSERT INTO consent_marks(name, stamp) VALUES ('obs', ?)
  123. ON CONFLICT(name) DO UPDATE SET stamp = MAX(stamp, excluded.stamp)
  124. """,
  125. (stamp,),
  126. )
  127. marks = dict(connection.execute("SELECT name, stamp FROM consent_marks").fetchall())
  128. obs = marks["obs"] # >= stamp; immune to clock rollback
  129. data = marks.get("data")
  130. open_row = connection.execute(
  131. "SELECT rowid FROM send_consent_windows WHERE closed_at IS NULL"
  132. ).fetchone()
  133. if send_enabled:
  134. if open_row is None:
  135. opened = max(x for x in (obs, data) if x is not None)
  136. connection.execute(
  137. "INSERT INTO send_consent_windows(opened_at, last_confirmed_at)"
  138. " VALUES (?, ?)",
  139. (opened, opened),
  140. )
  141. else:
  142. connection.execute(
  143. "UPDATE send_consent_windows"
  144. " SET last_confirmed_at = MAX(last_confirmed_at, ?)"
  145. " WHERE rowid = ?",
  146. (obs, open_row[0]),
  147. )
  148. elif open_row is not None:
  149. # Close at the last CONFIRMED moment, never after the closing observation's RAW
  150. # stamp. Both clamps are load-bearing: last_confirmed_at means an unobserved gap
  151. # (machine off, hand-edited config) is never asserted as consented; the raw
  152. # (pre-cap) stamp pulls a glitched-forward last_confirmed_at back to the true
  153. # revoke moment. A rolled-back clock only closes EARLIER — fail-closed.
  154. connection.execute(
  155. "UPDATE send_consent_windows"
  156. " SET closed_at = MIN(last_confirmed_at, ?)"
  157. " WHERE rowid = ?",
  158. (raw_stamp, open_row[0]),
  159. )
  160. #: Claim-time consent predicate: the package's period must fall entirely inside SOME
  161. #: recorded window. An open window vouches only up to its last confirmed moment, so a
  162. #: package running past it waits for the next reconcile heartbeat (fail-closed).
  163. CONSENT_GATE_SQL = """EXISTS (
  164. SELECT 1 FROM send_consent_windows w
  165. WHERE package_outbox.period_start >= w.opened_at
  166. AND package_outbox.period_end <=
  167. CASE WHEN w.closed_at IS NULL THEN w.last_confirmed_at
  168. ELSE w.closed_at END
  169. )"""
  170. class SharedMetricsSender:
  171. """Sends exported packages, one bounded pass at a time."""
  172. def __init__(
  173. self,
  174. store,
  175. endpoint: str,
  176. *,
  177. post=_post,
  178. sleep=time.sleep,
  179. now=_utc_now,
  180. max_attempts: int = MAX_ATTEMPTS,
  181. consent_check=None,
  182. ) -> None:
  183. self._store = store
  184. self._endpoint = endpoint
  185. self._post = post
  186. self._sleep = sleep
  187. self._now = now
  188. self._max_attempts = max_attempts
  189. # Called before every package. None disables the check for callers that have
  190. # already established consent out of band (tests, E2E).
  191. self._consent_check = consent_check
  192. @contextmanager
  193. def _write(self):
  194. """One store connection inside a write transaction."""
  195. with self._store._connection() as connection:
  196. with write_txn(connection):
  197. yield connection
  198. # -- selection ---------------------------------------------------------
  199. def _claim_next(self, now: datetime, seen: set[str]) -> dict | None:
  200. """Claim exactly ONE package, immediately before it is sent (see _CLAIM_LEASE_SECONDS).
  201. ``seen`` (packages this pass finished with) is excluded IN SQL: with LIMIT 1,
  202. returning None for a seen row would look like an empty queue and abandon everything
  203. behind it, and rows can legitimately become eligible again mid-pass.
  204. """
  205. with self._write() as connection:
  206. stamp = _isoformat(now)
  207. lease_until = now + timedelta(seconds=_CLAIM_LEASE_SECONDS)
  208. placeholders = ",".join("?" for _ in seen)
  209. exclusion = f" AND package_id NOT IN ({placeholders})" if seen else ""
  210. # Consent is a READ here: the claim must never mutate the window table.
  211. row = connection.execute(
  212. f"""
  213. SELECT package_id, payload_json, sent_install_id
  214. FROM package_outbox
  215. WHERE exported_at IS NOT NULL
  216. AND (send_state IS NULL OR send_state = 'pending')
  217. AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
  218. AND {CONSENT_GATE_SQL}
  219. AND send_attempts < ?
  220. {exclusion}
  221. ORDER BY created_at, package_id
  222. LIMIT 1
  223. """,
  224. (stamp, MAX_SEND_ATTEMPTS, *sorted(seen)),
  225. ).fetchone()
  226. if row is None:
  227. return None
  228. package_id = str(row[0])
  229. derived = row[2] or self._freeze_identity(connection, package_id, row[1])
  230. if derived is None:
  231. # Unusable row, already marked rejected. Tell the caller to continue.
  232. return {"package_id": package_id, "skip": True}
  233. token = str(uuid.uuid4())
  234. connection.execute(
  235. """
  236. UPDATE package_outbox
  237. SET send_state = 'pending',
  238. send_attempts = send_attempts + 1,
  239. next_attempt_at = ?,
  240. claim_token = ?
  241. WHERE package_id = ?
  242. """,
  243. # The token is this claim's identity: a reclaim after expiry mints a new one
  244. # and every later write by THIS claimant is compare-and-set against it.
  245. (_isoformat(lease_until), token, package_id),
  246. )
  247. return {
  248. "package_id": package_id, "payload_json": str(row[1]), "derived": str(derived),
  249. "claim_token": token, "skip": False,
  250. }
  251. @staticmethod
  252. def _freeze_identity(
  253. connection: sqlite3.Connection, package_id: str, payload_json
  254. ) -> str | None:
  255. """Record the transmitted install_id on the row, or reject an unusable one.
  256. Rejecting rather than raising matters: an exception would roll back the claim
  257. transaction and block every healthy package behind this one.
  258. """
  259. install_id = None
  260. try:
  261. payload = json.loads(payload_json)
  262. except (TypeError, ValueError):
  263. reason = "unreadable payload"
  264. else:
  265. # Valid JSON is not enough: a top-level array/string/number parses cleanly.
  266. install_id = payload.get("install_id") if isinstance(payload, dict) else None
  267. if not isinstance(payload, dict):
  268. reason = f"payload is {type(payload).__name__}, expected object"
  269. elif not isinstance(install_id, str) or not install_id.strip():
  270. reason = "payload has no usable install_id"
  271. else:
  272. reason = None
  273. if reason is not None:
  274. logger.warning("Shared-metrics package %s cannot be sent (%s)", package_id, reason)
  275. connection.execute(
  276. """
  277. UPDATE package_outbox
  278. SET send_state = 'rejected', last_error = ?
  279. WHERE package_id = ?
  280. """,
  281. (reason, package_id),
  282. )
  283. return None
  284. connection.execute(
  285. "UPDATE package_outbox SET sent_install_id = ? WHERE package_id = ?",
  286. (install_id, package_id),
  287. )
  288. return str(install_id)
  289. # -- transmission ------------------------------------------------------
  290. @staticmethod
  291. def _body(payload_json: str, transmitted_id: str) -> bytes:
  292. """Rebuild the exact bytes to send: deterministic json.dumps of the stored package
  293. with install_id from the frozen ``sent_install_id`` column (resends are byte-identical)."""
  294. payload = dict(json.loads(payload_json))
  295. payload["install_id"] = transmitted_id
  296. return json.dumps(payload, indent=2, sort_keys=True).encode("utf-8")
  297. def _mark(self, package_id: str, *, token: str | None = None, **columns) -> None:
  298. """Write send state for one package.
  299. Guarded on send_state so a lapsed pass cannot resurrect a row another process already
  300. finished. With ``token`` the write is also compare-and-set on claim_token: a
  301. superseded claimant writes zero rows.
  302. """
  303. assignments = ", ".join(f"{name} = ?" for name in columns)
  304. predicate = " AND (send_state IS NULL OR send_state = 'pending')"
  305. params: list = [*columns.values(), package_id]
  306. if token is not None:
  307. predicate += " AND claim_token = ?"
  308. params.append(token)
  309. with self._write() as connection:
  310. connection.execute(
  311. f"UPDATE package_outbox SET {assignments} WHERE package_id = ?{predicate}",
  312. params,
  313. )
  314. def _renew_claim(self, package_id: str, token: str | None) -> bool:
  315. """Atomically re-assert ownership and extend the lease (CAS, one row; rowcount == 1
  316. is the only grant).
  317. A read-only check is not enough: a claimant whose lease expired while suspended still
  318. has its token in the row and would POST while another process reclaims. Renewing in
  319. the same CAS keeps the 30s POST inside the 300s lease.
  320. """
  321. if token is None:
  322. return False
  323. try:
  324. now = self._now()
  325. lease_until = now + timedelta(seconds=_CLAIM_LEASE_SECONDS)
  326. with self._write() as connection:
  327. cursor = connection.execute(
  328. """
  329. UPDATE package_outbox
  330. SET next_attempt_at = ?
  331. WHERE package_id = ?
  332. AND claim_token = ?
  333. AND (send_state IS NULL OR send_state = 'pending')
  334. AND next_attempt_at > ?
  335. """,
  336. (_isoformat(lease_until), package_id, token, _isoformat(now)),
  337. )
  338. return cursor.rowcount == 1
  339. except Exception:
  340. # If renewal itself fails, do not transmit on unproven authority.
  341. logger.warning("Unable to renew shared-metrics claim", exc_info=True)
  342. return False
  343. def _defer(
  344. self, package_id: str, delay_seconds: int, reason: str, *, token: str | None = None
  345. ) -> None:
  346. # Clamp to >= 1s so a past deadline can never make the row instantly re-eligible.
  347. retry_at = self._now().timestamp() + max(1, int(delay_seconds))
  348. self._mark(
  349. package_id,
  350. token=token,
  351. send_state="pending",
  352. next_attempt_at=_isoformat(datetime.fromtimestamp(retry_at, tz=UTC)),
  353. last_error=reason[:500],
  354. )
  355. def _send_one(self, package: dict) -> str:
  356. """Try one package. Returns 'sent', 'rejected', or 'deferred'.
  357. Delivery is at-least-once: renewal plus token-fenced writes close the claim->POST
  358. and settle-after-reclaim gaps, but a suspension MID-POST can still duplicate.
  359. """
  360. package_id = package["package_id"]
  361. token = package.get("claim_token")
  362. body = self._body(package["payload_json"], package["derived"])
  363. def defer(delay: int, reason: str) -> str:
  364. self._defer(package_id, delay, reason, token=token)
  365. return "deferred"
  366. for attempt in range(1, self._max_attempts + 1):
  367. # Renew before EVERY external POST so a lapsed claimant yields even before anyone
  368. # reclaims. The ingest key is minute-prefixed: duplicates are distinct objects.
  369. if not self._renew_claim(package_id, token):
  370. logger.info(
  371. "Shared-metrics claim on %s superseded or expired; yielding", package_id
  372. )
  373. return "deferred"
  374. try:
  375. response = self._post(self._endpoint, body, timeout=REQUEST_TIMEOUT_SECONDS)
  376. except Exception as exc: # transport failure: offline, DNS, TLS
  377. reason = f"{type(exc).__name__}: {exc}"
  378. else:
  379. if response.status == 202:
  380. self._mark(
  381. package_id, token=token, send_state="sent",
  382. sent_at=_isoformat(self._now()), last_error=None,
  383. )
  384. return "sent"
  385. if response.status in _PERMANENT_STATUSES:
  386. logger.warning(
  387. "Telemetry package %s rejected with HTTP %s; not retrying",
  388. package_id,
  389. response.status,
  390. )
  391. self._mark(
  392. package_id, token=token, send_state="rejected",
  393. last_error=f"HTTP {response.status}: {response.body[:400]}",
  394. )
  395. return "rejected"
  396. if response.status == 429:
  397. return defer(
  398. _retry_after_seconds(response.retry_after, _FAILURE_BACKOFF_SECONDS),
  399. "rate limited",
  400. )
  401. # 5xx and anything unexpected: retryable.
  402. reason = f"HTTP {response.status}"
  403. if attempt >= self._max_attempts:
  404. return defer(_FAILURE_BACKOFF_SECONDS, reason)
  405. self._sleep(self._backoff(attempt))
  406. return defer(_FAILURE_BACKOFF_SECONDS, "attempts exhausted")
  407. @staticmethod
  408. def _backoff(attempt: int) -> float:
  409. """1s, 5s, 25s with full jitter."""
  410. return random.uniform(0, _BACKOFF_BASE_SECONDS * (_BACKOFF_FACTOR ** (attempt - 1)))
  411. # -- entry point -------------------------------------------------------
  412. def send_pending(self) -> SendOutcome:
  413. """Run one bounded pass, one claim+send at a time, re-checking consent before each
  414. send so revoking `send` mid-pass stops the remainder. Never raises."""
  415. outcome = SendOutcome()
  416. seen: set[str] = set()
  417. for _ in range(MAX_PACKAGES_PER_PASS):
  418. if not self._still_consented():
  419. # Reconcile through the single consent writer so the window closes at its
  420. # last confirmed moment.
  421. logger.info("Shared-metrics sending disabled mid-pass; stopping")
  422. self._reconcile(send_enabled=False)
  423. break
  424. try:
  425. package = self._claim_next(self._now(), seen)
  426. except Exception:
  427. logger.warning("Unable to select shared-metrics packages", exc_info=True)
  428. break
  429. if package is None:
  430. break
  431. seen.add(package["package_id"])
  432. if package.get("skip"):
  433. # Unusable row already marked rejected during the claim.
  434. result = "rejected"
  435. else:
  436. try:
  437. result = self._send_one(package)
  438. except Exception:
  439. logger.warning("Unable to send shared-metrics package", exc_info=True)
  440. result = "deferred"
  441. setattr(outcome, result, getattr(outcome, result) + 1)
  442. return outcome
  443. def _reconcile(self, *, send_enabled: bool) -> None:
  444. """Run the single consent writer from within a pass."""
  445. try:
  446. with self._write() as connection:
  447. reconcile_send_consent(connection, send_enabled, now=self._now())
  448. except Exception:
  449. logger.warning("Unable to reconcile shared-metrics consent", exc_info=True)
  450. def _still_consented(self) -> bool:
  451. """Re-read profile-owned send consent: docs promise ``send: false`` stops transmission
  452. immediately, and a pass can run for minutes."""
  453. if self._consent_check is None:
  454. return True
  455. try:
  456. return bool(self._consent_check())
  457. except Exception:
  458. # Fail CLOSED: if consent cannot be established, do not transmit.
  459. logger.warning("Unable to confirm shared-metrics send consent; stopping", exc_info=True)
  460. return False