shared_metrics_install_run.py 8.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200
  1. """hermes.install.run: fresh installs through scripts/install.sh and scripts/install.ps1.
  2. The installer runs before the user is asked about shared metrics and must never send anything, so a
  3. full-ladder run only leaves ONE small local receipt (closed tokens plus two epoch timestamps) under
  4. the profile's store dir. A later Hermes start that runs the process-exit reporter reads it:
  5. - collection off: ``shared_metrics_process.begin_process`` (and every opt-out answer) deletes every
  6. receipt unreported, the same rule as parked update receipts;
  7. - collection on, sending off: the row is recorded now (it stays on this machine);
  8. - collection on, sending on: the row is recorded only on a day whose package the sender's consent
  9. gate can ever pass, i.e. a send consent window opened at or before today 00:00Z. On the opt-in day
  10. itself (setup asked during the install) the receipt waits for the first start on a later day;
  11. - a receipt older than ``MAX_RECEIPT_AGE_SECONDS``, unparseable, or with any value outside the
  12. contract is deleted unreported (an old receipt would count under whatever version records it).
  13. """
  14. from __future__ import annotations
  15. import json
  16. import logging
  17. import os
  18. import re
  19. import shutil
  20. import sqlite3
  21. from datetime import datetime, timezone, UTC
  22. from pathlib import Path
  23. from typing import Any
  24. logger = logging.getLogger(__name__)
  25. PENDING_DIRNAME = "pending_installs"
  26. # One empty file per receipt id already counted: a delete that failed after the row was saved must
  27. # not count the same install again on the next start.
  28. RECORDED_DIRNAME = "recorded_installs"
  29. _RECORDED_KEEP = 64
  30. _RECEIPT_ID = re.compile(r"[0-9a-f]{32}")
  31. _RECEIPT_KEYS = frozenset({"id", "installer", "outcome", "failed_stage", "failure_class", "started_at", "finished_at"})
  32. # A receipt no start recorded within a week is dropped: the row would carry the recording client's
  33. # version, not the installer's, and per-version install failure rates would drift onto later releases.
  34. MAX_RECEIPT_AGE_SECONDS = 7 * 86_400
  35. def pending_installs_dir(home: Path) -> Path:
  36. return home / "telemetry" / "shared_metrics" / PENDING_DIRNAME
  37. def purge_pending_installs(home: Path) -> None:
  38. shutil.rmtree(pending_installs_dir(home), ignore_errors=True)
  39. def _epoch(value: Any) -> int | None:
  40. return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None
  41. def install_run_fields(receipt: Any) -> dict[str, str] | None:
  42. """hermes.install.run dims for one receipt, or None when anything is outside the contract."""
  43. from . import shared_metrics_contract as contract
  44. if not isinstance(receipt, dict) or set(receipt) != _RECEIPT_KEYS:
  45. return None
  46. if not isinstance(receipt["id"], str) or not _RECEIPT_ID.fullmatch(receipt["id"]):
  47. return None
  48. started, finished = _epoch(receipt["started_at"]), _epoch(receipt["finished_at"])
  49. if started is None or finished is None or finished < started:
  50. return None
  51. fields = {
  52. "duration_bucket": contract.update_duration_bucket((finished - started) * 1000),
  53. "failed_stage": receipt["failed_stage"],
  54. "failure_class": receipt["failure_class"],
  55. "installer": receipt["installer"],
  56. "outcome": receipt["outcome"],
  57. }
  58. if not contract.counter_dimensions_are_valid(contract.INSTALL_RUN_METRIC, fields):
  59. return None
  60. # A success names no stage and no class; a failure names both.
  61. succeeded = fields["outcome"] == "success"
  62. if succeeded != (fields["failed_stage"] == "none") or succeeded != (fields["failure_class"] == "none"):
  63. return None
  64. return fields
  65. def _recorded_latch(home: Path, receipt_id: str) -> Path:
  66. return home / "telemetry" / "shared_metrics" / RECORDED_DIRNAME / receipt_id
  67. def _mark_recorded(latch: Path) -> None:
  68. """Latch a receipt id AFTER its row is saved: a process that dies between the latch and the save
  69. (a short-lived start whose reporter thread is cut off at exit) must not drop the install."""
  70. try:
  71. latch.parent.mkdir(parents=True, exist_ok=True)
  72. latch.touch()
  73. for stale in sorted(latch.parent.iterdir(), key=lambda p: p.stat().st_mtime, reverse=True)[_RECORDED_KEEP:]:
  74. stale.unlink(missing_ok=True)
  75. except OSError: # a concurrent prune won; the next latch prunes again
  76. pass
  77. def _today_start() -> str:
  78. """Today's package ``period_start``, in the store's stamp format (``shared_metrics._isoformat``)."""
  79. from .shared_metrics import _isoformat, _utc_now
  80. return _isoformat(datetime.combine(_utc_now().date(), datetime.min.time(), tzinfo=UTC))
  81. def consented_day(home: Path) -> bool:
  82. """Whether a row recorded now lands in a package the user agreed to: collection on, and either
  83. sending off (it stays local) or a send consent window opened at or before today 00:00Z. The
  84. sender's ``CONSENT_GATE_SQL`` passes a day's package only when ``period_start >= opened_at``, so a
  85. row recorded on the opt-in day itself would sit in a package that is never sent."""
  86. from hermes_cli.config import read_raw_config_readonly
  87. from .shared_metrics_send_config import resolve_send_config
  88. send = resolve_send_config(read_raw_config_readonly() or {})
  89. if not send.enabled:
  90. return False
  91. if not send.send:
  92. return True
  93. from .shared_metrics import SharedMetricsStore
  94. root = home / "telemetry" / "shared_metrics"
  95. try:
  96. with SharedMetricsStore(root / "metrics.sqlite3", root / "outbox")._connection() as connection:
  97. row = connection.execute(
  98. "SELECT 1 FROM send_consent_windows WHERE closed_at IS NULL AND opened_at <= ? LIMIT 1",
  99. (_today_start(),),
  100. ).fetchone()
  101. except sqlite3.Error: # a busy store: try again on a later start
  102. return False
  103. return row is not None
  104. def _claim_receipt(path: Path) -> Path | None:
  105. """Rename a receipt so exactly one reporter owns it. Unlike the process-marker claim it never
  106. reads the file first: an unparseable receipt is claimed too, so it can be deleted."""
  107. from .shared_metrics_process import _REPORTING, _claimer_alive
  108. if path.name.endswith(_REPORTING) and _claimer_alive(path):
  109. return None
  110. claimed = path.with_name(f"{path.name.split('.json')[0]}.json.{os.getpid()}{_REPORTING}")
  111. try:
  112. os.replace(path, claimed)
  113. except OSError: # a concurrent reporter won
  114. return None
  115. return claimed
  116. def _stale(receipt: dict) -> bool:
  117. # The store's clock, the same one that picks the consented day: a second clock let a receipt
  118. # the day logic still considered fresh expire on wall time.
  119. from .shared_metrics import _utc_now
  120. return _utc_now().timestamp() - receipt["finished_at"] > MAX_RECEIPT_AGE_SECONDS
  121. def report_pending_installs(home: Path) -> None:
  122. """Record each receipt once, on a consented day (see the module docstring). Never raises."""
  123. try:
  124. from agent import relay_runtime
  125. from . import shared_metrics_contract as contract
  126. from .shared_metrics_events import emit_saved
  127. from .shared_metrics_process import settle_claim
  128. directory = pending_installs_dir(home)
  129. if not directory.is_dir():
  130. return
  131. # With Relay instrumentation off the store reports a row "settled" without recording it, which
  132. # would latch and delete the receipt uncounted: leave everything for a start that records.
  133. if not relay_runtime.relay_instrumentation_enabled():
  134. return
  135. reportable = consented_day(home)
  136. for path in sorted(directory.iterdir()):
  137. if path.name.startswith(".") or ".json" not in path.name:
  138. continue
  139. claimed = _claim_receipt(path) # a concurrent start that loses the rename records nothing
  140. if claimed is None:
  141. continue
  142. try:
  143. receipt = json.loads(claimed.read_text(encoding="utf-8-sig"))
  144. except (OSError, ValueError):
  145. receipt = None
  146. fields = install_run_fields(receipt)
  147. if fields is None or not isinstance(receipt, dict) or _stale(receipt): # never to be counted
  148. settle_claim(claimed, path, True)
  149. continue
  150. latch = _recorded_latch(home, receipt["id"])
  151. if latch.exists(): # counted before; only the delete had failed
  152. settle_claim(claimed, path, True)
  153. continue
  154. if not reportable: # kept for the first start on a fully consented day
  155. settle_claim(claimed, path, False)
  156. continue
  157. # The claim rename already keeps concurrent starts apart; a busy store keeps the receipt.
  158. saved = emit_saved([(contract.INSTALL_RUN_MARK, fields)]) == 1
  159. if saved:
  160. _mark_recorded(latch)
  161. settle_claim(claimed, path, saved)
  162. except Exception:
  163. logger.debug("Pending install receipts not reported", exc_info=True)