| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458 |
- """Shared-metrics facts for ``hermes update`` runs and Desktop self-updates.
- ``hermes.update.run`` / ``hermes.update.stage`` are DERIVED from the final update receipt in the
- process that finalizes it (``update_receipt.finalize_update_receipt``): the receipt already carries
- the outcome, the stage marks with timestamps, the admission refusal and the fleet matrix, so no
- stage is instrumented for metrics. A pre-pull interpreter must never import pulled code, so when
- it is the finalizer it parks the receipt under the store dir (stdlib-only code in update_receipt)
- and :func:`report_pending_updates` records it on the next Hermes start.
- Desktop's packaged updaters (electron-updater, App Installer, Store) never run ``hermes update``;
- Desktop reports their outcome through the ``shared_metrics.update_run`` RPC instead. Desktop's
- source-checkout hand-off DOES run ``hermes update``; that receipt is tagged ``initiator=desktop``
- and counted here, never by the RPC, so no run is counted twice.
- Run and stage rows agree on where a FAILED run stopped: a run whose ``failed_stage`` names a stage it
- never marked (it exited inside it, before the stage's END mark) also gets one failed stage row for it.
- A run that passed the commit point and only owes follow-ups (contract C3) is a ``success`` run, while
- the stage that failed (``deps``, ``build``, ``restart``, ``verify``) still reads ``failed`` at stage level.
- """
- from __future__ import annotations
- import builtins
- import json
- import logging
- import os
- import re
- import shutil
- import time
- from datetime import datetime
- from pathlib import Path
- from typing import Any
- logger = logging.getLogger(__name__)
- PENDING_DIRNAME = "pending_updates"
- # One empty file per update_id already counted in this profile: the completion child and the
- # parent's parked copy can both finalize the same run.
- RECORDED_DIRNAME = "recorded_updates"
- _RECORDED_KEEP = 64
- _UPDATE_ID = re.compile(r"[0-9a-f]{8,64}")
- # Receipt pm/Desktop outcome words → hermes.update.run outcome; anything else failed.
- _RUN_OUTCOMES = {"success": "success", "refused": "refused", "noop": "noop"}
- _FLEET_BAD_STATES = frozenset({"stale", "down"})
- # Desktop mechanism (updater strategy kind) → apply_mode.
- _DESKTOP_APPLY_MODES = {
- "electron-updater": "package", "app-installer": "package", "microsoft-store": "package",
- "windows-handoff": "git", "posix-handoff": "git",
- }
- # ---- iuf c1 ----
- # ``finalize_pending_update_receipt`` stores ``f"{type(exc).__name__}: {exc}"`` as the stop reason of
- # a run that ended on an exception (main.cmd_update, update_completion._finish). Only the leading
- # type name is read, never the message; other stop reasons ("sys.exit(1)", "completion exited 1",
- # "Windows gateway recovery failed: ...") do not start with a bare CamelCase name and a colon
- # (the fixed prefixes Hermes writes are matched by _STOP_REASON_PREFIX_CLASSES).
- _EXCEPTION_STOP_REASON = re.compile(r"([A-Z][A-Za-z0-9_]*):(?: |$)")
- # pm's own failure types (pm/package.py, pm/environment.py, pm/downloader.py, pm/lock.py, ...).
- _PM_ERROR_TYPES = frozenset({
- "BuildFailure", "DownloadError", "DownloadPaused", "DownloadTransportError", "FeatureProbeError",
- "HashError", "InstallError", "PinMismatch", "ResolutionConflict", "StaleLockRow",
- })
- # Built-in OSError and its subclasses (disk full, permissions, a file held open on Windows).
- _OS_ERROR_TYPES = frozenset(
- name for name, value in vars(builtins).items() if isinstance(value, type) and issubclass(value, OSError))
- # subprocess.SubprocessError and its subclasses that reached the command boundary uncaught.
- _SUBPROCESS_ERROR_TYPES = frozenset({"CalledProcessError", "SubprocessError", "TimeoutExpired"})
- # Exception type name -> class, first match wins; any other type reads ``exception``.
- _EXCEPTION_TYPE_CLASSES = (
- (frozenset({"PermissionError"}), "permission_denied"), (_PM_ERROR_TYPES, "deps_failed"),
- (_OS_ERROR_TYPES, "os_error"), (_SUBPROCESS_ERROR_TYPES, "subprocess_failed"),
- )
- # Fixed stop-reason prefixes Hermes itself writes (only the prefix is read, never what follows):
- # _update_takeover's preparation failure and update_completion's Windows gateway resume failure.
- _STOP_REASON_PREFIX_CLASSES = (
- ("historical takeover preparation failed", "deps_failed"), ("Windows gateway recovery failed", "restart_failed"),
- )
- # An OSError that ended the run, read by its errno token alone (never the text after it):
- # ENOSPC (28 on every platform), Windows ERROR_HANDLE_DISK_FULL (39) / ERROR_DISK_FULL (112).
- _DISK_FULL_ERRNO = re.compile(r"[A-Z][A-Za-z0-9_]*: \[(?:Errno 28|WinError (?:39|112))\]")
- # ---- end iuf c1 ----
- # ---- iuf c1 ----
- def _fleet_bad(receipt: dict[str, Any]) -> bool:
- fleet = receipt.get("fleet")
- return isinstance(fleet, list) and any(
- isinstance(row, dict) and row.get("state") in _FLEET_BAD_STATES for row in fleet)
- def _restart_incomplete(receipt: dict[str, Any]) -> bool:
- restart = receipt.get("gateway_restart")
- if isinstance(restart, dict) and (
- restart.get("incomplete") is True or restart.get("phase_error") or restart.get("failed_units")):
- return True
- outcomes = receipt.get("runtime_outcomes")
- return isinstance(outcomes, list) and any(
- isinstance(row, dict) and row.get("outcome") == "failed" for row in outcomes)
- def _named_stop(receipt: dict[str, Any], stages: list[dict[str, Any]]) -> str | None:
- """The class the run named itself: the user's parked changes on a committed run, or the closed
- ``stop_class`` a pre-apply exit recorded (read only while nothing was applied)."""
- from .shared_metrics_contract import UPDATE_STOP_CLASSES
- if receipt.get("outcome") == "partial" and receipt.get("user_action"):
- # Committed; only the user's stashed changes are owed (record_user_action, exit 1). Since C3
- # this is the only ``partial`` a receipt gets; older releases' verification wrote it too.
- return "local_changes_parked"
- stop_class = receipt.get("stop_class")
- if stop_class in UPDATE_STOP_CLASSES and not any(s["name"] == "apply" for s in stages):
- return stop_class
- return None
- def update_failure_class(receipt: dict[str, Any], stages: list[dict[str, Any]], outcome: str) -> str:
- """Why a failed/refused/partial run stopped, from fields the FINAL receipt already carries.
- A pre-apply exit names itself (``stop_class``, recorded on the line before the exit by
- update_receipt.record_stop_reason); only a run with no apply mark reads it, so a reason can
- never outlive the exit that recorded it. Parked copies (update_receipt._metric_receipt) carry
- every field read here except the free text; copies parked by older releases have no ``schema``.
- """
- if outcome not in {"failed", "refused"}:
- return "none"
- raw_steps = receipt.get("steps")
- steps: list[Any] = raw_steps if isinstance(raw_steps, list) else []
- if any(isinstance(s, dict) and s.get("name") == "admission" and not s.get("ok") for s in steps):
- return "managed_install"
- raw_reason = receipt.get("stop_reason")
- reason: str = raw_reason if isinstance(raw_reason, str) else ""
- exit_code = receipt.get("exit_code")
- match = _EXCEPTION_STOP_REASON.match(reason)
- exc_type = match.group(1) if match else ""
- if exc_type == "KeyboardInterrupt" or exit_code == 130:
- return "interrupted"
- if named := _named_stop(receipt, stages):
- return named
- if outcome == "refused":
- # Exit 2 at the command boundary is the updater's refusal convention; on main the only
- # open-receipt exit 2 is another updater holding the lock (update_finish).
- return "lock_held" if exit_code == 2 else "other"
- failed_marks = {s["name"] for s in stages if s.get("outcome") == "failed"}
- if "build" in failed_marks:
- return "build_failed"
- if _fleet_bad(receipt):
- return "fleet_stale"
- if "restart" in failed_marks or _restart_incomplete(receipt):
- return "restart_failed"
- if receipt.get("outcome") == "partial":
- return "fleet_unverified" # verification failed with no stale/down row and no failed restart
- if stages and stages[-1]["name"] == "restart" and stages[-1].get("outcome") == "skipped":
- return "restart_failed" # a skipped restart left the fleet owing one (completion exit 1)
- if by_reason := next((cls for prefix, cls in _STOP_REASON_PREFIX_CLASSES if reason.startswith(prefix)), None):
- return by_reason
- if _DISK_FULL_ERRNO.match(reason):
- return "disk_full"
- if exc_type:
- return next((cls for types, cls in _EXCEPTION_TYPE_CLASSES if exc_type in types), "exception")
- marked = {s["name"] for s in stages}
- if "apply" in marked:
- if "deps" not in marked:
- return "deps_failed" # the PM preparation child never reached --prepared
- if "build" not in marked:
- return "build_failed"
- return "other"
- if "schema" not in receipt:
- # A parked copy (update_receipt._metric_receipt): parked only when the checkout already
- # moved under the pre-pull interpreter, and it carries no stop reason or exit code.
- return "other"
- # update_cmd._handle_update_called_process_error is the one finalize("failed") with neither a
- # stop reason nor an exit code: a git/installer subprocess failed before the checkout moved.
- if not reason and exit_code is None:
- return "git_failed"
- return "aborted_before_apply"
- # ---- end iuf c1 ----
- def _epoch(value: Any) -> float | None:
- try:
- if isinstance(value, (int, float)) and not isinstance(value, bool):
- return float(value)
- return datetime.fromisoformat(str(value)).timestamp()
- except (OverflowError, TypeError, ValueError):
- return None
- def _elapsed_ms(start: Any, end: Any) -> float | None:
- first, last = _epoch(start), _epoch(end)
- return None if first is None or last is None or last < first else (last - first) * 1000
- def _receipt_stages(receipt: dict[str, Any]) -> list[dict[str, Any]]:
- """Stage marks in order, each with a duration since the previous mark (or the run start)."""
- from .shared_metrics_contract import UPDATE_STAGES
- stages: list[dict[str, Any]] = []
- previous = receipt.get("started_at")
- for mark in receipt.get("stages") or ():
- if not isinstance(mark, dict) or mark.get("name") not in UPDATE_STAGES:
- continue
- stages.append({**mark, "duration_ms": _elapsed_ms(previous, mark.get("at"))})
- previous = mark.get("at") or previous
- fleet = receipt.get("fleet")
- # The fleet matrix is only passed to finalize by the post-restart verification.
- if isinstance(fleet, list) and fleet and not any(s["name"] == "verify" for s in stages):
- # ---- iuf c1 ----
- # ``partial`` is written only by update_cmd_fleet._verify_fleet_after_update, which also
- # writes it for a failed build or restart; with no failed stage mark, verification itself
- # failed even when every fleet row reads current (zero rows expected, an unaccounted
- # runtime, a dashboard that did not come back, a Windows resume failure).
- bad = _fleet_bad(receipt) or (
- receipt.get("outcome") == "partial" and not any(s.get("outcome") == "failed" for s in stages))
- # ---- end iuf c1 ----
- stages.append({
- "name": "verify", "outcome": "failed" if bad else "success",
- "duration_ms": _elapsed_ms(previous, receipt.get("finished_at")),
- })
- return stages
- def _failed_stage(stages: list[dict[str, Any]]) -> str:
- """Where a failed/refused run stopped: the failing stage, else the stage it never reached."""
- from .shared_metrics_contract import UPDATE_STAGE_ORDER
- if not stages:
- return "other"
- last = stages[-1]
- terminal = last["name"] == "verify" or (last["name"] == "restart" and last.get("outcome") == "skipped")
- if last.get("outcome") == "failed" or terminal:
- failed = [s["name"] for s in stages if s.get("outcome") == "failed"]
- # ---- iuf c1 ----
- # A failed run whose last mark is a skipped restart failed AT the restart: the completion
- # exits 1 when a skipped restart leaves the fleet owing one, or the Windows resume fails.
- return failed[-1] if failed else last["name"]
- # ---- end iuf c1 ----
- index = UPDATE_STAGE_ORDER.index(last["name"])
- return UPDATE_STAGE_ORDER[index + 1] if index + 1 < len(UPDATE_STAGE_ORDER) else "other"
- def _apply_mode(receipt: dict[str, Any], stages: list[dict[str, Any]]) -> str:
- for stage in stages:
- if stage["name"] == "apply" and stage.get("mode") in {"git", "zip"}:
- return stage["mode"]
- # An admission refusal means the install is owned by something else (Docker, Nix, a package).
- if any(isinstance(s, dict) and s.get("name") == "admission" and not s.get("ok") for s in receipt.get("steps") or ()):
- return "external"
- return "unknown"
- def _committed_with_work_owed(receipt: dict[str, Any]) -> bool:
- """The code moved (an apply mark that did not fail), and the run closed owing work, not failed:
- interrupted after the commit point (finalize_interrupted_update_receipt) or the user's parked
- changes (record_user_action -> ``partial``). Follow-ups alone already finalize ``success`` (C3)."""
- applied = any(isinstance(mark, dict) and mark.get("name") == "apply" and mark.get("outcome") != "failed"
- for mark in receipt.get("stages") or ())
- owed = receipt.get("outcome") == "interrupted" or (
- receipt.get("outcome") == "partial" and bool(receipt.get("user_action")))
- return applied and owed
- def update_receipt_fields(receipt: dict[str, Any]) -> tuple[dict[str, str], list[dict[str, str]]] | None:
- """Bounded hermes.update.run + hermes.update.stage dimensions for one FINAL receipt."""
- from .shared_metrics_contract import update_duration_bucket, version_age_bucket
- if not isinstance(receipt, dict) or not receipt.get("finished_at"):
- return None
- stages = _receipt_stages(receipt)
- outcome = _RUN_OUTCOMES.get(str(receipt.get("outcome") or ""), "failed")
- if outcome == "success" and any(s["name"] == "apply" and s.get("outcome") == "skipped" for s in stages):
- outcome = "noop"
- if _committed_with_work_owed(receipt):
- outcome = "partial"
- started = _epoch(receipt.get("started_at"))
- committed = _epoch((receipt.get("pre_update") or {}).get("commit_date"))
- age_ms = None if started is None or committed is None else (started - committed) * 1000
- failure_class = update_failure_class(receipt, stages, "failed" if outcome == "partial" else outcome)
- run = {
- "apply_mode": _apply_mode(receipt, stages),
- "duration_bucket": update_duration_bucket(_elapsed_ms(receipt.get("started_at"), receipt.get("finished_at"))),
- "failed_stage": (
- # The parked autostash is the apply's last step (update_cmd._pull_updates restores it).
- "apply" if failure_class == "local_changes_parked"
- else _failed_stage(stages) if outcome in {"failed", "refused", "partial"} else "none"),
- "from_version_age_bucket": version_age_bucket(age_ms),
- "kind": "desktop" if receipt.get("initiator") == "desktop" else "cli",
- "outcome": outcome,
- # ---- iuf c1 ----
- "failure_class": failure_class,
- # ---- end iuf c1 ----
- }
- if run["outcome"] == "refused" and not stages:
- run["failed_stage"] = "none"
- stage_rows = [
- {
- "duration_bucket": update_duration_bucket(stage["duration_ms"]),
- "outcome": stage.get("outcome") if stage.get("outcome") in {"success", "failed", "skipped"} else "failed",
- "stage": stage["name"],
- }
- for stage in stages
- ]
- if died_in := _unmarked_failed_stage(run, stages):
- last_mark = next((s["at"] for s in reversed(stages) if s.get("at")), receipt.get("started_at"))
- stage_rows.append({
- "duration_bucket": update_duration_bucket(_elapsed_ms(last_mark, receipt.get("finished_at"))),
- "outcome": "failed", "stage": died_in,
- })
- return run, stage_rows
- def _unmarked_failed_stage(run: dict[str, str], stages: list[dict[str, Any]]) -> str | None:
- """The stage a failed run stopped in when the receipt holds no mark for it, else None.
- Stage marks are END marks, so a run that exits inside a stage (every pre-apply ``sys.exit``:
- fetch, channel, branch, merge, HEAD checks) leaves none for it and the stage rows would show no
- failure where the run row says it died. Only ``failed`` runs: a refusal is not a stage failure,
- and a committed run that owes follow-ups is a ``success`` whose failed stage already has its mark.
- """
- from .shared_metrics_contract import UPDATE_STAGES
- died_in = run["failed_stage"]
- if run["outcome"] != "failed" or died_in not in UPDATE_STAGES or any(s["name"] == died_in for s in stages):
- return None
- return died_in
- def _collection_on() -> bool:
- """Cheap pre-gate so a disabled install never loads the Relay runtime; `_emit` re-checks."""
- from hermes_cli.config import read_raw_config_readonly
- config: Any = read_raw_config_readonly() or {}
- for key in ("telemetry", "shared_metrics"):
- config = config.get(key) if isinstance(config, dict) else None
- return isinstance(config, dict) and config.get("enabled") is True
- def _claim_update_id(update_id: Any) -> tuple[bool, Path | None]:
- """``(first, latch)``: first is False when this profile already counted ``update_id``."""
- from hermes_constants import get_hermes_home
- if not isinstance(update_id, str) or not _UPDATE_ID.fullmatch(update_id):
- return True, None # nothing stable to dedupe on
- directory = get_hermes_home() / "telemetry" / "shared_metrics" / RECORDED_DIRNAME
- directory.mkdir(parents=True, exist_ok=True)
- latch = directory / update_id
- try:
- os.close(os.open(latch, os.O_CREAT | os.O_EXCL | os.O_WRONLY))
- except FileExistsError:
- return False, None
- try:
- for stale in sorted(directory.iterdir(), key=lambda p: p.stat().st_mtime, reverse=True)[_RECORDED_KEEP:]:
- stale.unlink(missing_ok=True)
- except OSError: # a concurrent prune won; the next claim prunes again
- pass
- return True, latch
- def record_update_receipt(receipt: dict[str, Any], *, wait_saved: bool = False) -> bool:
- """Record one run row plus its stage rows from a final receipt, once per update_id.
- ``wait_saved`` (parked receipts) blocks until the rows are in the store; False means the caller
- must keep the receipt for a later retry. True also covers "nothing to record". Never raises.
- """
- try:
- if not _collection_on():
- return True
- from . import shared_metrics_contract as contract
- from .shared_metrics_events import _emit, emit_saved
- derived = update_receipt_fields(receipt)
- if derived is None:
- return True
- first, latch = _claim_update_id(receipt.get("update_id"))
- if not first:
- return True
- run, stage_rows = derived
- rows = [(contract.UPDATE_RUN_MARK, run), *((contract.UPDATE_STAGE_MARK, row) for row in stage_rows)]
- if not wait_saved:
- for mark, row in rows:
- _emit(mark, lambda row=row: row)
- return True
- if emit_saved(rows):
- return True # even partly: a retry would count again the rows that did land
- if latch is not None:
- latch.unlink(missing_ok=True) # nothing landed: let the retry count it
- return False
- except Exception:
- logger.debug("Update shared metrics not recorded", exc_info=True)
- return False
- def pending_updates_dir(home: Path) -> Path:
- return home / "telemetry" / "shared_metrics" / PENDING_DIRNAME
- def purge_pending_updates(home: Path) -> None:
- shutil.rmtree(pending_updates_dir(home), ignore_errors=True)
- def report_pending_updates() -> None:
- """Record receipts a pre-pull interpreter parked (it must not import pulled code). Never raises."""
- try:
- from hermes_constants import get_hermes_home
- from .shared_metrics_process import _claim, settle_claim
- directory = pending_updates_dir(get_hermes_home())
- if not directory.is_dir():
- return
- for path in sorted(directory.iterdir()):
- if path.name.startswith(".") or ".json" not in path.name:
- continue
- claimed = _claim(path) # a concurrent start that loses the rename records nothing
- if claimed is None:
- continue
- try:
- receipt = json.loads(claimed.read_text(encoding="utf-8-sig"))
- except (OSError, ValueError):
- receipt = None
- saved = not isinstance(receipt, dict) or record_update_receipt(receipt, wait_saved=True)
- settle_claim(claimed, path, saved)
- except Exception:
- logger.debug("Pending update shared metrics not reported", exc_info=True)
- def desktop_update_fields(
- *, outcome: Any, failed_stage: Any, duration_ms: Any, mechanism: Any, from_commit_date: Any = None,
- ) -> dict[str, str]:
- """hermes.update.run dims for a Desktop packaged self-update (RPC-reported)."""
- from .shared_metrics_contract import (
- DESKTOP_UPDATE_STAGES, UPDATE_OUTCOMES, update_duration_bucket, version_age_bucket,
- )
- outcome_value = str(outcome or "").strip().lower()
- outcome_value = outcome_value if outcome_value in UPDATE_OUTCOMES else "failed"
- stage = str(failed_stage or "").strip().lower()
- commit = _epoch(from_commit_date)
- # Desktop sends seconds since epoch (install stamp commitDate); age is measured now.
- age_ms = None if commit is None else (time.time() - commit) * 1000
- return {
- "apply_mode": _DESKTOP_APPLY_MODES.get(str(mechanism or "").strip().lower(), "unknown"),
- "duration_bucket": update_duration_bucket(duration_ms),
- "failed_stage": (stage if stage in DESKTOP_UPDATE_STAGES else "other") if outcome_value == "failed" else "none",
- "from_version_age_bucket": version_age_bucket(age_ms),
- "kind": "desktop",
- "outcome": outcome_value,
- # ---- iuf c1 ----
- # The Desktop RPC carries a stage, never a reason.
- "failure_class": "unknown" if outcome_value in {"failed", "refused"} else "none",
- # ---- end iuf c1 ----
- }
- def record_desktop_update(**raw: Any) -> None:
- """Emit one Desktop self-update run through the enabled() gate. Never raises."""
- from . import shared_metrics_contract as contract
- from .shared_metrics_events import _emit
- _emit(contract.UPDATE_RUN_MARK, desktop_update_fields, **raw)
|