shared_metrics_process.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. """hermes.process.exit: how Hermes processes (and watchdog-killed turns) end.
  2. A crash or SIGKILL cannot record itself. Each long-lived entrypoint drops a small marker under its
  3. profile's store dir at start and restamps it on the way out (``clean``, ``crash`` + class from the
  4. excepthook, ``watchdog`` from a hard-exit watchdog). The NEXT Hermes start in that profile reports
  5. every marker whose owner is gone; one still ``running`` whose pid fails the canonical start-time
  6. liveness check was killed. Reporting runs on a daemon thread so startup never waits on the Relay
  7. runtime; a claimed-but-unreported marker (the reporter died mid-way) is reclaimed later.
  8. """
  9. from __future__ import annotations
  10. import atexit
  11. import json
  12. import logging
  13. import os
  14. import shutil
  15. import sys
  16. import threading
  17. import time
  18. from pathlib import Path
  19. from typing import Any
  20. logger = logging.getLogger(__name__)
  21. MARKER_DIRNAME = "process_markers"
  22. _REPORTING = ".reporting"
  23. # Most specific first: ModuleNotFoundError is an ImportError, RecursionError a RuntimeError.
  24. _CRASH_CLASSES = (
  25. (MemoryError, "memory_error"), (ImportError, "import_error"), (OSError, "os_error"),
  26. (RuntimeError, "runtime_error"),
  27. )
  28. #: Ceiling on ``flush_before_hard_exit``: a wedged flush costs its pending rows, never the exit.
  29. EXIT_FLUSH_TIMEOUT_S = 3.0
  30. _DRAIN_MARGIN_S = 0.1
  31. _STATE: dict[str, Any] = {}
  32. _STATE_LOCK = threading.Lock()
  33. def crash_class(exc: BaseException | None) -> str:
  34. return next((label for kind, label in _CRASH_CLASSES if isinstance(exc, kind)), "other")
  35. def markers_dir(home: Path) -> Path:
  36. return home / "telemetry" / "shared_metrics" / MARKER_DIRNAME
  37. def _write_marker(path: Path, record: dict[str, Any]) -> None:
  38. tmp = path.with_name(f".{path.name}.tmp")
  39. tmp.write_text(json.dumps(record), encoding="utf-8")
  40. os.replace(tmp, path)
  41. def stamp_exit(exit_kind: str, crash: str = "none") -> None:
  42. """Restamp this process's marker with how it is ending. Never raises; no-op when unmarked."""
  43. try:
  44. with _STATE_LOCK:
  45. record = _STATE.get("record")
  46. path = _STATE.get("path")
  47. if record is None or path is None or record.get("state") != "running":
  48. return
  49. record = {**record, "state": exit_kind, "crash_class": crash}
  50. _STATE["record"] = record
  51. _write_marker(path, record)
  52. except Exception:
  53. logger.debug("Process exit marker not stamped", exc_info=True)
  54. def flush_before_hard_exit(timeout_s: float = EXIT_FLUSH_TIMEOUT_S) -> bool:
  55. """For a graceful stop about to ``os._exit``: run the shared-metrics exit flush its skipped atexit
  56. hooks would have run (close each live runtime's sessions under its own profile, write queued rows,
  57. and write every store's deferred rows, the root profile's host-level store included, each into
  58. the store that deferred it).
  59. Bounded by ``timeout_s`` on a daemon thread, because the Relay flush waits on every managed call
  60. still in flight and a wedged one must not hold the exit. Never imports the runtime module:
  61. collection off (or never started) means nothing to flush and nothing created. Never raises; True
  62. when the flush finished in time."""
  63. runtimes = sys.modules.get("hermes_cli.observability.relay_shared_metrics")
  64. stores = sys.modules.get("hermes_cli.observability.shared_metrics")
  65. if runtimes is None and stores is None:
  66. return True
  67. deadline = time.monotonic() + max(0.0, timeout_s)
  68. def _flush() -> None:
  69. if runtimes is not None:
  70. runtimes.shutdown_runtimes()
  71. if stores is not None: # every store that deferred a write, root/Desktop ones included
  72. stores.drain_deferring_stores(deadline - _DRAIN_MARGIN_S) # done before the join gives up
  73. try:
  74. from agent.memory_provider import spawn_context_thread
  75. worker = spawn_context_thread(_flush, name="hermes-shared-metrics-exit-flush")
  76. worker.start()
  77. worker.join(max(0.0, deadline - time.monotonic()))
  78. except Exception:
  79. logger.debug("Shared-metrics exit flush not started", exc_info=True)
  80. return False
  81. if worker.is_alive():
  82. logger.warning("Shared-metrics exit flush still running after %.1fs; exiting without it", timeout_s)
  83. return False
  84. return True
  85. def _excepthook(previous: Any) -> Any:
  86. def hook(exc_type: type[BaseException], exc: BaseException, tb: Any) -> None:
  87. if not issubclass(exc_type, (KeyboardInterrupt, SystemExit)):
  88. stamp_exit("crash", crash_class(exc))
  89. previous(exc_type, exc, tb)
  90. return hook
  91. def purge_pending_receipts(home: Path) -> None:
  92. """Collection is off: drop the files kept for a later start to count (parked update receipts,
  93. installer receipts). Every opt-out answer calls this too, not only an opted-out start, so a
  94. receipt never survives a "No" to be counted after a later opt-in. Never raises."""
  95. try:
  96. from .shared_metrics_install_run import purge_pending_installs
  97. from .shared_metrics_update import purge_pending_updates
  98. purge_pending_updates(home)
  99. purge_pending_installs(home)
  100. except Exception:
  101. logger.debug("Pending shared-metrics receipts not purged", exc_info=True)
  102. def begin_process(kind: str) -> None:
  103. """Mark this process as a running ``kind`` and report dead predecessors. Once per process."""
  104. try:
  105. from hermes_constants import get_hermes_home
  106. from .shared_metrics_desktop import ONBOARDING_LATCH_DIRNAME
  107. from .shared_metrics_setup import markers_dir as setup_markers_dir
  108. from .shared_metrics_update import _collection_on
  109. if _STATE:
  110. return
  111. _STATE["kind"] = kind # watchdog turn rows name the surface even when this home is off
  112. if not _collection_on():
  113. home = get_hermes_home()
  114. purge_pending_receipts(home) # parked while on / left by the installer, never counted once off
  115. latches = home / "telemetry" / "shared_metrics" / ONBOARDING_LATCH_DIRNAME # an opt-out outside Desktop
  116. for directory in (markers_dir(home), setup_markers_dir(home), latches): # likewise pending exits/setups
  117. shutil.rmtree(directory, ignore_errors=True)
  118. return
  119. from gateway.status import get_process_start_time
  120. home = get_hermes_home()
  121. directory = markers_dir(home)
  122. directory.mkdir(parents=True, exist_ok=True)
  123. pid = os.getpid()
  124. path = directory / f"{kind}-{pid}.json"
  125. record = {"kind": kind, "pid": pid, "start_time": get_process_start_time(pid), "state": "running"}
  126. with _STATE_LOCK:
  127. _write_marker(path, record)
  128. _STATE.update(path=path, record=record)
  129. sys.excepthook = _excepthook(sys.excepthook)
  130. atexit.register(stamp_exit, "clean")
  131. threading.Thread(
  132. target=_report_dead_markers, args=(home, path), name="hermes-exit-metrics", daemon=True,
  133. ).start()
  134. except Exception:
  135. logger.debug("Process exit marker not started", exc_info=True)
  136. def current_process_kind() -> str:
  137. return _STATE.get("kind") or "other"
  138. def _claimer_alive(path: Path) -> bool:
  139. from gateway.status import runtime_status_pid_is_live
  140. try:
  141. claimer = int(path.name.rsplit(".", 2)[-2])
  142. except (IndexError, ValueError):
  143. return False
  144. return runtime_status_pid_is_live({"pid": claimer})
  145. def _claim(path: Path) -> Path | None:
  146. """Rename a reportable marker so exactly one reporter owns it; None when not reportable."""
  147. from gateway.status import runtime_status_pid_is_live
  148. if path.name.endswith(_REPORTING):
  149. if _claimer_alive(path):
  150. return None
  151. record_path = path
  152. else:
  153. record_path = None
  154. try:
  155. record = json.loads(path.read_text(encoding="utf-8-sig"))
  156. except (OSError, ValueError):
  157. return None
  158. if not isinstance(record, dict):
  159. return None
  160. if record_path is None and record.get("state") == "running" and runtime_status_pid_is_live(record):
  161. return None
  162. claimed = path.with_name(f"{path.name.split('.json')[0]}.json.{os.getpid()}{_REPORTING}")
  163. try:
  164. os.replace(path, claimed)
  165. except OSError: # a concurrent reporter won
  166. return None
  167. return claimed
  168. def process_exit_fields(record: dict[str, Any]) -> dict[str, str]:
  169. from .shared_metrics_contract import CRASH_CLASSES, PROCESS_EXIT_KINDS, PROCESS_KINDS
  170. state = record.get("state")
  171. exit_kind = "killed" if state == "running" or state not in PROCESS_EXIT_KINDS else state
  172. crash = record.get("crash_class") if exit_kind == "crash" else "none"
  173. kind = record.get("kind")
  174. return {
  175. "crash_class": crash if crash in CRASH_CLASSES else "other",
  176. "exit_kind": exit_kind,
  177. "process_kind": kind if kind in PROCESS_KINDS else "other",
  178. }
  179. def settle_claim(claimed: Path, original: Path, saved: bool) -> None:
  180. """Delete a claimed file once its row is saved; otherwise hand it back for the next start."""
  181. try:
  182. if saved:
  183. claimed.unlink(missing_ok=True)
  184. else:
  185. os.replace(claimed, original)
  186. except OSError: # a leftover claim is reclaimed once this reporter is gone
  187. logger.debug("Claimed shared-metrics file not settled", exc_info=True)
  188. def _report_dead_markers(home: Path, own: Path) -> None:
  189. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  190. from . import shared_metrics_contract as contract
  191. from .shared_metrics_events import emit_saved
  192. token = set_hermes_home_override(home) # a thread does not inherit the profile binding
  193. try:
  194. for path in sorted(markers_dir(home).iterdir()):
  195. if path == own or path.name.startswith(".") or ".json" not in path.name:
  196. continue
  197. claimed = _claim(path)
  198. if claimed is None:
  199. continue
  200. try:
  201. record = json.loads(claimed.read_text(encoding="utf-8-sig"))
  202. except (OSError, ValueError):
  203. record = None
  204. rows = [(contract.PROCESS_EXIT_MARK, process_exit_fields(record))] if isinstance(record, dict) else []
  205. settle_claim(claimed, path, emit_saved(rows) == len(rows))
  206. from .shared_metrics_setup import report_abandoned_setups
  207. from .shared_metrics_update import report_pending_updates
  208. report_pending_updates()
  209. report_abandoned_setups(home)
  210. # ---- iuf c2 ----
  211. from .shared_metrics_install_run import report_pending_installs
  212. report_pending_installs(home)
  213. # ---- end iuf c2 ----
  214. except Exception:
  215. logger.debug("Dead process markers not reported", exc_info=True)
  216. finally:
  217. reset_hermes_home_override(token)
  218. def arm_turn(agent: Any) -> None:
  219. """At turn entry (profile scope bound): the owning home for an off-thread watchdog abort, and
  220. a fresh once-per-turn latch shared by both turn watchdogs."""
  221. try:
  222. from hermes_constants import get_hermes_home
  223. agent._metrics_turn_home = str(get_hermes_home())
  224. agent._metrics_watchdog_abort_counted = False
  225. except Exception:
  226. logger.debug("Turn exit metrics not armed", exc_info=True)
  227. def record_watchdog_turn_abort(agent: Any) -> None:
  228. """One exit_kind=watchdog row per turn a watchdog killed, in the turn's own profile. Never raises."""
  229. try:
  230. home = getattr(agent, "_metrics_turn_home", None)
  231. # No armed turn means no known owning profile: never guess the scope.
  232. if not home or getattr(agent, "_metrics_watchdog_abort_counted", True):
  233. return
  234. agent._metrics_watchdog_abort_counted = True
  235. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  236. from . import shared_metrics_contract as contract
  237. from .shared_metrics_events import _emit
  238. token = set_hermes_home_override(home)
  239. try:
  240. _emit(contract.PROCESS_EXIT_MARK, lambda: {
  241. "crash_class": "none", "exit_kind": "watchdog", "process_kind": current_process_kind(),
  242. })
  243. finally:
  244. reset_hermes_home_override(token)
  245. except Exception:
  246. logger.debug("Watchdog turn abort not recorded", exc_info=True)