process_registry_checkpoint.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162
  1. """Running-process checkpoint persistence and PID-safe recovery."""
  2. import json
  3. import logging
  4. import time
  5. from typing import Any, Dict, List, Optional
  6. from agent.redact import redact_sensitive_text
  7. logger = logging.getLogger("tools.process_registry")
  8. class ProcessCheckpointMixin:
  9. def _detached_host_fate(self, pid: Optional[int], expected_start: Optional[int]) -> str:
  10. """Subclass supplies the live PID decision. See ProcessRegistry."""
  11. raise NotImplementedError
  12. # ----- Checkpoint (crash recovery) -----
  13. def _write_checkpoint(self, extra_entries: Optional[list[dict[str, Any]]] = None):
  14. """Write running process metadata to the checkpoint file atomically."""
  15. from tools.process_registry import _checkpoint_path, _CHECKPOINT_FIELDS
  16. try:
  17. with self._lock:
  18. entries = []
  19. for s in self._running.values():
  20. if s.exited:
  21. continue
  22. # Backfill the start time so recovery can detect PID recycling
  23. # even for sessions spawned before this field existed.
  24. if s.host_start_time is None and s.pid_scope == "host" and s.pid:
  25. s.host_start_time = self._safe_host_start_time(s.pid)
  26. entry = {"session_id": s.id, **{f: getattr(s, f) for f in _CHECKPOINT_FIELDS}}
  27. # Redact inline credentials before persisting (~/.hermes/processes.json).
  28. # Recovery uses command only for display (adoption re-validates the
  29. # PID, never re-runs it), so masking is lossless.
  30. # See #77484.
  31. entry["command"] = redact_sensitive_text(s.command, code_file=True)
  32. entry["owner_task_id"] = s.owner_task_id or s.task_id
  33. entries.append(entry)
  34. if extra_entries:
  35. tracked_ids = {item.get("session_id") for item in entries}
  36. entries.extend(item for item in extra_entries if item.get("session_id") not in tracked_ids)
  37. known = {*self._running, *self._finished}
  38. from hermes_cli.active_sessions import _FileLock
  39. from utils import atomic_json_write
  40. path = _checkpoint_path()
  41. # Every interpreter of the profile (owner, each managed worker) rewrites this file from
  42. # its own table; a worker's output-log handoff is its only path to the next one, so a
  43. # writer keeps the handoffs it does not track (read-merge-write under one file lock).
  44. with _FileLock(path.with_name(path.name + ".lock")):
  45. entries.extend(self._foreign_handoffs(path, known))
  46. atomic_json_write(path, entries)
  47. except Exception as e:
  48. logger.debug("Failed to write checkpoint file: %s", e, exc_info=True)
  49. def _foreign_handoffs(self, path, known) -> list[dict[str, Any]]:
  50. """On-disk worker handoffs this registry never tracked whose log still decides an outcome:
  51. the process is alive (same start time) or its recorded exit is waiting to be read."""
  52. from pathlib import Path
  53. from tools.process_registry_output_log import _exit_path, adoptable_log
  54. try:
  55. on_disk = json.loads(path.read_text(encoding="utf-8-sig"))
  56. except (OSError, ValueError):
  57. return []
  58. return [entry for entry in on_disk if isinstance(entry, dict) and entry.get("session_id") not in known
  59. and (log := adoptable_log(entry.get("output_log"))) and Path(log).exists()
  60. and (_exit_path(log).exists() or self._detached_host_fate(
  61. entry.get("pid"), entry.get("host_start_time")) == "running")]
  62. def recover_from_checkpoint(self) -> int:
  63. """On gateway startup, probe PIDs from the checkpoint file; returns how many
  64. were recovered as detached sessions."""
  65. from tools.process_registry import (
  66. ProcessSession, _CHECKPOINT_FIELDS, _checkpoint_path,
  67. _CHECKPOINT_DEFAULTS, _WATCHER_ROUTE_KEYS, _stop_systemd_unit,
  68. )
  69. from tools.process_registry_output_log import adoptable_log
  70. # This host adopts what a crashed predecessor left running, so the jobs it starts must
  71. # survive its own crash for the next one: they stay out of the owner-death supervisor.
  72. self._jobs_outlive_host = True
  73. checkpoint_path = _checkpoint_path()
  74. if not checkpoint_path.exists():
  75. return 0
  76. try:
  77. entries = json.loads(checkpoint_path.read_text(encoding="utf-8-sig"))
  78. except Exception:
  79. return 0
  80. recovered = 0
  81. unresolved_scope_entries: list[dict[str, Any]] = []
  82. for entry in entries:
  83. pid, pid_scope = entry.get("pid"), entry.get("pid_scope", "host")
  84. if not pid:
  85. continue
  86. # The registry is process-global, so every profile's checkpoint carries every live
  87. # process; a multiplexer recovering several homes must adopt each session once.
  88. with self._lock:
  89. already_tracked = entry.get("session_id") in self._running
  90. if already_tracked:
  91. continue
  92. if pid_scope != "host": # in-sandbox PIDs mean nothing once the env handle is gone
  93. logger.info(
  94. "Skipping recovery for non-host process: %s (pid=%s, scope=%s)",
  95. entry.get("command", "unknown")[:60], pid, pid_scope)
  96. continue
  97. # Alive and still ours: re-attach. A start-time probe that cannot
  98. # be read is not proof the PID was reused — dropping it would leave
  99. # a live child unsupervised, and marking it exited would invent a
  100. # completion. A positive mismatch means the number was recycled:
  101. # do not adopt it and do not signal it.
  102. fate = self._detached_host_fate(pid, entry.get("host_start_time"))
  103. # A worker spawn's output log and recorded exit outlive both the spawner and the PID,
  104. # so its reader decides the outcome (a recycled PID is never signalled: fate != running).
  105. log = adoptable_log(entry.get("output_log"))
  106. if log:
  107. fate = "running"
  108. if fate == "reused":
  109. logger.info(
  110. "Not recovering session %s: pid %d is alive but its "
  111. "start time no longer matches — PID was recycled onto "
  112. "an unrelated process; refusing to adopt or signal it.",
  113. entry.get("session_id", "?"), pid)
  114. continue
  115. if fate != "running":
  116. systemd_unit = entry.get("systemd_unit", "")
  117. if systemd_unit and not _stop_systemd_unit(systemd_unit):
  118. logger.warning(
  119. "Could not reap persisted scope %s for dead wrapper pid %s; "
  120. "retaining checkpoint entry for the next startup",
  121. systemd_unit, pid)
  122. unresolved_scope_entries.append(entry)
  123. continue
  124. fields = {f: entry.get(f, _CHECKPOINT_DEFAULTS[f]) for f in _CHECKPOINT_FIELDS}
  125. fields.update(
  126. command=entry.get("command", "unknown"),
  127. owner_task_id=entry.get("owner_task_id", "") or entry.get("task_id", ""),
  128. started_at=entry.get("started_at", time.time()))
  129. # detached: no Popen handle, but can report status + kill; a worker spawn's output log
  130. # (and recorded exit) outlives its spawner, so tail it from the start.
  131. fields["output_log"] = log
  132. session = ProcessSession(id=entry["session_id"], detached=True, **fields)
  133. if session.output_log:
  134. self._track_started(session, self._log_reader_loop, f"proc-log-reader-{session.id}")
  135. else:
  136. with self._lock:
  137. self._running[session.id] = session
  138. recovered += 1
  139. logger.info("Recovered detached process: %s (pid=%d)", session.command[:60], pid)
  140. # Re-enqueue watcher so gateway can resume notifications
  141. if session.watcher_interval > 0:
  142. self.pending_watchers.append({
  143. "session_id": session.id,
  144. "check_interval": session.watcher_interval,
  145. "session_key": session.session_key,
  146. **{key: getattr(session, f"watcher_{key}") for key in _WATCHER_ROUTE_KEYS},
  147. "notify_on_complete": session.notify_on_complete,
  148. "parent_session_id": session.parent_session_id,
  149. })
  150. self._write_checkpoint(extra_entries=unresolved_scope_entries)
  151. return recovered