update_serve_obligations.py 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158
  1. """Durable serve/dashboard restart reminders (manual serves, and unit-owned ones the update could not
  2. restart), independent of gateway restart receipts."""
  3. import json
  4. import logging
  5. import math
  6. import os
  7. import sys
  8. import tempfile
  9. from pathlib import Path
  10. from hermes_constants import get_hermes_home
  11. logger = logging.getLogger(__name__)
  12. def defer_manual_serve(runtime: dict, *, require_alive: bool = False) -> bool:
  13. """Transfer an identified manual runtime to its own durable restart reminder."""
  14. if runtime.get("kind") not in ("serve", "dashboard") or runtime.get("supervisor") != "manual-serve" or runtime.get("restart_via") != "respawn-argv":
  15. return False
  16. if not isinstance(runtime.get("detail"), dict):
  17. return False
  18. return _defer_identified(runtime, require_alive=require_alive)
  19. def _defer_identified(runtime: dict, *, extra: dict | None = None, require_alive: bool = False) -> bool:
  20. """Shared tail of the manual and unit deferrals: True when the incarnation is gone or its reminder
  21. (plus *extra* fields) was filed; False while it is still owed and nothing durable was written."""
  22. from hermes_cli.process_identity import _pid_alive_matches
  23. pid, created = runtime.get("pid"), runtime["detail"].get("create_time")
  24. if type(pid) is not int or pid <= 0:
  25. return False
  26. identified = type(created) in (int, float) and math.isfinite(created) and created > 0
  27. if not identified:
  28. # Without a recorded creation time no durable reminder can be filed (#116507);
  29. # only a provably dead pid discharges the row, anything less stays pending.
  30. return not require_alive and _pid_alive_matches(pid, None) is False
  31. try:
  32. alive = _pid_alive_matches(pid, created)
  33. if require_alive and alive is not True:
  34. return False
  35. if alive is False:
  36. return True
  37. _write_reminder({"kind": runtime["kind"], "profile": runtime.get("profile", "unknown"), "pid": pid,
  38. "create_time": created, **(extra or {})})
  39. return True
  40. except (OSError, ValueError, TypeError) as exc:
  41. logger.debug("Could not preserve serve restart obligation: %s", exc)
  42. return False
  43. def _write_reminder(row: dict) -> None:
  44. """Persist one reminder row, one immutable file per incarnation (no read/merge/write races
  45. between CLI startups). Raises OSError/ValueError/TypeError to the caller."""
  46. directory = get_hermes_home() / "serve_restart_pending"
  47. directory.mkdir(parents=True, exist_ok=True)
  48. target = directory / f"{row['pid']}-{float(row['create_time']).hex()}.json"
  49. temporary = None
  50. try:
  51. with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=directory, delete=False) as handle:
  52. temporary = Path(handle.name)
  53. json.dump(row, handle)
  54. handle.flush()
  55. os.fsync(handle.fileno())
  56. os.replace(temporary, target)
  57. finally:
  58. if temporary is not None:
  59. temporary.unlink(missing_ok=True)
  60. def is_unit_serve_row(row: object) -> bool:
  61. """A serve/dashboard row whose systemd unit owns it (MainPID), with the unit recorded (#141021)."""
  62. return (isinstance(row, dict) and row.get("kind") in ("serve", "dashboard") and row.get("supervisor") == "systemd"
  63. and isinstance(row.get("detail"), dict) and bool(row["detail"].get("systemd_unit")))
  64. def is_reminder_row(row: object) -> bool:
  65. """Rows the receipt-backed reminder flow carries: manual serves and unit-owned serves."""
  66. return is_unit_serve_row(row) or (
  67. isinstance(row, dict) and row.get("kind") in ("serve", "dashboard") and row.get("supervisor") == "manual-serve")
  68. def defer_unit_serve(runtime: dict) -> bool:
  69. """The unit-owned twin of :func:`defer_manual_serve`: file a durable reminder carrying the unit's
  70. restart command for a pre-update incarnation that is still alive. A unit-owned row moved off the
  71. manual-serve reminder when it was classified ``systemd`` (#141021), and the fleet obligation names
  72. only gateways, so without this a unit the update could not restart (no root, an aborted run,
  73. ``--no-gateway-restart``) is never mentioned again. True when the row is discharged (process gone)
  74. or the reminder was filed; False when it is still owed and nothing durable could be written."""
  75. from hermes_cli.main_dashboard import _systemd_restart_commands
  76. if not is_unit_serve_row(runtime):
  77. return False
  78. detail = runtime["detail"]
  79. return _defer_identified(runtime, extra={
  80. "restart": _systemd_restart_commands(str(detail["systemd_unit"]), detail.get("systemd_scope"))})
  81. def retain_receipt_manual_serves(receipt: dict) -> list[dict]:
  82. """Return transfers still owed so receipt rotation cannot discard failed writes.
  83. Covers manual serves and unit-owned serves (:func:`is_reminder_row`); the name predates the latter.
  84. ``carried_manual_serves`` is the durable running record's copy of the previous receipt's rows
  85. (update_receipt.begin): read it too, or the warning vanishes while an update runs and after a
  86. run killed before its ``plan`` stage.
  87. """
  88. plan = receipt.get("plan") or {}
  89. rows = list(plan.get("runtimes") or []) + list(receipt.get("pending_manual_serves") or [])
  90. rows += list(receipt.get("carried_manual_serves") or [])
  91. pending = []
  92. for row in rows:
  93. if not is_reminder_row(row):
  94. continue
  95. deferred = defer_unit_serve(row) if is_unit_serve_row(row) else defer_manual_serve(row)
  96. if not deferred and row not in pending:
  97. pending.append(row)
  98. return pending
  99. def warn_pending_manual_serves(*, startup: bool = False, pending_manual: list[dict] | None = None) -> None:
  100. """Warn about manual debt independently of gateway evidence; optionally reuse a snapshot's failed transfers."""
  101. from hermes_cli.main_dashboard import _format_restart_commands, _systemd_restart_commands
  102. from hermes_cli.process_identity import _pid_alive_matches
  103. from hermes_cli.update_receipt import read_latest_receipt
  104. stream = sys.stderr if startup else sys.stdout
  105. if pending_manual is None:
  106. pending_manual = retain_receipt_manual_serves(read_latest_receipt() or {})
  107. for row in pending_manual:
  108. unit_row = is_unit_serve_row(row)
  109. print(f" ⚠ {row['kind']} [{row.get('profile', 'unknown')}] pid {row.get('pid', 'unknown')}: {'unit' if unit_row else 'manual'} restart reminder could not be saved; restart remains pending in the update receipt.", file=stream)
  110. detail = row.get("detail") if isinstance(row.get("detail"), dict) else {}
  111. if unit_row:
  112. commands = _systemd_restart_commands(str(detail["systemd_unit"]), detail.get("systemd_scope"))
  113. print(f" Restart its unit: {_format_restart_commands(commands)}", file=stream)
  114. elif type(detail.get("create_time")) in (int, float):
  115. print(" Ask its owner to relaunch `hermes serve` / `hermes dashboard`; check reminder storage permissions and free space.", file=stream)
  116. else:
  117. # No usable creation time means identity, not storage, blocked the durable reminder.
  118. print(" This host could not read the process creation time, so no durable reminder could be filed; ask its owner to relaunch `hermes serve` / `hermes dashboard`, and the warning clears once the pid is confirmed gone.", file=stream)
  119. directory = get_hermes_home() / "serve_restart_pending"
  120. for path in sorted(directory.glob("*.json")):
  121. try:
  122. row = json.loads(path.read_text(encoding="utf-8-sig"))
  123. if _pid_alive_matches(row["pid"], row["create_time"]) is False:
  124. path.unlink(missing_ok=True)
  125. continue
  126. restart = row.get("restart")
  127. if isinstance(restart, list) and restart:
  128. print(f" ⚠ {row['kind']} [{row['profile']}] pid {row['pid']}: unit restart still pending; this process may still serve pre-update code.", file=stream)
  129. print(f" Restart its unit: {_format_restart_commands(restart)}", file=stream)
  130. continue
  131. print(f" ⚠ {row['kind']} [{row['profile']}] pid {row['pid']}: manual restart still pending; this process may still serve pre-update code.", file=stream)
  132. print(" Ask its owner to relaunch `hermes serve` / `hermes dashboard` (reconnect Desktop for an SSH backend).", file=stream)
  133. except (OSError, ValueError, KeyError, TypeError) as exc:
  134. logger.debug("Could not reconcile manual serve obligation %s: %s", path, exc)
  135. print(f" ⚠ Manual serve restart reminder could not be verified: {path.name}", file=stream)