update_cmd_stale_survivors.py 3.3 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970
  1. """Post-update escalation for gateways the fleet matrix proved stale.
  2. ``_verify_fleet_after_update`` compares every live gateway's stamped ``code_sha`` against the
  3. fresh checkout. A ``stale`` row once only failed the update (exit 1) and left the process
  4. running pre-update modules: its cron ticker then yields every tick to the "fresh gateway" it
  5. assumes exists, and nothing ever restarts it (#117275). A proven-stale survivor is now handed to
  6. the same drain-first ``request_restart`` path (SIGUSR1) the restart phase uses — a supervised
  7. gateway respawns on the new code, a bare ``gateway run`` stops and is listed for a manual restart.
  8. """
  9. from __future__ import annotations
  10. import logging
  11. logger = logging.getLogger(__name__)
  12. def stale_fleet_survivor_pids(fleet: list, already_signalled: set) -> list[int]:
  13. """Live PIDs the fleet matrix stamped ``stale`` that the restart phase never touched."""
  14. pids: list[int] = []
  15. for row in fleet or []:
  16. pid = row.get("pid") if isinstance(row, dict) else None
  17. if row.get("state") == "stale" and isinstance(pid, int) and pid > 0 and pid not in already_signalled:
  18. pids.append(pid)
  19. return pids
  20. def signal_stale_fleet_survivors(fleet: list, restart, drain_budget: float) -> list[int]:
  21. """Drain-first restart every proven-stale gateway; returns the PIDs signalled.
  22. Bookkeeping lands in ``restart.killed_pids`` so the receipt and the survivor sweep see them.
  23. Never raises: verification must still finalize the receipt (with an owed restart).
  24. """
  25. from hermes_cli.update_cmd_fleet import _drain_or_signal_gateway_for_update
  26. pids = stale_fleet_survivor_pids(fleet, set(restart.killed_pids))
  27. if not pids:
  28. return []
  29. try:
  30. from hermes_cli.gateway import _get_service_pids
  31. service_pids = set(_get_service_pids(all_profiles=True))
  32. except Exception:
  33. service_pids = set()
  34. labels = {row.get("pid"): str(row.get("profile") or "gateway") for row in fleet if isinstance(row, dict)}
  35. print()
  36. print(f" ⚠ {len(pids)} gateway process(es) still run the pre-update code — requesting a restart")
  37. signalled: list[int] = []
  38. manual: list[int] = []
  39. for pid in pids:
  40. label = f"{labels.get(pid, 'gateway')} (PID {pid})"
  41. try:
  42. if _drain_or_signal_gateway_for_update(pid, drain_budget, label):
  43. signalled.append(pid)
  44. restart.killed_pids.add(pid)
  45. if pid not in service_pids:
  46. manual.append(pid)
  47. except Exception as exc:
  48. logger.warning("Could not signal stale gateway PID %s: %s", pid, exc)
  49. print(f" ⚠ {label}: could not be signalled ({exc}) — restart it by hand")
  50. if manual:
  51. # No successor: the receipt's gateway_restart follow-up names them and the fleet
  52. # obligation stays armed, so the stop is never silent (codemap §6 V7).
  53. stopped = getattr(restart, "stopped_unmapped_pids", None)
  54. if isinstance(stopped, set):
  55. stopped.update(manual)
  56. print(f" → Stopped {len(manual)} manual gateway process(es) that had no supervisor to respawn them")
  57. print(" Restart manually: hermes gateway run")
  58. if len(manual) > 1:
  59. print(" (or: hermes -p <profile> gateway run for each profile)")
  60. return signalled