restart_loop_guard.py 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117
  1. """Auto-resume restart-loop breaker (defense-3).
  2. Defenses 1-2 (``_HERMES_GATEWAY`` guard on ``hermes gateway stop|restart`` /
  3. ``terminal_tool``, cron lifecycle filter) stop the agent scheduling its own restart
  4. but not every SIGTERM source: the supervisor respawns, the gateway auto-resumes the
  5. restart-interrupted session, whose next turn re-runs the offending logic. Boots are
  6. persisted to ``<HERMES_HOME>/gateway/restart_loop.json`` and CHAIN while gaps stay
  7. within ``max_gap_seconds`` (a ~150s watchdog-kill cycle trips like a ~10s loop).
  8. Tripped → caller SKIPS auto-resume. Any I/O failure fails OPEN, never wedging.
  9. """
  10. from __future__ import annotations
  11. import contextlib
  12. import json
  13. import logging
  14. import time
  15. from typing import List, Optional
  16. from hermes_constants import get_hermes_home
  17. logger = logging.getLogger("gateway.run")
  18. # A legitimate operator restart (or two) never trips; a ~10s respawn loop does
  19. # within a few cycles.
  20. DEFAULT_MAX_RESTARTS = 3
  21. DEFAULT_WINDOW_SECONDS = 60
  22. # Longest gap between consecutive restart-interrupted boots still counted as the
  23. # SAME loop. A fixed-window prune only sees cycles faster than the window (a slower
  24. # loop drops its history every boot and never trips); chaining on the inter-boot
  25. # gap is period-agnostic, and real quiet resets it.
  26. DEFAULT_MAX_GAP_SECONDS = 300
  27. # Only the newest ``max_restarts`` entries can change a verdict; the rest are forensics.
  28. _MAX_STORED_BOOTS = 50
  29. def _state_path():
  30. return get_hermes_home() / "gateway" / "restart_loop.json"
  31. def _load_boots() -> list[float]:
  32. try:
  33. # utf-8-sig: our BOM-tolerant read fix for the persisted boot log.
  34. raw = _state_path().read_text(encoding="utf-8-sig")
  35. data = json.loads(raw)
  36. boots = data.get("boots", [])
  37. return [float(t) for t in boots if isinstance(t, (int, float))]
  38. except (OSError, ValueError, TypeError):
  39. return []
  40. def _save_boots(boots: list[float]) -> None:
  41. with contextlib.suppress(OSError):
  42. path = _state_path()
  43. path.parent.mkdir(parents=True, exist_ok=True)
  44. path.write_text(json.dumps({"boots": boots}), encoding="utf-8")
  45. def _chain_gap(window_seconds: int, max_gap_seconds: int) -> float:
  46. """Inter-boot gap that still links two boots; floored by ``window_seconds`` so
  47. widening the window never makes the breaker *less* sensitive."""
  48. return float(max(1, window_seconds, max_gap_seconds))
  49. def _chain_ending_at(boots: list[float], ts: float, gap: float) -> list[float]:
  50. """Unbroken chain of boots leading up to ``ts`` (oldest first): walks backwards
  51. while each gap stays within ``gap``; the first wider gap ends the chain (older
  52. boots are a resolved episode). Empty when nothing is recent — how a healthy
  53. gateway forgets a loop."""
  54. chain: list[float] = []
  55. prev = ts
  56. for t in sorted(boots, reverse=True):
  57. if t > ts: # clock moved backwards (NTP step, restored file): future entry is adjacent, not a break
  58. chain.append(t)
  59. continue
  60. if prev - t > gap:
  61. break
  62. chain.append(t)
  63. prev = t
  64. return chain[::-1]
  65. def record_restart_interrupted_boot(
  66. window_seconds: int = DEFAULT_WINDOW_SECONDS, *, now: Optional[float] = None,
  67. max_gap_seconds: int = DEFAULT_MAX_GAP_SECONDS,
  68. ) -> list[float]:
  69. """Record a restart-interrupted boot; return the pruned chain + now (most recent
  70. last). A persistence failure returns the in-memory list without raising."""
  71. ts = time.time() if now is None else now
  72. boots = _chain_ending_at(_load_boots(), ts, _chain_gap(window_seconds, max_gap_seconds)) + [ts]
  73. _save_boots(boots[-_MAX_STORED_BOOTS:])
  74. return boots
  75. def clear() -> None:
  76. """Remove the persisted boot log (used on clean shutdown / by tests)."""
  77. with contextlib.suppress(OSError):
  78. _state_path().unlink(missing_ok=True)
  79. def check_and_record(
  80. max_restarts: int = DEFAULT_MAX_RESTARTS, window_seconds: int = DEFAULT_WINDOW_SECONDS, *,
  81. now: Optional[float] = None, max_gap_seconds: int = DEFAULT_MAX_GAP_SECONDS,
  82. ) -> bool:
  83. """Gateway entry point: record this boot; True when the chain reached
  84. ``max_restarts`` and auto-resume should be SKIPPED."""
  85. boots = record_restart_interrupted_boot(window_seconds, now=now, max_gap_seconds=max_gap_seconds)
  86. tripped = max_restarts > 0 and len(boots) >= max_restarts
  87. if tripped:
  88. logger.warning(
  89. "Restart-loop breaker TRIPPED: %d chained restart-interrupted gateway boots (no gap wider than %ds; "
  90. "threshold %d). Skipping auto-resume to break a suspected SIGTERM-respawn loop (#30719, #81642). "
  91. "Restart-interrupted sessions stay resume-pending and will continue on the next real user message. "
  92. "If this is a false positive, delete %s.",
  93. len(boots), int(_chain_gap(window_seconds, max_gap_seconds)), max_restarts, _state_path(),
  94. )
  95. return tripped