estop.py 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138
  1. """Global emergency stop (ESTOP) — a resumable pause for NEW work only.
  2. ``hermes pause`` writes a sentinel at ``$HERMES_HOME/ESTOP``; ``hermes resume``
  3. removes it. While it exists the cron scheduler, kanban dispatcher and new gateway
  4. turns skip work; in-flight work is never killed. The check is one or two uncached
  5. ``os.stat`` calls (process home + fleet root when they differ). The body is optional
  6. JSON ``{"reason", "engaged_at"}``; a corrupt/empty file still counts as engaged
  7. (fail safe, e.g. ``touch ~/.hermes/ESTOP``). Ported from gastownhall/gastown estop.go (MIT).
  8. """
  9. from __future__ import annotations
  10. import json
  11. import logging
  12. import threading
  13. from contextlib import suppress
  14. from datetime import datetime, timezone, UTC
  15. from pathlib import Path
  16. from typing import Optional
  17. # Same profile-aware / fleet-root resolvers the file-safety guards use (fail-open to ~/.hermes).
  18. from agent.file_safety import _hermes_home_path as _hermes_home, _hermes_root_path as _canonical_root
  19. SENTINEL_NAME = "ESTOP"
  20. # Per-component "logged already for this engagement" flags: log once per engagement, not per tick.
  21. _log_lock = threading.Lock()
  22. _logged_components: set[str] = set()
  23. def sentinel_path() -> Path:
  24. """Path of the ESTOP sentinel this process would write on `hermes pause`."""
  25. return _hermes_home() / SENTINEL_NAME
  26. def _candidate_sentinel_paths() -> list:
  27. """Profile home first, then the fleet root if it is a different directory: a profile
  28. gateway (HERMES_HOME=~/.hermes/profiles/<n>) must still honor an operator's ~/.hermes/ESTOP."""
  29. primary = sentinel_path()
  30. try:
  31. root = _canonical_root() / SENTINEL_NAME
  32. except Exception:
  33. return [primary]
  34. try:
  35. distinct = root.resolve() != primary.resolve()
  36. except Exception:
  37. # Non-Path test doubles fail .resolve(); plain equality still dedupes.
  38. distinct = root != primary
  39. return [primary, root] if distinct else [primary]
  40. def is_engaged() -> bool:
  41. """True if ANY candidate sentinel exists; fail SAFE (True) on stat errors."""
  42. saw_stat_error = False
  43. for path in _candidate_sentinel_paths():
  44. try:
  45. if path.exists():
  46. return True
  47. except OSError:
  48. saw_stat_error = True
  49. return saw_stat_error
  50. def engage(reason: Optional[str] = None) -> Path:
  51. """Create the ESTOP sentinel. Idempotent; re-engaging updates the file."""
  52. path = sentinel_path()
  53. payload = {"engaged_at": datetime.now(UTC).isoformat(), "reason": reason or None}
  54. try:
  55. path.parent.mkdir(parents=True, exist_ok=True)
  56. path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
  57. except OSError:
  58. with suppress(OSError): # Best effort: an empty/partial sentinel still pauses (fail safe).
  59. path.touch(exist_ok=True)
  60. return path
  61. def disengage() -> bool:
  62. """Remove every visible sentinel (process-local and fleet-root)."""
  63. lifted = False
  64. for path in _candidate_sentinel_paths():
  65. try:
  66. path.unlink()
  67. lifted = True
  68. except (OSError, AttributeError):
  69. continue
  70. return lifted
  71. def get_state() -> Optional[dict]:
  72. """Return ``{"reason", "engaged_at"}`` or None when not engaged; an unreadable/corrupt
  73. body still reports engaged with both fields None."""
  74. if not is_engaged():
  75. return None
  76. state = {"reason": None, "engaged_at": None}
  77. found = False
  78. for path in _candidate_sentinel_paths():
  79. try:
  80. if not path.exists():
  81. continue
  82. except OSError:
  83. return state
  84. except AttributeError:
  85. continue
  86. found = True
  87. with suppress(OSError, ValueError, AttributeError):
  88. raw = json.loads(path.read_text(encoding="utf-8-sig"))
  89. if isinstance(raw, dict):
  90. state = {"reason": raw.get("reason") or None, "engaged_at": raw.get("engaged_at") or None}
  91. break
  92. return state if found else None
  93. def paused_reply() -> Optional[str]:
  94. """Short user-facing notice for new gateway turns, or None if not paused."""
  95. state = get_state()
  96. if state is None:
  97. return None
  98. tag = f" ({state['reason']})" if state.get("reason") else ""
  99. return f"⏸️ Hermes is paused{tag}. New work is on hold; run `hermes resume` to pick things back up."
  100. def check_paused(component: str, logger: logging.Logger) -> bool:
  101. """Return True when engaged, logging once per engagement per component (re-armed after a resume)."""
  102. if not is_engaged():
  103. with _log_lock:
  104. _logged_components.discard(component)
  105. return False
  106. with _log_lock:
  107. first = component not in _logged_components
  108. _logged_components.add(component)
  109. if first:
  110. reason = (get_state() or {}).get("reason")
  111. suffix = f" (reason: {reason})" if reason else ""
  112. logger.info(
  113. "%s dispatch paused by global emergency stop%s — remove with `hermes resume` (%s)",
  114. component, suffix, sentinel_path(),
  115. )
  116. return True