base_wait.py 3.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475
  1. """Opt-in interrupt/poll tracing for ``BaseEnvironment._wait_for_process``
  2. (``HERMES_DEBUG_INTERRUPT=1``): loop entry/exit, interrupt/timeout detection and ~30s
  3. heartbeats so "agent never sees the interrupt" reports can be diagnosed from agent.log."""
  4. import threading
  5. import time
  6. from tools.interrupt import is_interrupted
  7. _FMT = {
  8. "ENTER": "timeout=%ss activity_cb=%s initial_interrupt=%s",
  9. "INTERRUPT DETECTED": "iter=%d elapsed=%.1fs — killing process group",
  10. "TIMEOUT": "iter=%d timeout=%ss",
  11. "HEARTBEAT": "iter=%d elapsed=%.0fs interrupt=%s activity_cb=%s%s",
  12. "EXCEPTION_EXIT": "iter=%d elapsed=%.1fs — killing subprocess group before re-raise",
  13. "EXIT (natural)": "iter=%d elapsed=%.1fs returncode=%s"}
  14. class _WaitTrace:
  15. """Per-wait debug tracer; every method is a no-op unless ``enabled``."""
  16. def __init__(self, proc, timeout, *, enabled: bool, logger):
  17. self.enabled = enabled
  18. self.iterations = 0
  19. if not enabled:
  20. return
  21. from tools.environments.base import get_activity_callback
  22. self._get_cb, self._log, self._timeout = get_activity_callback, logger, timeout
  23. self._tid = threading.current_thread().ident
  24. self._pid = getattr(proc, "pid", None)
  25. self._start = self._last_heartbeat = time.monotonic()
  26. self._cb_was_none = get_activity_callback() is None
  27. def _emit(self, event: str, *args) -> None:
  28. self._log.info(
  29. "[interrupt-debug] _wait_for_process %s tid=%s pid=%s " + _FMT[event],
  30. event, self._tid, self._pid, *args)
  31. def _elapsed(self) -> float:
  32. return time.monotonic() - self._start
  33. def enter(self) -> None:
  34. if self.enabled:
  35. self._emit("ENTER", self._timeout,
  36. "set" if not self._cb_was_none else "MISSING", is_interrupted())
  37. def interrupted(self) -> None:
  38. if self.enabled:
  39. self._emit("INTERRUPT DETECTED", self.iterations, self._elapsed())
  40. def timed_out(self) -> None:
  41. if self.enabled:
  42. self._emit("TIMEOUT", self.iterations, self._timeout)
  43. def heartbeat(self) -> None:
  44. """Every ~30s: proves the loop is alive and reports whether the thread-local
  45. activity callback got clobbered by nested tool calls / executor thread reuse."""
  46. if not self.enabled or time.monotonic() - self._last_heartbeat < 30.0:
  47. return
  48. cb_now_none = self._get_cb() is None
  49. self._emit(
  50. "HEARTBEAT", self.iterations, self._elapsed(), is_interrupted(),
  51. "set" if not cb_now_none else "MISSING",
  52. " (LOST during run)" if cb_now_none and not self._cb_was_none else "")
  53. self._last_heartbeat = time.monotonic()
  54. self._cb_was_none = cb_now_none
  55. def exception_exit(self) -> None:
  56. if self.enabled:
  57. self._emit("EXCEPTION_EXIT", self.iterations, self._elapsed())
  58. def natural_exit(self, returncode) -> None:
  59. if self.enabled:
  60. self._emit("EXIT (natural)", self.iterations, self._elapsed(), returncode)