approval_human_wait.py 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170
  1. """Human-wait accounting for :mod:`tools.approval` (per session).
  2. Tracks wall-clock time the agent spends verifiably blocked on a HUMAN prompt
  3. (CLI approval prompt, gateway approval round-trip). The concurrent tool batch
  4. deadline in agent/tool_executor.py excludes this time so a slow human answer
  5. never times a batch out — but ONLY this time. Measuring at the source (rather
  6. than residency in the authorization gate, which is arbitrary code) is what keeps
  7. a wedged pre_tool_call plugin or a dead approval client from growing the
  8. exclusion 1:1 with wall clock and defeating the deadline entirely. Keyed by
  9. session so one gateway session's pending approval cannot extend a different
  10. session's batch deadline; state is process-global like the rest of the approval
  11. state, bounded by _HUMAN_WAIT_MAX_SESSIONS.
  12. """
  13. import contextlib
  14. import threading
  15. import time
  16. # ========================================================================= Human-wait accounting (per
  17. # session) ========================================================================= Tracks the wall-clock
  18. # time the agent spends verifiably blocked on a HUMAN prompt (CLI approval prompt, gateway approval
  19. # round-trip). The concurrent tool batch deadline in agent/tool_executor.py excludes this time so a slow
  20. # human answer never times a batch out — but ONLY this time. Measuring human waits at the source (rather
  21. # than residency in the authorization gate, which is arbitrary code) is what keeps a wedged pre_tool_call
  22. # plugin or a dead approval client from growing the exclusion 1:1 with wall clock and defeating the deadline
  23. # entirely (#79719). Keyed by session so one gateway session's pending approval cannot extend a different
  24. # session's batch deadline. State is process-global like the rest of this module's approval state; entries
  25. # are bounded by _HUMAN_WAIT_MAX_SESSIONS.
  26. class _HumanWaitState:
  27. __slots__ = ("completed_seconds", "pending", "window_started")
  28. def __init__(self) -> None:
  29. self.pending = 0
  30. self.window_started: float | None = None
  31. self.completed_seconds = 0.0
  32. _human_wait_lock = threading.Lock()
  33. _human_wait_states: dict[str, _HumanWaitState] = {}
  34. _HUMAN_WAIT_MAX_SESSIONS = 256
  35. # Margin added on top of approvals.timeout when clamping a window's contribution (read-side AND close-side) and when
  36. # bounding the authorization gate's serialization-lock acquire in agent/tool_executor.py. One constant so the clamps
  37. # can't drift apart.
  38. HUMAN_WAIT_MARGIN_S = 60.0
  39. def human_wait_ceiling() -> float:
  40. """Max seconds a single window may contribute: approvals.timeout + margin.
  41. Every legitimate human wait self-terminates at ``approvals.timeout`` (the CLI
  42. prompt join and the gateway poll loop both enforce it), so a window that
  43. overstays this ceiling is itself wedged and must not keep extending a batch
  44. deadline. Also the bound on the authorization gate's serialization-lock
  45. acquire in agent/tool_executor.py, so the two cannot drift. Never call while
  46. holding ``_human_wait_lock`` — it reads the config cache.
  47. ``_get_approval_timeout`` caps at ``agent.deadline.MAX_SAFE_TIMEOUT_S`` so the
  48. value is always safe for ``Lock.acquire(timeout=...)`` / ``Thread.join(timeout=...)``."""
  49. from tools import approval_context
  50. from agent.deadline import MAX_SAFE_TIMEOUT_S
  51. return min(MAX_SAFE_TIMEOUT_S, float(approval_context._get_approval_timeout()) + HUMAN_WAIT_MARGIN_S)
  52. def _clamped_window_seconds(started: float, now: float, ceiling: float) -> float:
  53. """Seconds an open window contributes: elapsed, floored at 0, capped. Shared
  54. by the close-time accrual and the open-window read so the two clamps stay
  55. identical by construction."""
  56. return min(max(0.0, now - started), ceiling)
  57. def _human_wait_state(session_key: str) -> _HumanWaitState:
  58. """Return (creating if needed) the wait state for *session_key*. Caller must
  59. hold ``_human_wait_lock``. Evicts idle entries (no pending waiter)
  60. insertion-order-first until the table is under the cap so an army of
  61. short-lived session keys cannot grow it without bound. Entries with an open
  62. window are never evicted (that would corrupt live accounting), so the cap is
  63. best-effort under 256+ concurrently-pending sessions."""
  64. state = _human_wait_states.get(session_key)
  65. if state is None:
  66. for key in list(_human_wait_states):
  67. if len(_human_wait_states) < _HUMAN_WAIT_MAX_SESSIONS:
  68. break
  69. if _human_wait_states[key].pending == 0:
  70. del _human_wait_states[key]
  71. state = _human_wait_states[session_key] = _HumanWaitState()
  72. return state
  73. def _resolve_key(session_key: str | None) -> str:
  74. if session_key is not None:
  75. return session_key
  76. from tools import approval_context
  77. return approval_context.get_current_session_key()
  78. def activity_heartbeat(label: str):
  79. """Callable that pings the agent's inactivity tracker (at most every ~10s)
  80. while a human wait is parked, so the gateway watchdog does not kill the agent
  81. while the user is still answering. No-op in minimal tool-only environments."""
  82. try:
  83. from tools.environments.base import touch_activity_if_due
  84. except Exception: # pragma: no cover - minimal tool-only environments
  85. return lambda: None
  86. now = time.monotonic()
  87. state = {"last_touch": now, "start": now}
  88. return lambda: touch_activity_if_due(state, label)
  89. @contextlib.contextmanager
  90. def human_wait_window(session_key: str | None = None):
  91. """Mark the enclosed block as time spent blocked on a human prompt. Wrap ONLY
  92. code that is genuinely parked waiting for a user's answer (the CLI approval
  93. prompt, the gateway approval poll loop). The concurrent tool batch deadline
  94. excludes this time; wrapping anything else re-creates the hang where
  95. arbitrary wedged code pushes the deadline out forever. Overlapping windows
  96. for the same session coalesce (pending counter), so two serialized approval
  97. prompts don't double-count the same wall clock.
  98. See #79719.
  99. """
  100. key = _resolve_key(session_key)
  101. now = time.monotonic()
  102. with _human_wait_lock:
  103. state = _human_wait_state(key)
  104. if state.pending == 0:
  105. state.window_started = now
  106. state.pending += 1
  107. try:
  108. yield
  109. finally:
  110. now = time.monotonic()
  111. # Clamp the accrual too: a window that overstayed the ceiling was wedged —
  112. # record at most the ceiling, not the whole overstay.
  113. ceiling = human_wait_ceiling()
  114. with _human_wait_lock:
  115. state = _human_wait_states.get(key)
  116. if state is not None:
  117. state.pending -= 1
  118. if state.pending == 0:
  119. if state.window_started is not None:
  120. state.completed_seconds += _clamped_window_seconds(state.window_started, now, ceiling)
  121. state.window_started = None
  122. def human_wait_seconds(session_key: str | None = None) -> float:
  123. """Return total human-wait seconds recorded for the session: completed windows
  124. plus the currently open one (if any). Monotonically non-decreasing for the
  125. life of the process — except when an idle session's entry is evicted under
  126. cap pressure, which can only shrink a consumer's baseline delta to zero (the
  127. safe direction: the deadline fires sooner). Deadline consumers snapshot a
  128. baseline at batch start and use the delta. Each window's contribution is
  129. clamped to :func:`human_wait_ceiling` (belt-and-braces against the
  130. wedged-window hang).
  131. Each window's contribution is clamped to :func:`human_wait_ceiling`: every legitimate human wait
  132. self-terminates at ``approvals.timeout`` (both the CLI prompt join and the gateway poll loop enforce
  133. it), so a window that overstays that bound is itself wedged and must not keep extending a batch deadline
  134. (belt-and-braces for #79719).
  135. """
  136. key = _resolve_key(session_key)
  137. now = time.monotonic()
  138. # Resolve the clamp outside the lock: it reads the config cache, which must never nest under _human_wait_lock.
  139. ceiling = human_wait_ceiling()
  140. with _human_wait_lock:
  141. state = _human_wait_states.get(key)
  142. if state is None:
  143. return 0.0
  144. total = state.completed_seconds
  145. if state.window_started is not None:
  146. total += _clamped_window_seconds(state.window_started, now, ceiling)
  147. return total