replay_cleanup.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. """Replay-history sanitization shared by EVERY resume surface (messaging gateway, TUI/WebUI gateway).
  2. A turn that died mid-tool-loop (restart command, stale timeout, interrupt before the result was written)
  3. persists a dangling ``assistant(tool_calls)`` or interrupted ``assistant→tool`` tail; on resume the model
  4. re-issues the unanswered call → endless "thinking"/reboot loop. These pure helpers strip those tails."""
  5. from __future__ import annotations
  6. import json
  7. import logging
  8. import math
  9. import re
  10. import time
  11. from typing import Any, Dict, List, Optional
  12. from agent.tool_dispatch_helpers import make_tool_result_message
  13. from agent.tool_result_classification import tool_may_have_side_effect
  14. from agent.turn_context import drop_stale_api_content
  15. from hermes_cli.timefmt import coerce_epoch
  16. logger = logging.getLogger(__name__)
  17. # Orphan-recovery notices: (side-effecting, read-only) for an interrupted block vs a dangling tail.
  18. _INTERRUPTED_NOTICES = (
  19. "[Orphan recovery: interrupted side-effecting tool may have executed; its effect is UNKNOWN. Inspect state before retrying.]",
  20. "[Orphan recovery: interrupted read-only tool did not complete.]",
  21. )
  22. _DANGLING_NOTICES = (
  23. "[Orphan recovery: this tool may have executed before Hermes stopped; its effect is UNKNOWN. Inspect current state before retrying.]",
  24. "[Orphan recovery: this read-only tool did not complete and had no effect.]",
  25. )
  26. # Every executor ends the killed run's ``output`` with a bracketed marker line: "[Command
  27. # interrupted]" (tools/environments/, exit 130), "[Command interrupted - Modal ...]"
  28. # (managed_modal.py, exit 130), "[execution interrupted ...]" (code_execution_tool.py, exit -1).
  29. _INTERRUPT_MARKER_LINE = re.compile(r"^\[(?:command|execution) interrupted\b[^\n]*\]\s*$", re.IGNORECASE)
  30. def is_interrupted_tool_result(content: Any) -> bool:
  31. """True only when the result has the executor's interrupt SHAPE: the marker is the last
  32. line of the output (JSON envelope with a non-zero exit code, or a bare text result). A
  33. marker quoted inside successful output — a grep hit, a doc example — is ordinary data;
  34. this runs on every live request, so a false positive rewrites real tool output."""
  35. if not isinstance(content, str):
  36. return False
  37. output = content
  38. if content.lstrip().startswith("{"):
  39. try:
  40. envelope = json.loads(content)
  41. except ValueError:
  42. return False
  43. if not isinstance(envelope, dict) or envelope.get("exit_code") in (0, None):
  44. return False
  45. output = envelope.get("output")
  46. if not isinstance(output, str):
  47. return False
  48. last_line = output.rstrip().rsplit("\n", 1)[-1]
  49. return _INTERRUPT_MARKER_LINE.match(last_line) is not None
  50. def _call_name(call: dict[str, Any]) -> str:
  51. return str((call.get("function") or {}).get("name") or "")
  52. def _call_id(call: dict[str, Any]) -> str:
  53. return str(call.get("id") or call.get("call_id") or "")
  54. def _any_side_effecting(calls: list[dict[str, Any]]) -> bool:
  55. return any(tool_may_have_side_effect(_call_name(call)) for call in calls)
  56. def _orphan_recovery(name: str, notices: tuple) -> tuple:
  57. """(effect_disposition, content) for an interrupted/dangling call named ``name``."""
  58. if tool_may_have_side_effect(name):
  59. return "unknown", notices[0]
  60. return "none", notices[1]
  61. def strip_interrupted_tool_tails(agent_history: list[dict[str, Any]]) -> list[dict[str, Any]]:
  62. """Strip interrupted assistant→tool blocks anywhere in history (a queued user message may follow one).
  63. Read-only blocks are dropped; blocks with a side-effecting call are KEPT with the interrupted results
  64. rewritten as orphan-recovery notices, since the effect may have happened."""
  65. if not agent_history:
  66. return agent_history
  67. cleaned: list[dict[str, Any]] = []
  68. i, n = 0, len(agent_history)
  69. while i < n:
  70. msg = agent_history[i]
  71. if msg.get("role") == "assistant" and "tool_calls" in msg:
  72. j = i + 1
  73. while j < n and agent_history[j].get("role") == "tool":
  74. j += 1
  75. tool_results = agent_history[i + 1:j]
  76. if any(is_interrupted_tool_result(m.get("content", "")) for m in tool_results):
  77. calls = msg.get("tool_calls") or []
  78. if _any_side_effecting(calls):
  79. call_names = {_call_id(call): _call_name(call) for call in calls}
  80. cleaned.append(msg)
  81. for tool_result in tool_results:
  82. if is_interrupted_tool_result(tool_result.get("content", "")):
  83. name = call_names.get(str(tool_result.get("tool_call_id") or ""), "")
  84. disposition, content = _orphan_recovery(name, _INTERRUPTED_NOTICES)
  85. tool_result = {**tool_result, "effect_disposition": disposition, "content": content}
  86. cleaned.append(tool_result)
  87. else:
  88. logger.debug("Stripping interrupted read-only assistant→tool replay block (indices %d–%d, tool_results=%d)",
  89. i, j - 1, len(tool_results))
  90. i = j
  91. continue
  92. if msg.get("role") == "tool" and is_interrupted_tool_result(msg.get("content", "")):
  93. logger.debug("Stripping orphan interrupted tool result from replay history")
  94. else:
  95. cleaned.append(msg)
  96. i += 1
  97. return cleaned
  98. def strip_dangling_tool_call_tail(agent_history: list[dict[str, Any]]) -> list[dict[str, Any]]:
  99. """Strip a trailing ``assistant(tool_calls)`` with NO answers — a call that killed the gateway itself
  100. (``docker restart``) left zero ``tool`` rows, invisible to ``strip_interrupted_tool_tails``. A partially
  101. answered block still resumes. Read-only tails are dropped; side-effecting ones get UNKNOWN-effect results.
  102. On resume the model sees an unanswered tool call at the tail and naturally re-issues it — which restarts
  103. the gateway again, producing the infinite reboot loop in #49201. ``strip_interrupted_tool_tails`` does
  104. not catch this because there is no tool result to inspect for an interrupt marker.
  105. """
  106. if not agent_history:
  107. return agent_history
  108. last = agent_history[-1]
  109. if not (isinstance(last, dict) and last.get("role") == "assistant" and last.get("tool_calls")):
  110. return agent_history
  111. tool_calls = last.get("tool_calls") or []
  112. if _any_side_effecting(tool_calls):
  113. recovered = list(agent_history)
  114. for call in tool_calls:
  115. name = _call_name(call) or "unknown"
  116. disposition, content = _orphan_recovery(name, _DANGLING_NOTICES)
  117. recovered.append(make_tool_result_message(name, content, _call_id(call), effect_disposition=disposition))
  118. logger.warning("Recovered dangling side-effecting tool call(s) as UNKNOWN instead of erasing them")
  119. return recovered
  120. logger.debug("Stripping dangling unanswered read-only assistant(tool_calls) tail (%d call(s))", len(tool_calls))
  121. return agent_history[:-1]
  122. def sanitize_replay_history(agent_history: list[dict[str, Any]]) -> list[dict[str, Any]]:
  123. """Both strippers in canonical order (interrupted blocks, then dangling tail); same list object when nothing strips."""
  124. if not agent_history:
  125. return agent_history
  126. return strip_dangling_tool_call_tail(strip_interrupted_tool_tails(agent_history))
  127. def canonicalize_replay_history(
  128. agent_history: list[dict[str, Any]], *, now: Optional[float] = None
  129. ) -> list[dict[str, Any]]:
  130. """Apply every destructive replay transform in the shared, fixed order.
  131. Resume surfaces and the send path must serialize the same history bytes, or a
  132. resumed request diverges in the middle of the cached prefix.
  133. The input is never modified. ``now`` is the expiry clock; the send path passes the
  134. turn's admission time so every request in one turn sees the same bytes.
  135. """
  136. if not agent_history:
  137. return agent_history
  138. if now is None:
  139. now = time.time()
  140. cleaned = strip_interrupted_tool_tails(agent_history)
  141. cleaned = strip_dangling_tool_call_tail(cleaned)
  142. return strip_stale_dangerous_confirmations(cleaned, now=now)
  143. # --- Stale dangerous-confirmation text expiry ---
  144. # Short on purpose: a dangerous confirmation must not survive any restart or resume gap.
  145. # ────────────────────────────────────────────────────────────────────── Stale dangerous-confirmation text
  146. # expiry (#59607) ──────────────────────────────────────────────────────────────────────
  147. _DANGEROUS_CONFIRMATION_EXPIRY_SECONDS = 60.0
  148. # Phrases that unlock destructive host actions; case-insensitive substring match so trailing punctuation /
  149. # extra context still matches. Includes i18n variants from the original incident.
  150. _DANGEROUS_CONFIRMATION_PATTERNS: tuple = (
  151. "confirm forced restart", "confirm forced reboot", "confirm shutdown", "confirm reboot", "confirm power off",
  152. "yes, delete everything", "confirm wipe", "confirm factory reset",
  153. "確認強制重開機", "確認強制重開", "確認重啟",
  154. )
  155. # Redacting in place (not deleting the message) preserves strict user/assistant alternation in the replay.
  156. _EXPIRED_CONFIRMATION_SENTINEL = (
  157. "[A high-risk confirmation previously given here has EXPIRED and must "
  158. "not be acted on. Ask the user to re-confirm explicitly before "
  159. "performing any destructive action.]"
  160. )
  161. def is_dangerous_confirmation(content: Any) -> bool:
  162. """True if user-message text contains a known dangerous confirmation phrase."""
  163. if not isinstance(content, str):
  164. return False
  165. lowered = content.strip().lower()
  166. return any(pattern in lowered for pattern in _DANGEROUS_CONFIRMATION_PATTERNS)
  167. def strip_stale_dangerous_confirmations(
  168. agent_history: list[dict[str, Any]],
  169. *,
  170. now: float,
  171. expiry_seconds: float = _DANGEROUS_CONFIRMATION_EXPIRY_SECONDS,
  172. ) -> list[dict[str, Any]]:
  173. """Redact IN PLACE dangerous-confirmation text older than ``expiry_seconds`` in user messages: a confirmation
  174. surviving a restart reads as a fresh re-confirmation minutes later. Untimestamped messages (legacy
  175. transcripts, test scaffolding) are left untouched.
  176. See #59607.
  177. On the next inbound message — possibly a casual "are you there?" from the user minutes later — the LLM
  178. sees the stale confirmation and may interpret the new turn as a fresh re-confirmation, re-executing the
  179. destructive action. This is the failure mode reported in #59607.
  180. """
  181. if not agent_history:
  182. return agent_history
  183. cleaned: list[dict[str, Any]] = []
  184. for msg in agent_history:
  185. ts = msg.get("timestamp") if isinstance(msg, dict) and msg.get("role") == "user" else None
  186. if ts is None or not is_dangerous_confirmation(msg.get("content", "")):
  187. cleaned.append(msg)
  188. continue
  189. # A present-but-untrustworthy stamp (corrupt, or issued in the future relative to
  190. # the admission clock) is treated as expired: its age is unknowable, and keeping the
  191. # text (plus its api_content sidecar) would replay a live confirmation.
  192. ts_f = coerce_epoch(ts, field="message timestamp")
  193. age = math.inf if ts_f is None else now - ts_f
  194. if 0 <= age <= expiry_seconds:
  195. cleaned.append(msg)
  196. continue
  197. logger.debug(
  198. "Redacting stale dangerous-confirmation text in user message (age=%.1fs, expiry=%.1fs): %r",
  199. age, expiry_seconds, (msg.get("content") or "")[:80],
  200. )
  201. redacted = dict(msg)
  202. redacted["content"] = _EXPIRED_CONFIRMATION_SENTINEL
  203. # The api_content sidecar carries the exact bytes sent — the confirmation itself; replaying it would undo the redaction.
  204. drop_stale_api_content(redacted)
  205. cleaned.append(redacted)
  206. return cleaned