tool_result_classification.py 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118
  1. """Shared helpers for classifying tool result payloads."""
  2. from __future__ import annotations
  3. import json
  4. import re
  5. from typing import Any
  6. FILE_MUTATING_TOOL_NAMES = frozenset({"write_file", "patch"})
  7. # Tools whose interrupted/dangling execution is safe to discard because they
  8. # cannot mutate either external state or Hermes session state. Unknown/plugin/
  9. # MCP tools stay effect-capable by default.
  10. NO_EFFECT_TOOL_NAMES = frozenset({
  11. "read_file", "search_files", "session_search", "skill_view", "skills_list",
  12. "web_extract", "web_search", "vision_analyze", "browser_snapshot",
  13. "browser_get_images", "browser_console", "read_terminal",
  14. })
  15. def tool_may_have_side_effect(tool_name: str) -> bool:
  16. return tool_name not in NO_EFFECT_TOOL_NAMES
  17. # Set by a tool that REFUSED a call the harness judged redundant (repeated identical
  18. # read/search). The body still carries ``"error"`` so the model reads it as a stop
  19. # signal, but nothing failed: failure classifiers must not count it, or the cheap
  20. # refusal feeds the streak that fires ``repeated_exact_failure_block``.
  21. GUARDRAIL_REFUSAL_KEY = "guardrail_refusal"
  22. def is_guardrail_refusal(result: Any) -> bool:
  23. """Return True when ``result`` (JSON string or parsed dict) is a harness refusal."""
  24. data = result
  25. if isinstance(result, str):
  26. try:
  27. data = json.loads(result.strip())
  28. except Exception:
  29. return False
  30. return isinstance(data, dict) and data.get(GUARDRAIL_REFUSAL_KEY) is True
  31. # A quoted "error"/"failed" in a result's head marks it failed UNLESS it is a key whose value is
  32. # empty: success payloads routinely carry ``"error": null`` per item (web_extract) or a
  33. # ``"failed": 0`` count, and a bare substring test flagged every one of them.
  34. _FAILURE_MARKER_RE = re.compile(
  35. r'"(?:error|failed)"(?!\s*:\s*(?:null|false|0|""|\[\]|\{\})\s*(?:[,}\]]|$))', re.IGNORECASE)
  36. def has_failure_marker(result: str) -> bool:
  37. """Last-resort text test for a failed tool result: a non-empty ``"error"``/``"failed"``
  38. STARTING in the first 500 chars, or a body starting with ``Error``. The value is read past
  39. the cut, so an ``"error": null`` straddling char 500 is not mistaken for a truncated one."""
  40. head = result[:600]
  41. return any(m.start() < 500 for m in _FAILURE_MARKER_RE.finditer(head)) or result.startswith("Error")
  42. # ``timeout`` is deliberately absent: process(wait) returns it for a still-running job.
  43. _FAILED_STATUSES = frozenset({"failed", "error"})
  44. _RATE_LIMIT_RE = re.compile(r"rate[\s_-]?limit|too many requests|\b429\b", re.IGNORECASE)
  45. _RETRY_AFTER_KEYS = ("retry_after", "retryAfter", "retry_after_seconds", "retry_after_ms")
  46. def is_rate_limited(result: Any) -> bool:
  47. """True when a FAILED result says the backend throttled the call (a ``retry_after`` field, a 429
  48. status, or rate-limit wording in its error). Only meaningful for results already judged failed:
  49. a successful page can mention rate limits in its content."""
  50. data = result
  51. if isinstance(result, str):
  52. try:
  53. data = json.loads(result.strip())
  54. except ValueError:
  55. return bool(_RATE_LIMIT_RE.search(result[:500]))
  56. if not isinstance(data, dict):
  57. return False
  58. if any(data.get(k) for k in _RETRY_AFTER_KEYS) or 429 in (data.get("status"), data.get("status_code"), data.get("code")):
  59. return True
  60. error = data.get("error")
  61. text = error.get("message") or error.get("code") if isinstance(error, dict) else error
  62. return isinstance(text, str) and bool(_RATE_LIMIT_RE.search(text))
  63. def shape_failure_error(data: dict) -> str | None:
  64. """Error text when a JSON-object result failed by SHAPE rather than by its top-level
  65. ``error``: a ``status`` of failed/error, or a ``results`` batch where EVERY item carries an
  66. error (delegate_task with all children interrupted, a connector batch with every call
  67. refused). None for a clean or partial result. Top-level JSON objects skip the substring
  68. scan, so this is what keeps those failures counted."""
  69. if str(data.get("status", "")).lower() in _FAILED_STATUSES:
  70. return str(data.get("status"))
  71. items = data.get("results")
  72. if not isinstance(items, list) or not items or not all(isinstance(i, dict) for i in items):
  73. return None
  74. errors = [i.get("error") for i in items]
  75. if not all(errors):
  76. return None
  77. first = errors[0]
  78. return str(first.get("message") or first) if isinstance(first, dict) else str(first)
  79. def file_mutation_result_landed(tool_name: str, result: Any) -> bool:
  80. """Return True when a file mutation result proves the write landed."""
  81. if tool_name not in FILE_MUTATING_TOOL_NAMES or not isinstance(result, str):
  82. return False
  83. try:
  84. data = json.loads(result.strip())
  85. except Exception:
  86. return False
  87. if not isinstance(data, dict) or data.get("error"):
  88. return False
  89. if tool_name == "write_file":
  90. return "bytes_written" in data
  91. if tool_name == "patch":
  92. return data.get("success") is True
  93. return False