shared_metrics_harness.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471
  1. """Agent-harness accuracy shared metrics: file-edit matching, loop guards, recovery after a tool
  2. error, terminal command outcomes and bad model replies.
  3. These tune the agent loop itself: which fuzzy-match strategies earn their keep, how often the loop
  4. guards fire, whether a model recovers after a failed tool call, which kinds of commands fail, and
  5. which models return empty / refused / truncated replies. Producers pass RAW values; every builder
  6. maps them onto closed enums. Nothing here reads command text, file paths, tool arguments or reply
  7. text beyond the structural checks Hermes already makes (think-block stripping for "visible text").
  8. Hermes-owned agent loops (the background self-improvement review and the curator fork) are not user
  9. work and never count. Recording is a no-op unless shared metrics are enabled for the owning profile,
  10. and never raises into the caller. Stdlib-only at import time: tool modules import this on their hot
  11. path.
  12. """
  13. from __future__ import annotations
  14. import contextvars
  15. import json
  16. import logging
  17. import re
  18. from dataclasses import dataclass, field
  19. from typing import Any, Callable
  20. logger = logging.getLogger(__name__)
  21. _BACKGROUND_ORIGIN = "background_review"
  22. def _enabled() -> bool:
  23. try:
  24. from .relay_shared_metrics import enabled
  25. return enabled()
  26. except Exception:
  27. return False
  28. def _record(mark_attr: str, data: dict[str, str]) -> None:
  29. try:
  30. from . import shared_metrics_contract as contract
  31. from .relay_shared_metrics import record_process_mark
  32. record_process_mark(getattr(contract, mark_attr), data)
  33. except Exception:
  34. logger.debug("Shared-metrics %s not recorded", mark_attr, exc_info=True)
  35. def _background_tool_call() -> bool:
  36. try:
  37. from tools.skill_provenance import is_background_review
  38. return is_background_review()
  39. except Exception:
  40. return False
  41. def _user_agent(agent: Any) -> bool:
  42. """The review/curator forks tag themselves with the background write origin."""
  43. return agent is not None and getattr(agent, "_memory_write_origin", None) != _BACKGROUND_ORIGIN
  44. def _route(agent: Any) -> dict[str, str]:
  45. from .shared_metrics_model import model_route
  46. return model_route(getattr(agent, "provider", None), getattr(agent, "model", None))
  47. # ---- file edits ------------------------------------------------------------------------------
  48. @dataclass
  49. class _EditProbe:
  50. """What the fuzzy matcher saw during one edit tool call (several hunks for V4A)."""
  51. strategies: list[str] = field(default_factory=list)
  52. misses: list[str] = field(default_factory=list)
  53. _EDIT_PROBE: contextvars.ContextVar[_EditProbe | None] = contextvars.ContextVar(
  54. "shared_metrics_edit_probe", default=None,
  55. )
  56. def note_edit_match(strategy: str | None, miss: str | None = None) -> None:
  57. """Called by the fuzzy matcher: the strategy that landed a replacement, or why none did
  58. (``no_match`` / ``ambiguous``). A no-op outside a metered edit tool call."""
  59. probe = _EDIT_PROBE.get()
  60. if probe is None:
  61. return
  62. if strategy:
  63. probe.strategies.append(strategy)
  64. elif miss:
  65. probe.misses.append(miss)
  66. def _strategy_rank() -> dict[str, int]:
  67. from tools.fuzzy_match import STRATEGIES
  68. return {name: index for index, (name, _fn) in enumerate(STRATEGIES)}
  69. # The patch tool's ``mode`` argument: ``patch`` is the V4A multi-hunk format.
  70. _EDIT_MODES = {"replace": "replace", "patch": "v4a", "whole_file": "whole_file"}
  71. def file_edit_fields(*, tool: str, mode: Any, result: Any, probe: _EditProbe) -> dict[str, str]:
  72. from .shared_metrics_contract import FILE_EDIT_MODES, FILE_EDIT_STRATEGIES
  73. data = _parsed(result)
  74. # patch results carry ``success``; write_file reports bytes written, or an ``error``.
  75. if data.get("success") is True or (data and "success" not in data and not data.get("error")):
  76. outcome = "already_applied" if data.get("no_change") is True else "applied"
  77. elif "ambiguous" in probe.misses:
  78. outcome = "ambiguous"
  79. elif "no_match" in probe.misses:
  80. outcome = "no_match"
  81. else:
  82. outcome = "failed"
  83. strategy = "none"
  84. if outcome == "applied" and probe.strategies:
  85. # A V4A patch lands several hunks: report the loosest strategy it needed.
  86. rank = _strategy_rank()
  87. strategy = max(probe.strategies, key=lambda name: rank.get(name, -1))
  88. mode_value = _EDIT_MODES.get(str(mode or "").strip().lower(), "replace")
  89. return {
  90. "match_strategy": strategy if strategy in FILE_EDIT_STRATEGIES else "none",
  91. "mode": mode_value if mode_value in FILE_EDIT_MODES else "replace",
  92. "outcome": outcome,
  93. "tool": tool,
  94. }
  95. def record_file_edit(tool: str, mode: str, call: Callable[[], str]) -> str:
  96. """Run one ``patch`` / ``write_file`` tool call and count how its edit landed; returns the
  97. call's result unchanged. The matcher reports into a context-local probe, so strategies from
  98. unrelated callers (skill patches, V4A validation of another call) never mix in."""
  99. if not _enabled() or _background_tool_call():
  100. return call()
  101. probe = _EditProbe()
  102. token = _EDIT_PROBE.set(probe)
  103. try:
  104. result = call()
  105. finally:
  106. _EDIT_PROBE.reset(token)
  107. try:
  108. _record("FILE_EDIT_MARK", file_edit_fields(tool=tool, mode=mode, result=result, probe=probe))
  109. except Exception:
  110. logger.debug("Shared-metrics file edit not recorded", exc_info=True)
  111. return result
  112. def _parsed(result: Any) -> dict[str, Any]:
  113. if isinstance(result, dict):
  114. return result
  115. try:
  116. data = json.loads(result) if isinstance(result, (str, bytes)) else None
  117. except (TypeError, ValueError):
  118. return {}
  119. return data if isinstance(data, dict) else {}
  120. # ---- terminal command outcomes ---------------------------------------------------------------
  121. def _kinds(kind: str, *names: str) -> dict[str, str]:
  122. return dict.fromkeys(names, kind)
  123. _COMMAND_KINDS: dict[str, str] = {
  124. **_kinds("git", "git", "gh", "hub", "git-lfs", "tig"),
  125. **_kinds(
  126. "package_manager", "pip", "pip3", "uv", "uvx", "poetry", "pipx", "conda", "mamba", "micromamba",
  127. "npm", "pnpm", "yarn", "apt", "apt-get", "brew", "dnf", "yum", "pacman", "apk", "gem", "bundle",
  128. "composer", "nix", "snap", "choco", "winget", "port",
  129. ),
  130. **_kinds(
  131. "build", "make", "cmake", "ninja", "cargo", "go", "gradle", "gradlew", "mvn", "bazel", "meson",
  132. "gcc", "g++", "cc", "clang", "clang++", "rustc", "javac", "tsc", "dotnet", "msbuild", "scons",
  133. "swift", "xcodebuild", "zig",
  134. ),
  135. **_kinds(
  136. "test_runner", "pytest", "py.test", "tox", "nox", "jest", "vitest", "mocha", "rspec", "phpunit",
  137. "ctest", "playwright",
  138. ),
  139. **_kinds("python", "python", "python3", "ipython", "pypy", "pypy3"),
  140. **_kinds("node", "node", "npx", "deno", "bun", "ts-node", "tsx", "nodejs"),
  141. **_kinds(
  142. "shell_builtin", "cd", "echo", "export", "source", ".", "set", "unset", "pwd", "printf", "test",
  143. "[", "[[", "true", "false", "exit", "alias", "read", "eval", "type", "hash", "ulimit", "umask",
  144. "wait", "pushd", "popd", "shopt", "trap", "declare", "local", "if", "for", "while", "case", "{",
  145. ),
  146. **_kinds("shell", "bash", "sh", "zsh", "dash", "fish", "ksh", "pwsh", "powershell", "cmd"),
  147. **_kinds(
  148. "file_ops", "ls", "cat", "head", "tail", "cp", "mv", "rm", "mkdir", "rmdir", "touch", "find",
  149. "grep", "egrep", "rg", "sed", "awk", "chmod", "chown", "ln", "du", "df", "tree", "wc", "sort",
  150. "uniq", "diff", "tar", "zip", "unzip", "gzip", "gunzip", "fd", "stat", "file", "less", "more",
  151. "realpath", "readlink", "basename", "dirname", "cut", "tr", "xargs", "tee", "jq", "patch",
  152. "md5sum", "sha256sum", "truncate", "which",
  153. ),
  154. **_kinds(
  155. "network", "curl", "wget", "ssh", "scp", "rsync", "ping", "nc", "dig", "nslookup", "http", "telnet",
  156. "sftp", "ftp", "traceroute",
  157. ),
  158. **_kinds(
  159. "container", "docker", "podman", "kubectl", "docker-compose", "helm", "nerdctl", "minikube", "kind",
  160. "k9s", "apptainer", "singularity",
  161. ),
  162. }
  163. # Prefixes that only change how the real program runs; the program after them is the kind.
  164. _COMMAND_WRAPPERS = frozenset({"sudo", "env", "time", "nohup", "exec", "command", "builtin", "nice", "stdbuf"})
  165. _TOKEN_RE = re.compile(r"\s*([^\s;|&()<>]+)")
  166. _ASSIGNMENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*=")
  167. _PYTHON_RE = re.compile(r"python\d+(\.\d+)?")
  168. # Only the command's opening is inspected; a huge heredoc never gets scanned.
  169. _COMMAND_HEAD_CHARS = 256
  170. _MAX_PREFIX_TOKENS = 4
  171. def command_kind(command: Any) -> str:
  172. """A closed kind from the command's first program word (after env assignments and wrappers
  173. like ``sudo``). The word itself is never exported."""
  174. text = command[:_COMMAND_HEAD_CHARS] if isinstance(command, str) else ""
  175. position = 0
  176. for _ in range(_MAX_PREFIX_TOKENS + 1):
  177. match = _TOKEN_RE.match(text, position)
  178. if match is None:
  179. return "other"
  180. position = match.end()
  181. word = match.group(1).strip("'\"")
  182. if _ASSIGNMENT_RE.match(word) or word.lower() in _COMMAND_WRAPPERS or word.startswith("-"):
  183. continue # env assignments, wrappers and their flags (``sudo -E``)
  184. name = word.rsplit("/", 1)[-1].lower()
  185. if name in _COMMAND_KINDS:
  186. return _COMMAND_KINDS[name]
  187. return "python" if _PYTHON_RE.fullmatch(name) else "other"
  188. return "other"
  189. # Exit statuses a shell reports for a foreground child killed by SIGKILL / SIGTERM (OOM killer,
  190. # an external kill); a negative status is Python's own "terminated by signal N".
  191. _KILLED_STATUSES = frozenset({137, 143})
  192. def terminal_outcome(result: Any) -> str | None:
  193. """How one foreground command ended, from the backend's raw result dict. Hermes' own deadline
  194. and interrupt set flags on the result, so a command's own ``exit 124`` reads ``nonzero``."""
  195. if not isinstance(result, dict):
  196. return None
  197. if result.get("hermes_timed_out") is True:
  198. return "timeout"
  199. if result.get("hermes_interrupted") is True:
  200. return "killed"
  201. code = result.get("returncode")
  202. if not isinstance(code, int) or isinstance(code, bool):
  203. return None
  204. if code == 0:
  205. return "ok"
  206. return "killed" if code < 0 or code in _KILLED_STATUSES else "nonzero"
  207. def terminal_outcome_fields(*, command: Any, backend: Any, outcome: str) -> dict[str, str]:
  208. from .shared_metrics_contract import TERMINAL_BACKENDS
  209. backend_value = str(backend or "").strip().lower()
  210. return {
  211. "backend": backend_value if backend_value in TERMINAL_BACKENDS else "other",
  212. "command_kind": command_kind(command),
  213. "outcome": outcome,
  214. }
  215. def record_terminal_outcome(command: Any, backend: Any, result: Any = None, *, outcome: str | None = None) -> None:
  216. """Count one foreground terminal command that reached an exit status (or Hermes' deadline).
  217. Commands handed to the background, refused by a guard, or whose backend failed never ran to
  218. an exit status and are not counted here (``hermes.execution_backend.count`` covers them)."""
  219. try:
  220. if not _enabled() or _background_tool_call():
  221. return
  222. from .shared_metrics_loop import _UNMETERED
  223. if _UNMETERED.get():
  224. return
  225. resolved = outcome or terminal_outcome(result)
  226. if resolved is None:
  227. return
  228. _record("TERMINAL_OUTCOME_MARK", terminal_outcome_fields(command=command, backend=backend, outcome=resolved))
  229. except Exception:
  230. logger.debug("Shared-metrics terminal outcome not recorded", exc_info=True)
  231. # ---- per-turn agent state (loop guards, recovery) -----------------------------------------------
  232. @dataclass
  233. class _TurnState:
  234. """Per-turn harness bookkeeping; ``agent._harness_metrics_turn`` is reset at turn start."""
  235. round_outcomes: list[tuple[str, bool]] = field(default_factory=list)
  236. pending_errors: list[str] = field(default_factory=list)
  237. guard_signals: set[tuple[str, str]] = field(default_factory=set)
  238. def _turn_state(agent: Any) -> _TurnState:
  239. state = getattr(agent, "_harness_metrics_turn", None)
  240. if not isinstance(state, _TurnState):
  241. state = _TurnState()
  242. agent._harness_metrics_turn = state
  243. return state
  244. def _active(agent: Any) -> bool:
  245. return _user_agent(agent) and _enabled()
  246. # ---- loop guards -----------------------------------------------------------------------------
  247. _GUARD_DETECTORS = {
  248. **dict.fromkeys(("repeated_exact_failure_block", "repeated_exact_failure_warning"), "exact_failure"),
  249. **dict.fromkeys(("idempotent_no_progress_block", "idempotent_no_progress_warning"), "idempotent_no_progress"),
  250. **dict.fromkeys(("same_tool_failure_halt", "same_tool_failure_warning"), "same_tool_failure"),
  251. "identical_call_streak_halt": "identical_call_streak", "identical_call_streak": "identical_call_streak",
  252. "identical_cycle_halt": "identical_cycle", "identical_cycle": "identical_cycle",
  253. "loop_web_search_cap": "web_search_cap", "loop_subagent_cap": "subagent_cap",
  254. }
  255. def _record_guard(agent: Any, signal: str, detector: str) -> None:
  256. state = _turn_state(agent)
  257. if (signal, detector) in state.guard_signals:
  258. return # a detector that keeps firing is one stuck turn, not N
  259. state.guard_signals.add((signal, detector))
  260. _record("LOOP_GUARD_MARK", {**_route(agent), "detector": detector, "signal": signal})
  261. def record_guardrail_decision(agent: Any, action: Any, code: Any) -> None:
  262. """A tool-loop guardrail verdict: ``warn`` (the call ran, the model was told it is repeating)
  263. is ``repeated_tool_call``; ``block``/``halt`` (the harness stopped it) is ``loop_detected``.
  264. Once per turn per detector and signal."""
  265. try:
  266. detector = _GUARD_DETECTORS.get(str(code or ""))
  267. signal = {"warn": "repeated_tool_call", "block": "loop_detected", "halt": "loop_detected"}.get(str(action or ""))
  268. if detector is None or signal is None or not _active(agent):
  269. return
  270. _record_guard(agent, signal, detector)
  271. except Exception:
  272. logger.debug("Shared-metrics loop guard not recorded", exc_info=True)
  273. def record_guardrail_warnings(agent: Any, decision: Any, stall_kind: str | None) -> None:
  274. """The warnings one tool result carried: a guardrail ``warn`` verdict and/or an identical-call
  275. stall notice (``stall_kind`` names its detector)."""
  276. if getattr(decision, "action", None) == "warn":
  277. record_guardrail_decision(agent, "warn", getattr(decision, "code", None))
  278. if stall_kind:
  279. record_guardrail_decision(agent, "warn", stall_kind)
  280. _BUDGET_EXIT_PREFIXES = ("budget_exhausted", "max_iterations_reached(")
  281. # ---- recovery after a tool error -------------------------------------------------------------
  282. def _tool_name(name: Any) -> str:
  283. from .shared_metrics_contract import tool_metric_name
  284. return tool_metric_name({"tool_name": name})
  285. def observe_tool_outcome(agent: Any, tool_name: Any, is_error: bool) -> None:
  286. """Every committed tool result of the current round, in the model's emission order."""
  287. try:
  288. if _active(agent):
  289. _turn_state(agent).round_outcomes.append((str(tool_name or ""), bool(is_error)))
  290. except Exception:
  291. logger.debug("Shared-metrics tool outcome not observed", exc_info=True)
  292. def _record_recovery(agent: Any, route: dict[str, str], tool: str, next_tool: str, next_outcome: str) -> None:
  293. _record("TOOL_RECOVERY_MARK", {
  294. **route, "next_outcome": next_outcome, "next_tool": next_tool, "tool": _tool_name(tool),
  295. })
  296. def finish_tool_round(agent: Any) -> None:
  297. """Resolve the previous round's failed calls against this round's first calls, then carry
  298. this round's failures forward. The next call to the same tool is the retry; with none, the
  299. model switched tools and its first call is what it tried instead."""
  300. try:
  301. if not _active(agent):
  302. return
  303. state = _turn_state(agent)
  304. outcomes, state.round_outcomes = state.round_outcomes, []
  305. if not outcomes:
  306. return
  307. if state.pending_errors:
  308. route = _route(agent)
  309. for tool in state.pending_errors:
  310. same = next((failed for name, failed in outcomes if name == tool), None)
  311. failed = outcomes[0][1] if same is None else same
  312. _record_recovery(
  313. agent, route, tool, "different" if same is None else "same", "error" if failed else "success",
  314. )
  315. state.pending_errors = [name for name, failed in outcomes if failed]
  316. except Exception:
  317. logger.debug("Shared-metrics tool recovery not recorded", exc_info=True)
  318. def finish_turn(agent: Any, turn_exit_reason: Any, final_response: Any, *, interrupted: Any, failed: Any) -> None:
  319. """Turn end: the iteration cap, and failed calls the model never followed with another
  320. call — ``no_tool_call`` when it answered in text, ``gave_up`` when the turn ended without a
  321. reply from it (halted, budget spent, interrupted, errored)."""
  322. try:
  323. if not _active(agent):
  324. return
  325. state = _turn_state(agent)
  326. reason = str(turn_exit_reason or "")
  327. if reason.startswith(_BUDGET_EXIT_PREFIXES):
  328. _record_guard(agent, "iteration_cap", "iteration_budget")
  329. pending, state.pending_errors = state.pending_errors, []
  330. if not pending:
  331. return
  332. answered = (
  333. reason.startswith("text_response(") and bool(final_response) and not interrupted and not failed
  334. )
  335. route = _route(agent)
  336. for tool in pending:
  337. _record_recovery(agent, route, tool, "none", "no_tool_call" if answered else "gave_up")
  338. except Exception:
  339. logger.debug("Shared-metrics turn end not recorded", exc_info=True)
  340. # ---- model reply issues ------------------------------------------------------------------------
  341. _FINISH_ISSUES = {"content_filter": "refusal", "length": "truncated_length", "incomplete": "truncated_length"}
  342. def record_reply_finish(agent: Any, response: Any, finish_reason: Any) -> None:
  343. """A primary response whose structured finish reason is a refusal or an output-length stop
  344. (transports already promote Anthropic ``stop_reason=refusal`` and an OpenAI ``message.refusal``
  345. to ``content_filter``). Those mostly leave the loop before normal intake, so they are counted
  346. where the reason is derived; the latch keeps intake from counting the same response again."""
  347. try:
  348. issue = _FINISH_ISSUES.get(str(finish_reason or ""))
  349. if issue is None or not _active(agent):
  350. return
  351. agent._harness_metrics_reply = response
  352. _record("MODEL_REPLY_ISSUE_MARK", {**_route(agent), "issue": issue})
  353. except Exception:
  354. logger.debug("Shared-metrics reply finish not recorded", exc_info=True)
  355. def reply_content_issue(agent: Any, assistant_message: Any) -> str:
  356. if getattr(assistant_message, "tool_calls", None):
  357. return "none"
  358. content = getattr(assistant_message, "content", None)
  359. if isinstance(content, str) and content.strip() and agent._has_content_after_think_block(content):
  360. return "none"
  361. return "reasoning_only" if agent._extract_reasoning(assistant_message) else "empty"
  362. def record_reply_content(agent: Any, response: Any, assistant_message: Any) -> None:
  363. """Every other primary response: ``empty`` (no visible text, no tool call, no reasoning),
  364. ``reasoning_only`` (reasoning but nothing to show or run), else ``none`` — one row per
  365. response, so issue rates have their own denominator."""
  366. try:
  367. if not _active(agent):
  368. return
  369. if getattr(agent, "_harness_metrics_reply", None) is response:
  370. agent._harness_metrics_reply = None # already counted at the finish reason
  371. return
  372. _record("MODEL_REPLY_ISSUE_MARK", {**_route(agent), "issue": reply_content_issue(agent, assistant_message)})
  373. except Exception:
  374. logger.debug("Shared-metrics reply content not recorded", exc_info=True)