verification_stop.py 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244
  1. """Turn-end verification guard for coding edits. Policy-only: it never runs
  2. checks itself, it turns the passive verification ledger into a bounded follow-up
  3. when the model tries to finish right after editing code without fresh evidence."""
  4. from __future__ import annotations
  5. import os
  6. import tempfile
  7. from pathlib import Path
  8. from typing import Any, Iterable
  9. from agent.compression_marker import elide
  10. from agent.verification_evidence import last_unattributed_run
  11. _MAX_CHANGED_PATHS_IN_NUDGE = 8
  12. _MAX_STATUS_SUMMARY_CHARS = 1200
  13. _MAX_UNATTRIBUTED_COMMAND_CHARS = 200
  14. _UNATTRIBUTED_RUN_WARNING = (
  15. " Your last check run (`{command}`) did not count: the shell could not tell the "
  16. "check's own exit status apart from the commands around it. Run the check directly; "
  17. "if you pipe it, keep the check as the first command in the pipeline. The terminal "
  18. "already truncates long output and spills the rest to a file."
  19. )
  20. # Prose/data extensions and extension-less prose filenames (case-insensitive) with
  21. # no verifiable runtime behavior: a turn touching ONLY these suppresses the nudge
  22. # (a SKILL.md/README edit must never demand a /tmp verification script).
  23. _NON_CODE_VERIFY_EXTENSIONS = frozenset(
  24. {".md", ".markdown", ".mdx", ".rst", ".txt", ".text", ".adoc", ".asciidoc", ".org", ".log", ".csv", ".tsv"}
  25. )
  26. _NON_CODE_VERIFY_FILENAMES = frozenset(
  27. {"license", "licence", "notice", "authors", "contributors", "changelog", "codeowners"}
  28. )
  29. _FALSY_TOKENS = {"0", "false", "no", "off"}
  30. _TRUTHY_TOKENS = {"1", "true", "yes", "on"}
  31. def _is_non_code_path(raw: str) -> bool:
  32. """True when a changed path is documentation/prose with nothing to verify."""
  33. try:
  34. p = Path(str(raw))
  35. except Exception:
  36. return False
  37. suffix = p.suffix.lower()
  38. return suffix in _NON_CODE_VERIFY_EXTENSIONS or (not suffix and p.name.lower() in _NON_CODE_VERIFY_FILENAMES)
  39. def _session_is_messaging_surface() -> bool:
  40. """Whether this turn is delivered over a human messaging channel. An
  41. unreachable gateway package means no messaging channel (verify-on-stop stays on)."""
  42. try:
  43. from gateway.session_context import session_is_messaging_surface
  44. return session_is_messaging_surface()
  45. except Exception:
  46. return False
  47. def verify_on_stop_enabled(config: dict[str, Any] | None = None) -> bool:
  48. """Return whether edit -> verify-before-finish behavior is enabled.
  49. Precedence: ``HERMES_VERIFY_ON_STOP`` env var, then ``agent.verify_on_stop``
  50. config; default OFF (opt-in). A bool forces the behavior; ``"auto"`` is the
  51. legacy surface-aware mode: ON for interactive coding surfaces and
  52. programmatic callers, OFF for messaging surfaces where the verification
  53. narrative is chat noise. Missing/unrecognized values fall back to OFF.
  54. """
  55. env = os.environ.get("HERMES_VERIFY_ON_STOP")
  56. if env is not None:
  57. return env.strip().lower() not in _FALSY_TOKENS
  58. if config is None:
  59. try:
  60. from hermes_cli.config import load_config_readonly
  61. config = load_config_readonly()
  62. except Exception:
  63. config = {}
  64. agent_cfg = (config or {}).get("agent") if isinstance(config, dict) else None
  65. cfg_val = agent_cfg.get("verify_on_stop") if isinstance(agent_cfg, dict) else None
  66. if isinstance(cfg_val, bool):
  67. return cfg_val
  68. token = cfg_val.strip().lower() if isinstance(cfg_val, str) else ""
  69. if token == "auto":
  70. return not _session_is_messaging_surface()
  71. return token in _TRUTHY_TOKENS
  72. def _candidate_cwds(paths: Iterable[str]) -> list[Path]:
  73. """Distinct resolved directories (a file's parent) for the edited paths, in order."""
  74. seen: dict[str, None] = {}
  75. for raw in filter(None, paths):
  76. try:
  77. path = Path(raw).expanduser()
  78. seen.setdefault(str((path if path.is_dir() else path.parent).resolve()))
  79. except Exception:
  80. continue
  81. return [Path(p) for p in seen]
  82. def _verification_snapshot(
  83. *, session_id: str | None, changed_paths: list[str]
  84. ) -> tuple[dict[str, Any], dict[str, Any]] | None:
  85. """``(status, facts)`` for the first edited workspace needing proof, else the
  86. first recognized workspace when every one is ``passed``."""
  87. try:
  88. from agent.coding_context import project_facts_for
  89. from agent.verification_evidence import verification_status
  90. except Exception:
  91. return None
  92. first_snapshot: tuple[dict[str, Any], dict[str, Any]] | None = None
  93. for cwd in _candidate_cwds(changed_paths):
  94. facts = project_facts_for(cwd)
  95. if not facts:
  96. continue
  97. status = verification_status(session_id=session_id, cwd=cwd)
  98. first_snapshot = first_snapshot or (status, facts)
  99. if str(status.get("status") or "unverified") != "passed":
  100. return status, facts
  101. return first_snapshot
  102. def _format_changed_paths(paths: list[str]) -> str:
  103. lines = [f"- `{path}`" for path in paths[:_MAX_CHANGED_PATHS_IN_NUDGE]]
  104. if len(paths) > _MAX_CHANGED_PATHS_IN_NUDGE:
  105. lines.append(f"- ... and {len(paths) - _MAX_CHANGED_PATHS_IN_NUDGE} more")
  106. return "\n".join(lines)
  107. def _workspace_has_runnable_recipe(root: Any) -> bool:
  108. """Whether ``hermes verify`` has a runtime recipe here: a saved
  109. ``.hermes/environment.json`` or a statically detected recipe with a start
  110. command. Fail-silent and cheap — it only decorates the nudge text."""
  111. if not root:
  112. return False
  113. try:
  114. from agent.verify.environment import manifest_path
  115. from agent.verify.recipes import detect_recipe
  116. root_path = Path(str(root))
  117. if manifest_path(root_path).is_file():
  118. return True
  119. recipe = detect_recipe(root_path)
  120. return bool(recipe is not None and recipe.start)
  121. except Exception:
  122. return False
  123. def _status_detail(status: dict[str, Any]) -> str:
  124. state = str(status.get("status") or "unverified")
  125. evidence = status.get("evidence") if isinstance(status.get("evidence"), dict) else None
  126. if not evidence:
  127. return state
  128. command = evidence.get("canonical_command") or evidence.get("command")
  129. summary = str(evidence.get("output_summary") or "").strip()
  130. parts = [state]
  131. if command:
  132. parts.append(f"last command `{command}`")
  133. if summary:
  134. summary = elide(summary, _MAX_STATUS_SUMMARY_CHARS)
  135. parts.append(f"last output:\n{summary}")
  136. return "\n".join(parts)
  137. def build_verify_on_stop_nudge(
  138. *, session_id: str | None, changed_paths: Iterable[str], attempts: int=0, max_attempts: int=2,
  139. ) -> str | None:
  140. """Return a synthetic follow-up when edited code lacks fresh verification."""
  141. # Prose-only turns (markdown, skills, README, LICENSE, ...) have nothing to verify.
  142. paths = sorted({str(p) for p in changed_paths if p and not _is_non_code_path(p)})
  143. if not paths or attempts >= max_attempts:
  144. return None
  145. snapshot = _verification_snapshot(session_id=session_id, changed_paths=paths)
  146. if snapshot is None:
  147. return None
  148. status, facts = snapshot
  149. if str(status.get("status") or "unverified") == "passed":
  150. return None
  151. unattributed = last_unattributed_run(
  152. session_id=status.get("session_id"), root=status.get("root"),
  153. since=(status.get("evidence") or {}).get("created_at"))
  154. run_warning = _UNATTRIBUTED_RUN_WARNING.format(
  155. command=elide(unattributed, _MAX_UNATTRIBUTED_COMMAND_CHARS)) if unattributed else ""
  156. verify_commands = [str(cmd).strip() for cmd in (facts.get("verifyCommands") or []) if str(cmd).strip()]
  157. has_recipe = _workspace_has_runnable_recipe(facts.get("root"))
  158. # Optional shipped coding guidance, only paid when this evidence gate fires.
  159. try:
  160. from agent.verify_hooks import coding_verify_guidance
  161. guidance = coding_verify_guidance()
  162. except Exception:
  163. guidance = None
  164. addendum = f"\n\n{guidance}" if guidance else ""
  165. if verify_commands:
  166. command_instruction = (
  167. "Run the relevant verification command now ("
  168. + ", ".join(f"`{cmd}`" for cmd in verify_commands[:3])
  169. + (", ..." if len(verify_commands) > 3 else "")
  170. + "), read any failure, repair the code, and summarize what passed."
  171. )
  172. if has_recipe:
  173. command_instruction += (
  174. " For a full check including a runtime boot (build + test + "
  175. "start + readiness), prefer `hermes verify --json` — a passing "
  176. "run records verification evidence for this workspace."
  177. )
  178. elif has_recipe:
  179. command_instruction = (
  180. "No canonical test/lint/build command was detected, but the "
  181. "project has a runnable verification recipe. Run `hermes verify "
  182. "--json` (detect -> build -> test -> boot -> readiness poll); a "
  183. "passing run records verification evidence for this workspace. "
  184. "Read any failure, repair the code, and summarize what passed."
  185. )
  186. else:
  187. temp_dir = os.path.realpath(tempfile.gettempdir())
  188. command_instruction = (
  189. "No canonical test/lint/build command was detected. Create a focused "
  190. f"temporary verification script under `{temp_dir}` using an OS-safe "
  191. "`tempfile` path with a `hermes-verify-` filename prefix, run it "
  192. "against the changed behavior, clean it up when possible, and "
  193. "summarize it explicitly as ad-hoc verification rather than suite "
  194. "green."
  195. )
  196. return (
  197. "[System: You edited code in this turn, but the workspace does not have "
  198. "fresh passing verification evidence yet.\n\n"
  199. f"Verification status: {_status_detail(status)}\n\n"
  200. f"Changed paths:\n{_format_changed_paths(paths)}\n\n"
  201. f"{command_instruction}{run_warning} If verification is not possible, "
  202. "explain the concrete blocker instead of claiming the work is fully verified."
  203. f"{addendum}]"
  204. )
  205. __all__ = ["build_verify_on_stop_nudge", "verify_on_stop_enabled"]