verification_evidence.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805
  1. """Coding verification evidence ledger: records what the agent actually proved in
  2. a code workspace. Deliberately passive — it never runs a suite, never blocks
  3. completion, and never upgrades targeted checks into "repo green"."""
  4. from __future__ import annotations
  5. import json
  6. import re
  7. import shlex
  8. import sqlite3
  9. import tempfile
  10. import threading
  11. from dataclasses import dataclass
  12. from datetime import datetime, timedelta, timezone, UTC
  13. from pathlib import Path
  14. from typing import Any, Optional
  15. from hermes_constants import get_hermes_home
  16. _DB_LOCK = threading.Lock()
  17. _MAX_OUTPUT_SUMMARY_CHARS = 2000
  18. _MAX_EVIDENCE_AGE_DAYS = 30
  19. _MAX_EVENTS_PER_SESSION_ROOT = 100
  20. _MAX_TOTAL_UNREFERENCED_EVENTS = 10_000
  21. _AD_HOC_SCRIPT_NAME_PREFIXES = ("hermes-verify-", "hermes-ad-hoc-")
  22. _VERIFY_SCHEMA_VERSION = 1
  23. # Latest verify-command run per (session, root) whose own exit status the shell hid, with the
  24. # time it ran. Read by the stop nudge in this same process, so it stays in memory; bounded for
  25. # long-lived gateways.
  26. _UNATTRIBUTED_RUNS: dict[tuple[str, str], tuple[str, str]] = {}
  27. _MAX_UNATTRIBUTED_RUNS = 256
  28. _INTERPRETERS = {"python", "python3", "py", "node", "bash", "sh", "ruby", "perl"}
  29. # Windows spells the same interpreters `python.exe` / `py.exe` (a venv's absolute Scripts path).
  30. _WINDOWS_EXE_SUFFIX_RE = re.compile(r"\.(?:exe|bat|cmd)$", re.IGNORECASE)
  31. # The same interpreter is reached as `python3`, `python3.12`, `/usr/bin/python3.12`, or
  32. # `env python3`. Matching only the bare token recorded no evidence for the invocation shapes
  33. # the verify-on-stop nudge itself hands the agent, so a passing run left the workspace
  34. # "unverified" and the nudge returned every turn.
  35. _INTERPRETER_NAME_RE = re.compile(
  36. r"^(?:" + "|".join(sorted(_INTERPRETERS, key=len, reverse=True)) + r")(?:[0-9]+(?:\.[0-9]+)*)?$"
  37. )
  38. _TARGET_EXTENSIONS = (".py", ".js", ".jsx", ".ts", ".tsx", ".rs", ".go", ".java")
  39. _TARGET_PREFIXES = ("test_", "tests", "spec", "__tests__")
  40. # Ordered: first matching keyword group wins; "check" only counts when the
  41. # command is not itself a test command; anything else is a test.
  42. _KIND_KEYWORDS = (
  43. (("lint", "eslint", "ruff"), "lint"),
  44. (("typecheck", "tsc", "mypy", "pyright", "ty"), "typecheck"), (("build",), "build"),
  45. (("fmt", "format"), "format"),
  46. )
  47. _PYTEST_SPELLINGS = (
  48. ["python", "-m", "pytest"], ["python3", "-m", "pytest"],
  49. ["uv", "run", "pytest"], ["poetry", "run", "pytest"], ["pipenv", "run", "pytest"],
  50. )
  51. _SCHEMA_DDL = (
  52. """
  53. CREATE TABLE IF NOT EXISTS meta (
  54. key TEXT PRIMARY KEY,
  55. value TEXT NOT NULL
  56. )
  57. """,
  58. """
  59. CREATE TABLE IF NOT EXISTS verification_events (
  60. id INTEGER PRIMARY KEY AUTOINCREMENT,
  61. created_at TEXT NOT NULL,
  62. session_id TEXT NOT NULL,
  63. cwd TEXT NOT NULL,
  64. root TEXT NOT NULL,
  65. command TEXT NOT NULL,
  66. canonical_command TEXT NOT NULL,
  67. kind TEXT NOT NULL,
  68. scope TEXT NOT NULL,
  69. status TEXT NOT NULL,
  70. exit_code INTEGER NOT NULL,
  71. output_summary TEXT NOT NULL
  72. )
  73. """,
  74. """
  75. CREATE TABLE IF NOT EXISTS verification_state (
  76. session_id TEXT NOT NULL,
  77. root TEXT NOT NULL,
  78. last_event_id INTEGER,
  79. last_edit_at TEXT,
  80. changed_paths_json TEXT NOT NULL DEFAULT '[]',
  81. PRIMARY KEY (session_id, root)
  82. )
  83. """,
  84. """
  85. CREATE INDEX IF NOT EXISTS idx_verification_events_session_root
  86. ON verification_events(session_id, root, id DESC)
  87. """,
  88. )
  89. @dataclass(frozen=True)
  90. class _ShellSegment:
  91. tokens: list[str]
  92. following_operator: str | None = None
  93. @dataclass(frozen=True)
  94. class VerificationEvidence:
  95. """A classified command result worth recording."""
  96. command: str
  97. canonical_command: str
  98. kind: str
  99. scope: str
  100. status: str
  101. exit_code: int
  102. cwd: str
  103. root: str
  104. session_id: str
  105. output_summary: str = ""
  106. def _utc_now() -> str:
  107. return datetime.now(UTC).isoformat()
  108. def _db_path() -> Path:
  109. return get_hermes_home() / "verification_evidence.db"
  110. def _ledger_enabled() -> bool:
  111. """The ledger exists only to feed verify-on-stop; when that guard is off nothing may
  112. record, read, or even create the database (an unconsumed ledger is pure disk churn)."""
  113. from agent.verification_stop import verify_on_stop_enabled
  114. return verify_on_stop_enabled()
  115. def _connect() -> sqlite3.Connection:
  116. from hermes_cli.sqlite_util import open_db
  117. return open_db(_db_path(), db_label="verification_evidence.db", initialize=_ensure_schema)
  118. def _transaction():
  119. from hermes_cli.sqlite_util import transaction
  120. return transaction(_connect())
  121. def _ensure_schema(conn: sqlite3.Connection) -> None:
  122. for ddl in _SCHEMA_DDL:
  123. conn.execute(ddl)
  124. conn.execute(
  125. "INSERT OR REPLACE INTO meta(key, value) VALUES ('schema_version', ?)",
  126. (str(_VERIFY_SCHEMA_VERSION),),
  127. )
  128. conn.commit()
  129. def _split_shell_segments(command: str, *, posix: bool = True) -> list[_ShellSegment]:
  130. """Tokenize top-level shell commands while preserving their control operators.
  131. Returns ``[]`` for anything unparseable (unbalanced quotes, empty segment,
  132. trailing operator other than ``;``) so callers never match a partial parse.
  133. """
  134. raw_segments: list[tuple[str, str | None]] = []
  135. start = 0
  136. quote: str | None = None
  137. escaped = False
  138. index = 0
  139. while index < len(command):
  140. char = command[index]
  141. if escaped or (char == "\\" and quote != "'"):
  142. escaped = not escaped # consume the escaped char / start an escape
  143. index += 1
  144. continue
  145. if quote or char in "'\"":
  146. quote = None if char == quote else (quote or char) # close / stay / open
  147. index += 1
  148. continue
  149. operator = None
  150. if command.startswith(("&&", "||", "|&"), index):
  151. operator = command[index:index + 2]
  152. elif char == "\n":
  153. operator = ";"
  154. elif char in ";|" or (
  155. char == "&"
  156. and (index == 0 or command[index - 1] not in "<>")
  157. and not command.startswith(("&>", "&>>"), index)
  158. ):
  159. operator = char
  160. if operator is None:
  161. index += 1
  162. continue
  163. raw = command[start:index].strip()
  164. if not raw:
  165. return []
  166. raw_segments.append((raw, operator))
  167. index += 1 if char == "\n" else len(operator)
  168. start = index
  169. if quote or escaped:
  170. return []
  171. trailing = command[start:].strip()
  172. if trailing:
  173. raw_segments.append((trailing, None))
  174. elif raw_segments and raw_segments[-1][1] != ";":
  175. return []
  176. try:
  177. segments = [_ShellSegment(shlex.split(raw, posix=posix), operator) for raw, operator in raw_segments]
  178. except ValueError:
  179. return []
  180. return segments if all(s.tokens for s in segments) else []
  181. def _segment_status(
  182. segments: list[_ShellSegment], match_index: int, exit_code: int,
  183. pipeline_status: Optional[list[int]] = None,
  184. ) -> Optional[int]:
  185. """The matched segment's own exit status when the shell proves it, else ``None``.
  186. Only the last ``;``-separated sequence reports its status; backgrounding and ``||``
  187. hide it; an ``&&`` chain proves each member only on success. A pipeline hides every
  188. status but the last segment's, unless ``pipeline_status`` (the shell's PIPESTATUS
  189. array) is supplied and the match leads the sequence's last pipeline. Behind ``&&``
  190. that pipeline may never have run, leaving PIPESTATUS from an earlier one
  191. (``true | false && pytest | tail``); a short-circuited chain exits non-zero, so
  192. there the array counts only when the shell's own code is 0, as for plain ``&&``.
  193. """
  194. if not segments or not 0 <= match_index < len(segments) or any(s.following_operator == "&" for s in segments):
  195. return None
  196. sequence_start = max(
  197. (i + 1 for i, s in enumerate(segments[:-1]) if s.following_operator == ";"), default=0
  198. )
  199. if match_index < sequence_start:
  200. return None
  201. operators = {segment.following_operator for segment in segments[sequence_start:-1]}
  202. if "||" in operators:
  203. return None
  204. if operators & {"|", "|&"}:
  205. pipeline_start = max(
  206. (
  207. sequence_start + index + 1
  208. for index, operator in enumerate(
  209. segment.following_operator for segment in segments[sequence_start:-1]
  210. )
  211. if operator not in {"|", "|&"}
  212. ),
  213. default=sequence_start,
  214. )
  215. chain_ran = "&&" not in operators or int(exit_code) == 0
  216. if chain_ran and pipeline_status and len(pipeline_status) >= 2 and match_index == pipeline_start:
  217. return pipeline_status[0]
  218. return None
  219. if not operators:
  220. return int(exit_code)
  221. if operators == {"&&"}:
  222. return int(exit_code) if int(exit_code) == 0 else None
  223. return None
  224. def _canonical_tokens(canonical: str) -> list[str]:
  225. """Tokenize a canonical command, stripping leading ``./`` from each token."""
  226. try:
  227. return [re.sub(r"^(?:\./)+", "", t.strip()) for t in shlex.split(canonical) if t]
  228. except ValueError:
  229. return []
  230. # Status-preserving wrappers -> (options that still run the child, options taking a value).
  231. # Anything else (``--help``, ``--version``) may exit without running the child: no match.
  232. _WRAPPER_OPTIONS: dict[str, tuple[frozenset[str], frozenset[str]]] = {
  233. "timeout": (frozenset({"-p", "--preserve-status", "-f", "--foreground", "-v", "--verbose"}),
  234. frozenset({"-s", "--signal", "-k", "--kill-after"})),
  235. "nice": (frozenset(), frozenset({"-n", "--adjustment"})),
  236. "nohup": (frozenset(), frozenset()),
  237. "stdbuf": (frozenset(), frozenset({"-i", "-o", "-e", "--input", "--output", "--error"})),
  238. }
  239. # A ``timeout`` duration: ``300``, ``5m``, ``1.5h``.
  240. _TIMEOUT_DURATION_RE = re.compile(r"^\d+(?:\.\d+)?[smhd]?$")
  241. def _wrapper_option_width(token: str, flags: frozenset[str], valued: frozenset[str]) -> int:
  242. """Tokens one wrapper option spans (``-n 5`` 2, ``-n5``/``--signal=KILL`` 1), 0 if unknown."""
  243. if token in flags:
  244. return 1
  245. if token in valued:
  246. return 2
  247. attached = any(token.startswith(f"{v}=") if v.startswith("--") else token.startswith(v) for v in valued)
  248. return 1 if attached else 0
  249. def _skip_wrapper(tokens: list[str], i: int) -> Optional[int]:
  250. """Index of the child command after the wrapper at ``tokens[i]``, ``None`` unless the
  251. wrapper's options and arguments are ones that run the child."""
  252. flags, valued = _WRAPPER_OPTIONS[tokens[i]]
  253. timeout = tokens[i] == "timeout"
  254. i += 1
  255. while i < len(tokens) and tokens[i].startswith("-") and tokens[i] != "--":
  256. width = _wrapper_option_width(tokens[i], flags, valued)
  257. if not width:
  258. return None
  259. i += width
  260. i += int(i < len(tokens) and tokens[i] == "--")
  261. if timeout:
  262. if i >= len(tokens) or not _TIMEOUT_DURATION_RE.match(tokens[i]):
  263. return None
  264. i += 1
  265. return i
  266. def _strip_command_prefix(tokens: list[str]) -> list[str]:
  267. """Remove harmless command prefixes: env assignments, command/time/noglob, and
  268. status-preserving wrappers with the options and duration that still run the child."""
  269. i = 1 if tokens and tokens[0] == "env" else 0
  270. while i < len(tokens) and "=" in tokens[i] and not tokens[i].startswith("-"):
  271. i += 1
  272. while i < len(tokens):
  273. if tokens[i] in {"command", "time", "noglob"}:
  274. i += 1
  275. elif tokens[i] in _WRAPPER_OPTIONS and (child := _skip_wrapper(tokens, i)) is not None:
  276. i = child
  277. else:
  278. break
  279. return list(tokens[i:])
  280. def _interpreter_family(token: str) -> Optional[str]:
  281. """Family name for an interpreter token wherever it lives, else ``None``.
  282. ``./venv/bin/python3.12``, ``python3`` and ``/usr/bin/python`` are all ``python``:
  283. the detected command is ``pytest`` and the agent runs it through a venv interpreter.
  284. """
  285. name = token.replace("\\", "/").rsplit("/", 1)[-1]
  286. name = _WINDOWS_EXE_SUFFIX_RE.sub("", re.sub(r"^(?:\./)+", "", name))
  287. if not _INTERPRETER_NAME_RE.match(name):
  288. return None
  289. family = re.sub(r"[0-9.]+$", "", name)
  290. return "python" if family == "py" else family
  291. _INTERPRETER_FLAG_RE = re.compile(r"^-(?:[IBuEsSOd]+|3(?:\.\d+)?|[WX].*)$")
  292. def _is_harmless_interpreter_flag(token: str) -> bool:
  293. return bool(_INTERPRETER_FLAG_RE.match(token))
  294. def _normalize_tokens(tokens: list[str]) -> list[str]:
  295. """Comparison form: no leading ``./``, interpreters collapsed, Python's harmless flags
  296. dropped. Only Python's: the same letters mean other things elsewhere (``bash -s``
  297. reads commands from stdin and never runs the script named after it)."""
  298. normalized: list[str] = []
  299. i = 0
  300. while i < len(tokens):
  301. family = _interpreter_family(tokens[i])
  302. normalized.append(family or re.sub(r"^(?:\./)+", "", tokens[i]))
  303. i += 1
  304. if family == "python":
  305. while i < len(tokens) and _is_harmless_interpreter_flag(tokens[i]):
  306. i += 2 if tokens[i] in {"-W", "-X"} else 1
  307. return normalized
  308. def _equivalent_needles(needle: list[str]) -> list[list[str]]:
  309. """Return command spellings equivalent to the detected canonical command."""
  310. candidates = [needle]
  311. if len(needle) >= 3 and needle[1] == "run" and needle[0] in {"npm", "pnpm", "yarn", "bun"}:
  312. candidates.append([needle[0], needle[2]])
  313. if len(needle) == 1 and "/" in needle[0]:
  314. candidates += [["bash", needle[0]], ["sh", needle[0]]]
  315. if needle == ["pytest"]:
  316. candidates += _PYTEST_SPELLINGS
  317. return candidates
  318. def _find_canonical_match(
  319. command: str, canonical_commands: list[str], exit_code: int,
  320. pipeline_status: Optional[list[int]] = None, *, attributed: bool = True,
  321. ) -> Optional[tuple[str, list[str], Optional[int]]]:
  322. """Return ``(canonical, trailing_args, status)`` for the first detected command.
  323. ``attributed=False`` also matches a run whose own status the shell hid (status ``None``).
  324. """
  325. segments = _split_shell_segments(command)
  326. for canonical in canonical_commands:
  327. needle = _canonical_tokens(canonical)
  328. if not needle:
  329. continue
  330. for index, segment in enumerate(segments):
  331. candidate_tokens = _normalize_tokens(_strip_command_prefix(segment.tokens))
  332. status = _segment_status(segments, index, exit_code, pipeline_status)
  333. if status is None and attributed:
  334. continue
  335. for candidate in _equivalent_needles(needle):
  336. candidate = _normalize_tokens(candidate)
  337. if candidate_tokens[:len(candidate)] == candidate:
  338. return canonical, candidate_tokens[len(candidate):], status
  339. return None
  340. def _kind_for_command(canonical: str) -> str:
  341. lowered = canonical.lower()
  342. kind = next((k for words, k in _KIND_KEYWORDS if any(w in lowered for w in words)), None)
  343. return kind or ("check" if "check" in lowered and "test" not in lowered else "test")
  344. def _looks_like_target(arg: str) -> bool:
  345. return bool(arg) and not arg.startswith("-") and "=" not in arg and (
  346. any(m in arg for m in ("/", "\\", "::")) or arg.endswith(_TARGET_EXTENSIONS) or arg.startswith(_TARGET_PREFIXES)
  347. )
  348. def _is_under(token: str, base: str | Path | None) -> bool:
  349. """Whether absolute path ``token`` is ``base`` or lies beneath it (resolved)."""
  350. if not base:
  351. return False
  352. try:
  353. path = Path(token).expanduser()
  354. if not path.is_absolute():
  355. return False
  356. resolved, base_path = path.resolve(), Path(base).expanduser().resolve()
  357. return resolved == base_path or base_path in resolved.parents
  358. except Exception:
  359. return False
  360. def _is_temp_script_path(token: str, root: str | Path | None) -> bool:
  361. """An ad-hoc verify script: prefixed name, under the temp dir, outside the repo."""
  362. try:
  363. name = Path(token).expanduser().name
  364. except Exception:
  365. return False
  366. return name.startswith(_AD_HOC_SCRIPT_NAME_PREFIXES) and _is_under(token, tempfile.gettempdir()) and not _is_under(token, root)
  367. def _is_interpreter_token(token: str) -> bool:
  368. """Whether a command word names a shell interpreter.
  369. ``python3``, ``python3.12`` and ``/usr/bin/python3.12`` are the same interpreter; matching
  370. only the bare token left the ad-hoc branch blind to the invocation shapes the nudge hands
  371. the agent, so a passing run recorded no evidence and the workspace stayed ``unverified``.
  372. """
  373. # Basename by hand: on POSIX ``Path`` treats a Windows backslash path as one component.
  374. name = token.replace("\\", "/").rsplit("/", 1)[-1]
  375. return bool(_INTERPRETER_NAME_RE.match(_WINDOWS_EXE_SUFFIX_RE.sub("", name)))
  376. def _ad_hoc_script_args(tokens: list[str], root: str | Path | None) -> Optional[list[str]]:
  377. candidate_tokens = _strip_command_prefix(tokens)
  378. if not candidate_tokens:
  379. return None
  380. command = candidate_tokens[0]
  381. if _is_temp_script_path(command, root):
  382. return candidate_tokens[1:]
  383. if Path(command).name == "env" and len(candidate_tokens) > 1 and _is_interpreter_token(candidate_tokens[1]):
  384. # `/usr/bin/env python3 script` names the same interpreter as `python3 script`.
  385. candidate_tokens, command = candidate_tokens[1:], candidate_tokens[1]
  386. if _is_interpreter_token(command):
  387. # Skip interpreter flags; the first positional must be the script.
  388. for idx, token in enumerate(candidate_tokens[1:], start=1):
  389. if _is_temp_script_path(token, root):
  390. return candidate_tokens[idx + 1:]
  391. if token != "--" and not token.startswith("-"):
  392. return None
  393. return None
  394. def _find_ad_hoc_match(command: str, root: str | Path | None, exit_code: int = 0) -> Optional[list[str]]:
  395. # posix=False is retried so Windows backslash script paths survive splitting.
  396. for posix in (True, False):
  397. segments = _split_shell_segments(command, posix=posix)
  398. for index, segment in enumerate(segments):
  399. trailing_args = _ad_hoc_script_args(segment.tokens, root)
  400. if trailing_args is not None and _segment_status(segments, index, exit_code) is not None:
  401. return trailing_args
  402. return None
  403. def _summarize_output(output: str) -> str:
  404. text = (output or "").strip()
  405. if len(text) <= _MAX_OUTPUT_SUMMARY_CHARS:
  406. return text
  407. head = _MAX_OUTPUT_SUMMARY_CHARS // 3
  408. return (
  409. f"{text[:head]}\n... [{len(text) - _MAX_OUTPUT_SUMMARY_CHARS} chars omitted] ...\n"
  410. f"{text[head - _MAX_OUTPUT_SUMMARY_CHARS:]}"
  411. )
  412. def _prune_old_events(conn: sqlite3.Connection, *, session_id: str, root: str) -> None:
  413. """Bound ledger growth without deleting the current state pointer.
  414. Order matters: per-(session, root) cap, expire stale state rows, then expire
  415. old events and cap the total — never dropping an event still referenced
  416. by a ``verification_state.last_event_id``.
  417. """
  418. cutoff = (datetime.now(UTC) - timedelta(days=_MAX_EVIDENCE_AGE_DAYS)).isoformat()
  419. conn.execute(
  420. "DELETE FROM verification_events WHERE session_id = ? AND root = ? AND id NOT IN ("
  421. " SELECT id FROM verification_events WHERE session_id = ? AND root = ?"
  422. " ORDER BY id DESC LIMIT ?)",
  423. (session_id, root, session_id, root, _MAX_EVENTS_PER_SESSION_ROOT),
  424. )
  425. conn.execute(
  426. "DELETE FROM verification_state"
  427. " WHERE (last_edit_at IS NOT NULL AND last_edit_at < ?)"
  428. " OR (last_edit_at IS NULL AND last_event_id IN ("
  429. " SELECT id FROM verification_events WHERE created_at < ?))",
  430. (cutoff, cutoff),
  431. )
  432. conn.execute(
  433. "DELETE FROM verification_events WHERE created_at < ? AND id NOT IN ("
  434. " SELECT last_event_id FROM verification_state WHERE last_event_id IS NOT NULL)",
  435. (cutoff,),
  436. )
  437. conn.execute(
  438. "DELETE FROM verification_events WHERE id NOT IN ("
  439. " SELECT id FROM verification_events ORDER BY id DESC LIMIT ?)"
  440. " AND id NOT IN ("
  441. " SELECT last_event_id FROM verification_state WHERE last_event_id IS NOT NULL)",
  442. (_MAX_TOTAL_UNREFERENCED_EVENTS,),
  443. )
  444. def _project_facts(cwd: str | Path | None) -> Optional[dict[str, Any]]:
  445. """Workspace facts for ``cwd``; ``None`` when detection fails or finds nothing."""
  446. try:
  447. from agent.coding_context import project_facts_for
  448. return project_facts_for(cwd)
  449. except Exception:
  450. return None
  451. def _root_for(facts: dict[str, Any] | None, cwd: str | Path | None) -> str:
  452. return str((facts or {}).get("root") or Path(cwd or ".").resolve())
  453. def _load_changed_paths(raw: Any) -> list[Any]:
  454. try:
  455. return json.loads(raw or "[]")
  456. except (TypeError, ValueError):
  457. return []
  458. def classify_verification_command(
  459. command: str, *, cwd: str | Path | None = None, session_id: str | None = None, exit_code: int = 0,
  460. output: str = "", pipeline_status: Optional[list[int]] = None,
  461. ) -> Optional[VerificationEvidence]:
  462. """Classify a terminal command as verification evidence, if applicable.
  463. ``pipeline_status`` is the shell's ``PIPESTATUS`` array: when the verifier is the
  464. leading command of a pipeline, its own status comes from there rather than from the
  465. shell's code (which belongs to the last filter).
  466. Ad-hoc temp scripts only count when the project has no canonical verify
  467. commands at all, so they never shadow a real suite.
  468. """
  469. if not command or not isinstance(command, str):
  470. return None
  471. facts = _project_facts(cwd)
  472. if not facts:
  473. return None
  474. return _classify_in_workspace(
  475. command, facts, cwd=cwd, session_id=session_id, exit_code=exit_code, output=output,
  476. pipeline_status=pipeline_status,
  477. )
  478. def _classify_in_workspace(
  479. command: str, facts: dict[str, Any], *, cwd: str | Path | None, session_id: str | None,
  480. exit_code: int, output: str, pipeline_status: Optional[list[int]],
  481. ) -> Optional[VerificationEvidence]:
  482. verify_commands = list(facts.get("verifyCommands") or [])
  483. match = _find_canonical_match(command, verify_commands, int(exit_code), pipeline_status)
  484. if match is not None and match[2] is not None:
  485. canonical, trailing_args, status = match
  486. kind = _kind_for_command(canonical)
  487. scope = "targeted" if any(map(_looks_like_target, trailing_args)) else "full"
  488. else:
  489. if verify_commands or _find_ad_hoc_match(command, facts.get("root"), int(exit_code)) is None:
  490. return None
  491. canonical, kind, scope = "ad-hoc verification script", "ad_hoc", "targeted"
  492. status = int(exit_code)
  493. return VerificationEvidence(
  494. command=command, canonical_command=canonical, kind=kind, scope=scope,
  495. status="passed" if int(status) == 0 else "failed", exit_code=int(exit_code),
  496. cwd=str(Path(cwd or ".").resolve()), root=_root_for(facts, cwd),
  497. session_id=str(session_id or "default"), output_summary=_summarize_output(output),
  498. )
  499. def record_terminal_result(
  500. *, command: str, cwd: str | Path | None, session_id: str | None, exit_code: int, output: str = "",
  501. pipeline_status: Optional[list[int]] = None,
  502. ) -> Optional[dict[str, Any]]:
  503. """Record a foreground terminal result when it is verification evidence.
  504. A run of a detected verify command whose own status the shell hid (``pytest || true``,
  505. a pipe on a backend without ``PIPESTATUS``) is remembered so the stop nudge can say why
  506. it did not count.
  507. """
  508. if not _ledger_enabled() or not command or not isinstance(command, str):
  509. return None
  510. facts = _project_facts(cwd)
  511. if not facts:
  512. return None
  513. key = (str(session_id or "default"), _root_for(facts, cwd))
  514. evidence = _classify_in_workspace(
  515. command, facts, cwd=cwd, session_id=session_id, exit_code=exit_code, output=output,
  516. pipeline_status=pipeline_status,
  517. )
  518. if evidence is not None:
  519. return _insert_evidence(evidence)
  520. verify_commands = list(facts.get("verifyCommands") or [])
  521. if _find_canonical_match(command, verify_commands, int(exit_code), attributed=False):
  522. _UNATTRIBUTED_RUNS.pop(key, None)
  523. _UNATTRIBUTED_RUNS[key] = (command, _utc_now())
  524. while len(_UNATTRIBUTED_RUNS) > _MAX_UNATTRIBUTED_RUNS:
  525. _UNATTRIBUTED_RUNS.pop(next(iter(_UNATTRIBUTED_RUNS)))
  526. return None
  527. def last_unattributed_run(
  528. *, session_id: str | None, root: str | None, since: str | None = None,
  529. ) -> Optional[str]:
  530. """The latest verify-command run in ``root`` whose own exit status could not be read,
  531. unless evidence recorded at ``since`` came after it. ``hermes verify`` usually records
  532. from its own process, so this process's memory alone cannot know about that run.
  533. """
  534. run = _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root)))
  535. if run is None or (since and since >= run[1]):
  536. return None
  537. return run[0]
  538. def record_verify_run(
  539. *, root: str | Path, session_id: str | None = None, ok: bool, command: str = "hermes verify",
  540. scope: str = "full", output: str = "",
  541. ) -> Optional[dict[str, Any]]:
  542. """Record a completed ``hermes verify`` run as verification evidence.
  543. A pass marks the workspace ``passed`` for the verify-on-stop guard like a
  544. canonical test command would. ``root`` is re-resolved through project facts
  545. so it matches what :func:`verification_status` derives later.
  546. """
  547. if not _ledger_enabled():
  548. return None
  549. resolved = str(Path(root).resolve())
  550. return _insert_evidence(VerificationEvidence(
  551. command=command, canonical_command="hermes verify", kind="verify",
  552. scope=scope if scope in {"full", "targeted"} else "full",
  553. status="passed" if ok else "failed", exit_code=0 if ok else 1, cwd=resolved,
  554. root=str((_project_facts(root) or {}).get("root") or resolved),
  555. session_id=str(session_id or "default"), output_summary=_summarize_output(output),
  556. ))
  557. def _insert_evidence(evidence: VerificationEvidence) -> dict[str, Any]:
  558. """Insert a classified evidence row and repoint the workspace state."""
  559. created_at = _utc_now()
  560. e = evidence
  561. _UNATTRIBUTED_RUNS.pop((e.session_id, e.root), None)
  562. with _DB_LOCK, _transaction() as conn:
  563. cur = conn.execute(
  564. "INSERT INTO verification_events("
  565. " created_at, session_id, cwd, root, command, canonical_command,"
  566. " kind, scope, status, exit_code, output_summary"
  567. ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
  568. (created_at, e.session_id, e.cwd, e.root, e.command, e.canonical_command, e.kind, e.scope,
  569. e.status, e.exit_code, e.output_summary),
  570. )
  571. if cur.lastrowid is None:
  572. raise RuntimeError("verification event insert did not return an id")
  573. event_id = int(cur.lastrowid)
  574. conn.execute(
  575. "INSERT INTO verification_state("
  576. " session_id, root, last_event_id, last_edit_at, changed_paths_json"
  577. ") VALUES (?, ?, ?, NULL, '[]')"
  578. " ON CONFLICT(session_id, root) DO UPDATE SET"
  579. " last_event_id = excluded.last_event_id,"
  580. " last_edit_at = NULL,"
  581. " changed_paths_json = '[]'",
  582. (e.session_id, e.root, event_id),
  583. )
  584. # A full passing ``hermes verify`` proves the workspace. Partial or
  585. # failed runs must not wipe another session's edit record (#103650).
  586. if e.kind == "verify" and e.status == "passed" and e.scope == "full":
  587. conn.execute(
  588. "UPDATE verification_state"
  589. " SET last_event_id = ?, last_edit_at = NULL, changed_paths_json = '[]'"
  590. " WHERE root = ? AND session_id != ?",
  591. (event_id, e.root, e.session_id),
  592. )
  593. _prune_old_events(conn, session_id=e.session_id, root=e.root)
  594. conn.commit()
  595. return {"id": event_id, **e.__dict__, "created_at": created_at}
  596. def mark_workspace_edited(
  597. *, session_id: str | None, cwd: str | Path | None, paths: list[str] | tuple[str, ...] | None = None
  598. ) -> Optional[dict[str, Any]]:
  599. """Mark verification evidence stale after a successful file edit."""
  600. if not _ledger_enabled():
  601. return None
  602. facts = _project_facts(cwd)
  603. if not facts:
  604. return None
  605. sid = str(session_id or "default")
  606. root = _root_for(facts, cwd)
  607. changed_paths = sorted({str(p) for p in (paths or []) if p})
  608. edited_at = _utc_now()
  609. with _DB_LOCK, _transaction() as conn:
  610. row = conn.execute(
  611. "SELECT changed_paths_json FROM verification_state WHERE session_id = ? AND root = ?",
  612. (sid, root),
  613. ).fetchone()
  614. # Merge with what was already recorded, bounded to the last 200 paths.
  615. existing = _load_changed_paths(row["changed_paths_json"]) if row is not None else []
  616. merged = sorted(set(existing) | set(changed_paths))[-200:]
  617. conn.execute(
  618. "INSERT INTO verification_state("
  619. " session_id, root, last_event_id, last_edit_at, changed_paths_json"
  620. ") VALUES (?, ?, NULL, ?, ?)"
  621. " ON CONFLICT(session_id, root) DO UPDATE SET"
  622. " last_edit_at = excluded.last_edit_at,"
  623. " changed_paths_json = excluded.changed_paths_json",
  624. (sid, root, edited_at, json.dumps(merged)),
  625. )
  626. conn.commit()
  627. return {"session_id": sid, "root": root, "last_edit_at": edited_at, "changed_paths": changed_paths}
  628. def _newest_workspace_verify(
  629. conn: sqlite3.Connection, *, root: str, since: str | None
  630. ) -> Optional[sqlite3.Row]:
  631. """Latest full passing ``hermes verify`` for ``root``, optionally not older than ``since``."""
  632. if since:
  633. return conn.execute(
  634. "SELECT * FROM verification_events"
  635. " WHERE root = ? AND kind = 'verify' AND status = 'passed' AND scope = 'full'"
  636. " AND created_at >= ?"
  637. " ORDER BY id DESC LIMIT 1",
  638. (root, since),
  639. ).fetchone()
  640. return conn.execute(
  641. "SELECT * FROM verification_events"
  642. " WHERE root = ? AND kind = 'verify' AND status = 'passed' AND scope = 'full'"
  643. " ORDER BY id DESC LIMIT 1",
  644. (root,),
  645. ).fetchone()
  646. def verification_status(*, session_id: str | None, cwd: str | Path | None) -> dict[str, Any]:
  647. """Return the best known verification state for a session/workspace.
  648. Evidence recorded before the latest edit is reported as ``stale``. A
  649. ``hermes verify`` event for the same ``root`` that is at least as new as
  650. that edit satisfies the guard even if another session recorded it.
  651. """
  652. if not _ledger_enabled():
  653. return {"status": "disabled", "evidence": None}
  654. facts = _project_facts(cwd)
  655. if not facts:
  656. return {"status": "not_applicable", "evidence": None}
  657. sid = str(session_id or "default")
  658. root = _root_for(facts, cwd)
  659. with _DB_LOCK, _transaction() as conn:
  660. state = conn.execute(
  661. "SELECT last_event_id, last_edit_at, changed_paths_json"
  662. " FROM verification_state WHERE session_id = ? AND root = ?",
  663. (sid, root),
  664. ).fetchone()
  665. if state is None:
  666. return {"status": "unverified", "evidence": None, "root": root, "session_id": sid, "changed_paths": []}
  667. event = None
  668. if state["last_event_id"] is not None:
  669. event = conn.execute("SELECT * FROM verification_events WHERE id = ?", (state["last_event_id"],)).fetchone()
  670. last_edit_at = state["last_edit_at"]
  671. stale = bool(event is not None and last_edit_at and last_edit_at > event["created_at"])
  672. if event is None or stale:
  673. shared = _newest_workspace_verify(conn, root=root, since=last_edit_at)
  674. if shared is not None:
  675. event = shared
  676. stale = False
  677. result = {
  678. "evidence": None, "root": root, "session_id": sid,
  679. "changed_paths": _load_changed_paths(state["changed_paths_json"]),
  680. }
  681. if event is None:
  682. return {"status": "unverified", **result}
  683. evidence = dict(event)
  684. return {"status": "stale" if stale else evidence["status"], **result, "evidence": evidence}