skill_ledger.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633
  1. """Per-mutation skill audit ledger + single-edit rollback.
  2. Every skill mutation (any actor) appends one JSONL entry to
  3. ``~/.hermes/skills/.curator_ledger.jsonl`` with before/after file manifests whose
  4. contents are stored content-addressed (sha256-deduped) under
  5. ``~/.hermes/.curator_backups/blobs/``. JSONL, not the state DB: durable, greppable,
  6. survives DB resets. TELEMETRY, NOT A GATE: every public write path swallows and
  7. logs — except ``rollback_entry``, which FAILS CLOSED when its safety capture fails.
  8. """
  9. from __future__ import annotations
  10. import contextvars
  11. import hashlib
  12. from contextlib import suppress
  13. import json
  14. import logging
  15. import os
  16. import re
  17. import tarfile
  18. import time as _time
  19. import uuid
  20. from datetime import datetime, timezone, UTC
  21. from pathlib import Path
  22. from typing import Any, Dict, List, Optional, Tuple
  23. from hermes_constants import get_hermes_home
  24. logger = logging.getLogger(__name__)
  25. # Snapshot-id shape of agent.curator_backup (duplicated to avoid importing the backup stack).
  26. _BACKUP_ID_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z(-\d{2})?$")
  27. # ".archive/<name>-YYYYMMDDHHMMSS" collision suffix added by archive_skill.
  28. _ARCHIVE_TS_SUFFIX_RE = re.compile(r"^(.+)-\d{14}$")
  29. # Rollback of these must restore a COMPLETE package: consolidation may have re-homed
  30. # support files first, so a disk-only capture would restore a hollow skill.
  31. _PACKAGE_RESTORE_ACTIONS = frozenset({"delete", "archive", "purge"})
  32. _VALID_ACTORS = {"curator", "agent", "user"}
  33. _DEFAULT_LEDGER_MAX_BYTES = 5 * 1024 * 1024 # skills.ledger_max_bytes default (5 MB)
  34. _BLOB_GC_GRACE_SECS = 3600 # blobs younger than this may belong to an in-flight capture: never GC them
  35. _NON_PACKAGE_TOPS = {".curator_backups", ".hub", ".archive", ".locks"}
  36. # Transient/regeneratable local artifacts that must never be swept into a
  37. # snapshot, no matter how deep they sit under the skill dir — a stray venv or
  38. # node_modules turns a multi-KB ledger capture into gigabytes of blobs (#107539).
  39. # agent.curator_backup applies the same set to the whole-tree tarball.
  40. TRANSIENT_DIRS = frozenset({
  41. ".venv", "venv", "env", ".env",
  42. "node_modules", "__pycache__",
  43. ".pytest_cache", ".mypy_cache", ".ruff_cache",
  44. ".git",
  45. })
  46. _SNAPSHOT_EXCLUDE_DIRS = TRANSIENT_DIRS
  47. # Explicit actor override: the CLI sets "user", the curator walk sets "curator".
  48. _actor_override: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
  49. "skill_ledger_actor", default=None)
  50. def set_ledger_actor(actor: Optional[str]) -> contextvars.Token:
  51. """Bind an explicit actor for this context; reset_ledger_actor(token) in a finally."""
  52. return _actor_override.set(actor)
  53. def reset_ledger_actor(token: contextvars.Token) -> None:
  54. _actor_override.reset(token)
  55. def derive_actor() -> str:
  56. """Explicit override, else background-review provenance -> curator, else agent."""
  57. override = _actor_override.get()
  58. if override in _VALID_ACTORS:
  59. return override
  60. with suppress(Exception):
  61. from tools.skill_provenance import is_background_review
  62. if is_background_review():
  63. return "curator"
  64. return "agent"
  65. def _skills_dir() -> Path:
  66. return get_hermes_home() / "skills"
  67. def ledger_path() -> Path:
  68. return _skills_dir() / ".curator_ledger.jsonl"
  69. def blobs_dir() -> Path:
  70. return get_hermes_home() / ".curator_backups" / "blobs"
  71. def _ledger_lock():
  72. """Exclusive cross-process lock on the ledger file, held across an append AND the maintenance
  73. sweep, and across the CLI compact/GC path. Without it a concurrent appender's O_APPEND write can
  74. land on the inode ``compact_ledger``/``_trim_oldest`` are about to ``os.replace`` — the row is
  75. silently lost and ``gc_blobs`` then deletes its blobs. Same idiom as ``_skill_mutation_lock``:
  76. ``<skills>/.locks/ledger.lock``, thread-re-entrant, no-op where neither fcntl nor msvcrt exists."""
  77. from tools.skill_usage import skill_file_lock
  78. return skill_file_lock(_skills_dir() / ".locks" / "ledger.lock")
  79. def _skills_cfg(key: str, default):
  80. """``skills.<key>`` from the read-only merged config (no deepcopy), or *default* when the
  81. read fails. Lazy import keeps this module importable without the CLI."""
  82. try:
  83. from hermes_cli.config import cfg_get, load_config_readonly # read-only hot path: no deepcopy
  84. return cfg_get(load_config_readonly(), "skills", key, default=default)
  85. except Exception as e: # pragma: no cover — best-effort config read
  86. logger.debug("skill_ledger: config read failed (%s); skills.%s defaults to %r", e, key, default)
  87. return default
  88. def ledger_enabled() -> bool:
  89. """Config gate ``skills.ledger`` (default True)."""
  90. return bool(_skills_cfg("ledger", True))
  91. def _max_ledger_bytes() -> int:
  92. """Config ``skills.ledger_max_bytes`` (default 5 MB, 0 disables): above it the
  93. next append triggers the maintenance sweep instead of growing the file forever."""
  94. return int(_skills_cfg("ledger_max_bytes", _DEFAULT_LEDGER_MAX_BYTES))
  95. def _rel_posix(path: Path | str, root: Path) -> Optional[str]:
  96. """POSIX path of ``path`` relative to ``root`` (both normalized), or None when outside."""
  97. try:
  98. return Path(os.path.normpath(str(path))).relative_to(os.path.normpath(str(root))).as_posix()
  99. except (ValueError, TypeError):
  100. return None
  101. def _is_within(root: Path, path: Path) -> bool:
  102. """True when *path* (normalized, no symlink resolution) sits under *root*."""
  103. return _rel_posix(path, root) is not None
  104. def _store_blob(data: bytes) -> str:
  105. """Write *data* keyed by sha256 (existing blob left alone). Returns the hash."""
  106. digest = hashlib.sha256(data).hexdigest()
  107. dest = blobs_dir() / digest
  108. if not dest.exists():
  109. dest.parent.mkdir(parents=True, exist_ok=True)
  110. tmp = dest.with_name(f".tmp-{uuid.uuid4().hex[:8]}-{digest}")
  111. tmp.write_bytes(data)
  112. os.replace(tmp, dest)
  113. return digest
  114. def read_blob(sha256: str) -> Optional[bytes]:
  115. """Return blob content or None when missing/invalid."""
  116. if not sha256 or not all(c in "0123456789abcdef" for c in sha256):
  117. return None
  118. with suppress(OSError):
  119. return (blobs_dir() / sha256).read_bytes() if (blobs_dir() / sha256).exists() else None
  120. return None
  121. def snapshot_paths(root: Optional[Path], *, complete_package: bool = False) -> list[dict[str, str]]:
  122. """{path, sha256} for every file under *root*, each stored as a blob; [] when root is
  123. None/missing. Transient local artifacts (venvs, node_modules, caches, .git) are
  124. excluded wherever they appear under *root*. Raises on I/O failure — callers decide
  125. whether that is fatal (rollback safety capture) or swallowed (telemetry).
  126. ``complete_package`` unions in the newest curator tarball's files (disk hashes win)."""
  127. if root is None:
  128. return []
  129. root = Path(root) # gone from disk -> []; the complete_package fill may still recover it
  130. files = ([root] if root.is_file()
  131. else sorted(p for p in root.rglob("*") if p.is_file()
  132. and not any(part in _SNAPSHOT_EXCLUDE_DIRS
  133. for part in p.relative_to(root).parts[:-1]))
  134. if root.is_dir() else [])
  135. out = [{"path": str(f), "sha256": _store_blob(f.read_bytes())} for f in files]
  136. return fill_snapshot_from_curator_backup(root, out) if complete_package else out
  137. def _package_rel(root: Path) -> Optional[str]:
  138. """Relative POSIX path of a skill dir under ``skills/``; None when outside it
  139. or under backup/hub/archive metadata roots (never a package)."""
  140. posix = (_rel_posix(root, _skills_dir()) or "").strip("/")
  141. if not posix or posix.split("/", 1)[0] in _NON_PACKAGE_TOPS:
  142. return None
  143. return posix
  144. def _strip_archive_timestamp(name: str) -> str:
  145. match = _ARCHIVE_TS_SUFFIX_RE.match(name)
  146. return match.group(1) if match else name
  147. def _skill_md_parents(items: Optional[list[dict[str, str]]]) -> list[Path]:
  148. return [p.parent for p in (Path(str(i.get("path", ""))) for i in items or []) if p.name == "SKILL.md"]
  149. def package_prefixes(
  150. root: Optional[Path] = None, skill: Optional[str] = None,
  151. before: Optional[list[dict[str, str]]] = None) -> list[str]:
  152. """Tar member prefixes of this skill's package: live location under ``skills/``,
  153. the package parent from the before-state SKILL.md path (rollback fills where
  154. *root* is gone), the bare skill name, and the name minus an archive suffix."""
  155. candidates = [_package_rel(Path(root)) if root is not None else None]
  156. candidates += [_package_rel(p) for p in _skill_md_parents(before)]
  157. candidates += [skill, _strip_archive_timestamp(skill) if skill else None]
  158. return list(dict.fromkeys(p for p in ((c or "").strip("/") for c in candidates) if p))
  159. def _read_package_files_from_latest_backup(prefixes: list[str]) -> dict[str, bytes]:
  160. """``{posix-relpath: bytes}`` under *prefixes* in the newest ``skills/.curator_backups/*/
  161. skills.tar.gz``; malicious member names (absolute, ``..`` traversal) are rejected."""
  162. backups = _skills_dir() / ".curator_backups"
  163. try:
  164. children = list(backups.iterdir()) if prefixes and backups.is_dir() else []
  165. except OSError:
  166. return {}
  167. candidates = [
  168. child / "skills.tar.gz" for child in children
  169. if child.is_dir() and _BACKUP_ID_RE.match(child.name) and (child / "skills.tar.gz").is_file()]
  170. if not candidates:
  171. return {}
  172. # Parent dirs sort lexicographically == chronologically for the id shape.
  173. archive = max(candidates, key=lambda p: p.parent.name)
  174. prefixed = tuple(p if p.endswith("/") else p + "/" for p in prefixes)
  175. exact = set(prefixes)
  176. out: dict[str, bytes] = {}
  177. try:
  178. with tarfile.open(archive, "r:gz") as tf:
  179. for member in tf.getmembers():
  180. name = member.name.replace("\\", "/").lstrip("./")
  181. if (not member.isfile() or not name or name.startswith("/")
  182. or ".." in Path(name).parts
  183. or (name not in exact and not name.startswith(prefixed))):
  184. continue
  185. extracted = tf.extractfile(member)
  186. if extracted is not None:
  187. out[name] = extracted.read()
  188. except (OSError, tarfile.TarError) as exc:
  189. logger.debug("skill_ledger: could not read curator backup package: %s", exc)
  190. return {}
  191. return out
  192. def fill_snapshot_from_curator_backup(
  193. root: Optional[Path], existing: Optional[list[dict[str, str]]] = None, *,
  194. skill: Optional[str] = None) -> list[dict[str, str]]:
  195. """Union missing skill-package files from the newest curator snapshot. Completeness fill, not
  196. a gate: failures return *existing* unchanged, and only ABSENT paths are filled. Fill targets go
  197. where rollback must restore them: under *root* when known (for purge that is
  198. ``.archive/<name>/``, NOT the live tree), else the live skills dir; the tar's leading
  199. package-dir segment is stripped when *root* already names the package. Every target must stay
  200. under ``skills/`` and HERMES_HOME."""
  201. out = list(existing or [])
  202. prefixes = package_prefixes(root, skill, out)
  203. if not prefixes:
  204. return out
  205. try:
  206. extra = _read_package_files_from_latest_backup(prefixes)
  207. except Exception as exc:
  208. logger.debug("skill_ledger: backup package fill failed: %s", exc)
  209. return out
  210. if not extra:
  211. return out
  212. skills = _skills_dir()
  213. dest_root = Path(root) if root is not None else None
  214. pkg_names = {dest_root.name, _strip_archive_timestamp(dest_root.name)} if dest_root else set()
  215. have = {rel for rel in (_rel_posix(str(i.get("path", "")), skills) for i in out) if rel is not None}
  216. for rel, data in extra.items():
  217. parts = rel.split("/")
  218. if dest_root is not None and parts and parts[0] in pkg_names:
  219. parts = parts[1:]
  220. if not parts:
  221. continue
  222. dest = (dest_root if dest_root is not None else skills).joinpath(*parts)
  223. if not _is_within(skills, dest) or not _is_within(get_hermes_home(), dest):
  224. continue
  225. rel_key = _rel_posix(dest, skills)
  226. if rel_key is None or rel_key in have:
  227. continue
  228. try:
  229. out.append({"path": str(dest), "sha256": _store_blob(data)})
  230. except Exception as exc:
  231. logger.debug("skill_ledger: backup blob store failed for %s: %s", rel, exc)
  232. continue
  233. have.add(rel_key)
  234. return out
  235. def _delta(before: list[dict[str, str]], after: list[dict[str, str]]) -> tuple[list[dict[str, str]], list[dict[str, str]]]:
  236. """Drop paths whose hash is identical on both sides. ``rollback_entry`` writes every *before*
  237. path and removes *after*-only paths, so unchanged files are dead weight there — and a full
  238. package manifest per edit made a 4,000-file skill cost 1.5 MB of ledger per patch (650 MB
  239. over one summer). Deletions/creations are preserved: a path present on one side only stays."""
  240. b_sha = {str(i.get("path")): i.get("sha256") for i in before}
  241. a_sha = {str(i.get("path")): i.get("sha256") for i in after}
  242. same = {p for p, s in b_sha.items() if a_sha.get(p) == s}
  243. return ([i for i in before if str(i.get("path")) not in same],
  244. [i for i in after if str(i.get("path")) not in same])
  245. def append_entry(
  246. action: str, skill: str, before: Optional[list[dict[str, str]]] = None,
  247. after: Optional[list[dict[str, str]]] = None, actor: Optional[str] = None,
  248. evidence: Optional[dict[str, Any]] = None) -> Optional[str]:
  249. """Append one entry -> id, or None when disabled / write failed (never raises)."""
  250. if not ledger_enabled():
  251. return None
  252. try:
  253. # pre-rollback deliberately records before == after (the current state of every touched path).
  254. if action != "pre-rollback":
  255. before, after = _delta(before or [], after or [])
  256. entry = {
  257. "id": uuid.uuid4().hex[:12], "ts": datetime.now(UTC).isoformat(),
  258. "actor": actor if actor in _VALID_ACTORS else derive_actor(),
  259. "action": action, "skill": skill, "evidence": evidence or {},
  260. "before": before or [], "after": after or []}
  261. path = ledger_path()
  262. path.parent.mkdir(parents=True, exist_ok=True)
  263. with _ledger_lock():
  264. with open(path, "a", encoding="utf-8") as fh:
  265. fh.write(json.dumps(entry, ensure_ascii=False) + "\n")
  266. _maintain_size()
  267. return entry["id"]
  268. except Exception as e:
  269. logger.warning("skill_ledger: failed to append entry (%s) — mutation unaffected", e)
  270. return None
  271. def _read_ledger(what: str, *, quiet_missing: bool = False) -> Optional[bytes]:
  272. """Raw ledger bytes, or ``None`` (after a warning) when the file cannot be read or is not UTF-8.
  273. ``quiet_missing`` keeps a merely absent ledger silent — normal for a fresh install."""
  274. try:
  275. raw = ledger_path().read_bytes()
  276. # Windows editors may prepend a UTF-8 BOM. Normalize it away so every
  277. # consumer can decode ordinary UTF-8 without treating the first row as malformed.
  278. return raw.decode("utf-8-sig").encode("utf-8")
  279. except (OSError, UnicodeError) as exc:
  280. if not (quiet_missing and isinstance(exc, FileNotFoundError)):
  281. logger.warning("skill_ledger: ledger unreadable (%s); %s", exc, what)
  282. return None
  283. _TRIM_LOW_WATER = 0.8 # trim target as a fraction of ``skills.ledger_max_bytes``
  284. def _maintain_size() -> None:
  285. """Keep the ledger bounded: once it crosses ``skills.ledger_max_bytes`` run the
  286. delta-dedup rewrite, and if genuinely-divergent entries still exceed the cap,
  287. drop the oldest ones until it fits. Best-effort telemetry, never a gate: any
  288. failure is logged and the just-appended entry stays on disk.
  289. Trimming stops at a LOW-WATER mark (``_TRIM_LOW_WATER`` of the cap), not at the
  290. cap itself: a ledger trimmed to exactly the cap is over it again on the very next
  291. append, so every append would pay compact + trim + blob GC. ``gc_blobs`` only
  292. runs when the trim actually dropped rows — that is the only way a blob can
  293. become orphaned here."""
  294. max_bytes = _max_ledger_bytes()
  295. if max_bytes <= 0:
  296. return
  297. try:
  298. if ledger_path().stat().st_size <= max_bytes:
  299. return
  300. _, _, size_after = compact_ledger()
  301. low_water = int(max_bytes * _TRIM_LOW_WATER)
  302. dropped = _trim_oldest(low_water) if size_after > low_water else 0
  303. if dropped:
  304. gc_blobs()
  305. except Exception as e:
  306. logger.warning(
  307. "skill_ledger: maintenance sweep failed (%s) — ledger left as-is", e
  308. )
  309. def _rewrite_ledger(path: Path, lines: list[bytes], op: str) -> bytes:
  310. """Atomically replace the ledger at *path* with *lines* (one physical row each, no
  311. terminators): write ``<name>.<op>.tmp`` fully, then ``os.replace`` it over the ledger.
  312. Returns the bytes written so callers can report the new size."""
  313. data = b"\n".join(lines) + b"\n" if lines else b""
  314. tmp = path.with_name(path.name + f".{op}.tmp")
  315. tmp.write_bytes(data)
  316. os.replace(tmp, path)
  317. return data
  318. def _trim_oldest(max_bytes: int) -> int:
  319. """Rewrite the ledger without its oldest lines until it is at most *max_bytes*;
  320. the newest entry always survives. Lines in the retained tail are never rewritten
  321. or parsed — malformed lines there survive verbatim; trimming drops the oldest
  322. lines regardless of shape. Returns the number of lines dropped (0 = untouched)."""
  323. with _ledger_lock():
  324. return _trim_oldest_locked(max_bytes)
  325. def _trim_oldest_locked(max_bytes: int) -> int:
  326. path = ledger_path()
  327. raw = _read_ledger("trim skipped")
  328. if raw is None:
  329. return 0
  330. # Split on the physical row terminator only: ``str.splitlines`` would also split on
  331. # U+2028/U+2029/U+0085, which ``ensure_ascii=False`` rows may legitimately contain.
  332. lines = raw.split(b"\n")
  333. if lines and lines[-1] == b"":
  334. lines.pop()
  335. kept: list[bytes] = []
  336. size = 0
  337. for line in reversed(lines): # the first iteration always keeps lines[-1]: the newest entry
  338. addition = len(line) + 1
  339. if kept and size + addition > max_bytes:
  340. break
  341. kept.append(line)
  342. size += addition
  343. kept.reverse()
  344. dropped = len(lines) - len(kept)
  345. if not dropped:
  346. return 0
  347. _rewrite_ledger(path, kept, "trim")
  348. return dropped
  349. def compact_ledger() -> tuple[int, int, int]:
  350. """Rewrite the ledger with every entry's unchanged paths dropped (see ``_delta``); ids, order and
  351. rollback semantics are preserved. Returns ``(entries, bytes_before, bytes_after)``. Atomic: the
  352. new file replaces the old only once fully written. Malformed lines are kept verbatim. Follow with
  353. ``gc_blobs()``: dropped references leave blobs nothing can restore."""
  354. with _ledger_lock():
  355. return _compact_ledger_locked()
  356. def _compact_ledger_locked() -> tuple[int, int, int]:
  357. path = ledger_path()
  358. raw = _read_ledger("compaction skipped")
  359. if raw is None:
  360. return 0, 0, 0
  361. text = raw.decode("utf-8")
  362. out, kept = [], 0
  363. for line in text.splitlines():
  364. if not line.strip():
  365. continue
  366. try:
  367. row = json.loads(line)
  368. except json.JSONDecodeError:
  369. out.append(line)
  370. continue
  371. if isinstance(row, dict) and row.get("action") != "pre-rollback":
  372. row["before"], row["after"] = _delta(row.get("before") or [], row.get("after") or [])
  373. line = json.dumps(row, ensure_ascii=False)
  374. out.append(line)
  375. kept += 1
  376. lines = [line.encode("utf-8") for line in out]
  377. if (b"\n".join(lines) + b"\n" if lines else b"") == raw:
  378. return kept, len(raw), len(raw) # already compact (append-time _delta): no rewrite to pay
  379. data = _rewrite_ledger(path, lines, "compact")
  380. return kept, len(raw), len(data)
  381. def gc_blobs() -> tuple[int, int]:
  382. """Delete blobs no ledger entry references; returns ``(deleted, bytes_freed)``. The store was
  383. write-only: on one install 98.9% of 47k blobs (1.18 GB) were unreachable after a venv walk
  384. (#107539). Malformed ledger lines, or an unreadable/undecodable ledger, abort the sweep
  385. (blobs are kept) — an entry we cannot read may still hold references. Blobs newer than
  386. ``_BLOB_GC_GRACE_SECS`` are kept too: ``snapshot_paths`` stores them before the referencing row.
  387. """
  388. with _ledger_lock():
  389. return _gc_blobs_locked()
  390. def _gc_blobs_locked() -> tuple[int, int]:
  391. blobs = blobs_dir()
  392. if not blobs.is_dir():
  393. return 0, 0
  394. referenced: set = set()
  395. raw = _read_ledger("blob GC skipped")
  396. if raw is None: # an unavailable ledger is not evidence that its blobs are unreferenced
  397. return 0, 0
  398. for line in raw.decode("utf-8").splitlines():
  399. if not line.strip():
  400. continue
  401. try:
  402. row = json.loads(line)
  403. except json.JSONDecodeError:
  404. row = None
  405. if not isinstance(row, dict):
  406. logger.warning("skill_ledger: malformed ledger line; blob GC skipped")
  407. return 0, 0
  408. for item in (row.get("before") or []) + (row.get("after") or []):
  409. referenced.add(str(item.get("sha256", "")))
  410. deleted = freed = 0
  411. fresh_after = _time.time() - _BLOB_GC_GRACE_SECS
  412. for blob in blobs.iterdir():
  413. if blob.is_file() and blob.name not in referenced:
  414. try:
  415. st = blob.stat()
  416. if st.st_mtime > fresh_after: # in-flight capture (incl. .tmp-*): row not appended yet
  417. continue
  418. size = st.st_size
  419. blob.unlink()
  420. except OSError:
  421. continue
  422. deleted += 1
  423. freed += size
  424. return deleted, freed
  425. def record_mutation(
  426. action: str, skill: str, before_root: Optional[Path] = None,
  427. before: Optional[list[dict[str, str]]] = None, after_root: Optional[Path] = None,
  428. actor: Optional[str] = None, evidence: Optional[dict[str, Any]] = None) -> Optional[str]:
  429. """Mutation hook: after-state from *after_root* (before = pre-captured list or captured from
  430. *before_root*), then append. NEVER raises. delete/archive/purge capture a COMPLETE package
  431. (filled from the newest curator backup) so rollback never restores a shell."""
  432. if not ledger_enabled():
  433. return None
  434. try:
  435. _complete = action in _PACKAGE_RESTORE_ACTIONS
  436. if before is None:
  437. before = snapshot_paths(before_root, complete_package=_complete)
  438. elif _complete:
  439. before = fill_snapshot_from_curator_backup(before_root, before, skill=skill)
  440. return append_entry(action, skill, before=before, after=snapshot_paths(after_root),
  441. actor=actor, evidence=evidence)
  442. except Exception as e:
  443. logger.warning("skill_ledger: record_mutation failed (%s) — mutation unaffected", e)
  444. return None
  445. def capture_before(
  446. root: Optional[Path], *, complete_package: bool = False, skill: Optional[str] = None,
  447. ) -> Optional[list[dict[str, str]]]:
  448. """Best-effort pre-mutation capture; None on failure/disabled (pass straight to
  449. record_mutation). ``complete_package=True`` for delete/archive/purge."""
  450. if not ledger_enabled():
  451. return None
  452. try:
  453. captured = snapshot_paths(root)
  454. return fill_snapshot_from_curator_backup(root, captured, skill=skill) if complete_package else captured
  455. except Exception as e:
  456. logger.warning("skill_ledger: before-capture failed (%s) — mutation unaffected", e)
  457. return None
  458. def list_entries(skill: Optional[str] = None, limit: Optional[int] = None) -> list[dict[str, Any]]:
  459. """Read the ledger, newest first. Malformed lines are skipped."""
  460. raw = _read_ledger("listing empty", quiet_missing=True) # missing/unreadable/undecodable == empty
  461. if raw is None:
  462. return []
  463. rows: list[dict[str, Any]] = []
  464. for line in raw.decode("utf-8").splitlines():
  465. with suppress(json.JSONDecodeError):
  466. row = json.loads(line) if line.strip() else None
  467. if isinstance(row, dict) and (not skill or row.get("skill") == skill):
  468. rows.append(row)
  469. rows.reverse()
  470. return rows[:limit] if limit is not None and limit >= 0 else rows
  471. def get_entry(entry_id: str) -> Optional[dict[str, Any]]:
  472. return next((r for r in list_entries() if r.get("id") == entry_id), None) if entry_id else None
  473. def _validate_entry_paths(entry: dict[str, Any]) -> Optional[str]:
  474. """Every entry path must be under HERMES_HOME — a hand-edited ledger must not
  475. become a write-anywhere primitive."""
  476. home = get_hermes_home()
  477. for section in ("before", "after"):
  478. for item in entry.get(section) or []:
  479. p = Path(str(item.get("path", "")))
  480. if not _is_within(home, p):
  481. return f"entry references a path outside {home}: {p}"
  482. return None
  483. def rollback_entry(entry_id: str) -> tuple[bool, str]:
  484. """Restore the before-state of mutation *entry_id*. Fail-closed (mirrors
  485. agent/curator_backup.rollback): every before-blob must exist BEFORE any change, and a
  486. pre-rollback safety entry of every touched path's CURRENT state is appended first.
  487. 1. 2. See #63366.
  488. """
  489. entry = get_entry(entry_id)
  490. if entry is None:
  491. return False, f"no ledger entry with id '{entry_id}'"
  492. if path_err := _validate_entry_paths(entry):
  493. return False, f"refusing rollback: {path_err}"
  494. before = list(entry.get("before") or [])
  495. after = list(entry.get("after") or [])
  496. # Historical hollow delete/archive/purge entries (SKILL.md only): fill from the
  497. # newest curator backup so the complete package is restored. Entry hashes win;
  498. # only missing paths are added, and the filled set is re-validated.
  499. if entry.get("action") in _PACKAGE_RESTORE_ACTIONS:
  500. before = fill_snapshot_from_curator_backup(
  501. next(iter(_skill_md_parents(before)), None), before,
  502. skill=str(entry.get("skill") or "") or None)
  503. if path_err := _validate_entry_paths({**entry, "before": before, "after": after}):
  504. return False, f"refusing rollback: {path_err}"
  505. # Pre-check every blob we need so we never fail mid-restore.
  506. for item in before:
  507. if read_blob(str(item.get("sha256", ""))) is None:
  508. return False, (f"missing blob {item.get('sha256')} for {item.get('path')}; "
  509. "rollback aborted, nothing was changed")
  510. # Safety entry: CURRENT state of every touched path, so the rollback itself is undoable.
  511. touched = {str(i["path"]) for i in before + after if i.get("path")}
  512. try:
  513. safety_before = [{"path": p, "sha256": _store_blob(Path(p).read_bytes())}
  514. for p in sorted(touched) if Path(p).is_file()]
  515. safety_id = append_entry(
  516. "pre-rollback", entry.get("skill", "?"), before=safety_before, after=safety_before,
  517. evidence={"rollback_target": entry_id})
  518. except Exception as e:
  519. return False, (f"pre-rollback safety capture failed ({e}); rollback aborted and "
  520. "current skills were not changed")
  521. if safety_id is None:
  522. return False, ("pre-rollback safety capture failed (ledger disabled or "
  523. "unwritable); rollback aborted and current skills were not changed")
  524. # Restore: write every before-file, remove files the mutation created.
  525. before_paths = {str(i["path"]) for i in before}
  526. for item in before:
  527. fp = Path(str(item["path"]))
  528. fp.parent.mkdir(parents=True, exist_ok=True)
  529. fp.write_bytes(read_blob(str(item["sha256"]))) # pre-checked above
  530. restored, removed = len(before), 0
  531. for item in after:
  532. p = str(item.get("path", ""))
  533. if p and p not in before_paths:
  534. try:
  535. if Path(p).is_file():
  536. Path(p).unlink()
  537. removed += 1
  538. except OSError as e:
  539. logger.warning("skill_ledger: could not remove %s during rollback: %s", p, e)
  540. append_entry(
  541. "rollback", entry.get("skill", "?"), before=safety_before, after=before,
  542. evidence={"rollback_target": entry_id, "restored": restored, "removed": removed})
  543. return True, (
  544. f"rolled back entry {entry_id} ({entry.get('action')} on "
  545. f"'{entry.get('skill')}'): {restored} file(s) restored, {removed} removed. "
  546. f"Safety entry {safety_id} captured the pre-rollback state.")