skill_manager_batch.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308
  1. """Atomic multi-op batch path for ``skill_manage``. Origin state
  2. (``skill_manage``/``_find_skill``/``_skill_gate_bypass``) is reached lazily
  3. through ``tools.skill_manager_tool`` so that module owns it."""
  4. from contextlib import suppress
  5. import json
  6. import logging
  7. import posixpath
  8. import shutil
  9. import tempfile
  10. from pathlib import Path
  11. logger = logging.getLogger("tools.skill_manager_tool")
  12. _BATCH_OP_ACTIONS = {"create", "patch", "write_file", "remove_file"}
  13. _BATCH_MAX_OPS = 20
  14. # --- Per-op argument shape (checked before any effect) ---------------------------------
  15. # action -> (arg, is_missing, error) checks run before the handler.
  16. _MISSING, _IS_NONE = (lambda v: not v), (lambda v: v is None)
  17. _REQUIRED_ARGS = {
  18. "create": [("content", _MISSING,
  19. "content is required for 'create'. Provide the full SKILL.md text (frontmatter + body).")],
  20. "edit": [("content", _MISSING,
  21. "content is required for a full rewrite. Provide the full updated SKILL.md text.")],
  22. "write_file": [
  23. ("file_path", _MISSING, "file_path is required for 'write_file'. Example: 'references/api-guide.md'"),
  24. ("file_content", _IS_NONE, "file_content is required for 'write_file'.")],
  25. "remove_file": [("file_path", _MISSING, "file_path is required for 'remove_file'.")]}
  26. # A bare "required" error is a dead end: the model retries blindly and often escapes to
  27. # action='write_file', clobbering the whole file.
  28. _PATCH_NEEDS_OLD_STRING = (
  29. "old_string is required for 'patch' and must be the EXACT text currently in the file. "
  30. "Read the target file first (read_file on the skill's SKILL.md, or the file named by "
  31. "file_path) and copy the snippet verbatim, then retry 'patch'. Do NOT fall back to "
  32. "action='write_file' — that rewrites the entire file and destroys unrelated content.")
  33. _PATCH_NEEDS_NEW_STRING = "new_string is required for 'patch'. Use an empty string to delete matched text."
  34. _PATCH_EITHER_OR = ("Pass EITHER content (full SKILL.md rewrite) OR old_string/new_string "
  35. "(targeted replacement), not both.")
  36. # Text-slot keys a model confuses: key -> the action that reads it. A 27B model that just
  37. # used write_file's file_content re-emits it on create/patch and then replays the identical
  38. # payload when the error only says the right key is "required" — the hint has to name where
  39. # the text actually landed so the retry can move it.
  40. _TEXT_SLOT_OWNER = {"content": "create (and a full-rewrite patch)",
  41. "new_string": "a targeted patch (with old_string)",
  42. "file_content": "write_file"}
  43. # action -> (text slots it reads, where misfiled text belongs)
  44. _TEXT_SLOT_FOR = {
  45. "create": (("content",), "'content'"),
  46. "edit": (("content",), "'content'"),
  47. "patch": (("content", "new_string"), "old_string/new_string (targeted) or 'content' (full rewrite, last resort)"),
  48. "write_file": (("file_content",), "'file_content'")}
  49. def _misplaced_text_hint(action: str, args: dict) -> str:
  50. """Sentence naming the text-slot key(s) this op carries that ``action`` never reads, or ''."""
  51. if action not in _TEXT_SLOT_FOR:
  52. return "" # delete/remove_file/unknown: no text slot, so no destination to point at
  53. reads, destination = _TEXT_SLOT_FOR[action]
  54. stray = [k for k in _TEXT_SLOT_OWNER if k not in reads and args.get(k) is not None]
  55. if not stray:
  56. return ""
  57. carried = " and ".join(f"'{k}' (that key is for {_TEXT_SLOT_OWNER[k]})" for k in stray)
  58. return f" Note: this op carries {carried} — move that text to {destination}."
  59. def _op_shape_error(action: str, args: dict):
  60. """Argument-shape error text for one op, or None. Only shape misses carry the misplaced-text
  61. hint: a patch whose real problem is an unmatched old_string must not be steered to a full
  62. rewrite. Pure function so the batch can reject a misfiled op BEFORE any sibling is applied."""
  63. for arg, missing, message in _REQUIRED_ARGS.get(action, ()):
  64. if missing(args.get(arg)):
  65. return message + _misplaced_text_hint(action, args)
  66. if action == "patch":
  67. # Every patch shape miss is decided here, not in the handler, so a batch never applies
  68. # op[0] only to roll it back over op[1]'s missing new_string or content+old_string mix.
  69. if args.get("content") and (args.get("old_string") or args.get("new_string") is not None):
  70. return _PATCH_EITHER_OR
  71. if not args.get("old_string") and not args.get("content"):
  72. return _PATCH_NEEDS_OLD_STRING + _misplaced_text_hint(action, args)
  73. if not args.get("content") and args.get("new_string") is None:
  74. return _PATCH_NEEDS_NEW_STRING
  75. return None
  76. def _validate_batch_ops(operations, default_name, tool_error):
  77. """Shape checks with no side effects. Returns (names, None) or (None, error_json)."""
  78. from tools.skill_manager_guards import _background_review_preflight
  79. from tools.skill_manager_tool import _validate_category
  80. def fail(i, msg):
  81. return None, tool_error(f"operations[{i}]{msg}", success=False)
  82. names = []
  83. for i, op in enumerate(operations):
  84. if not isinstance(op, dict) or not op.get("action"):
  85. return fail(i, " needs an 'action'.")
  86. act = op["action"]
  87. if act not in _BATCH_OP_ACTIONS:
  88. return fail(i, f": unknown action '{act}'. Batchable: "
  89. f"{', '.join(sorted(_BATCH_OP_ACTIONS))}; delete must be sole.")
  90. nm = op.get("name") or default_name
  91. if not nm:
  92. return fail(i, " needs a 'name' (the skill it targets).")
  93. # Reject a misfiled op here, before any sibling is applied: a runtime failure on
  94. # op[1] would first apply op[0] and then roll the whole batch back.
  95. if (shape_err := _op_shape_error(act, op)) is not None:
  96. return fail(i, f" ({act} on '{nm}'): {shape_err}")
  97. # create's category is resolved to a target dir before the snapshot: reject a bad one here
  98. # so it returns a JSON error (not a TypeError) and never leaks the snapshot tempdir.
  99. if act == "create" and (cat_err := _validate_category(op.get("category"))) is not None:
  100. return fail(i, f" ({act} on '{nm}'): {cat_err}")
  101. names.append(nm)
  102. if act == "create" and nm in names[:-1]:
  103. return fail(i, f": create for '{nm}' must precede that skill's other ops.")
  104. if (preflight := _background_review_preflight(act, nm)) is not None:
  105. return None, json.dumps(preflight, ensure_ascii=False)
  106. # Clobber guard: a DESTRUCTIVE op (create/write_file/remove_file/full rewrite) on
  107. # a file an earlier op touched would SILENTLY discard its work — reject it.
  108. # Additive patches are always legal. Paths are normalized against spelling variants.
  109. touched_files = set()
  110. for i, op in enumerate(operations):
  111. act, nm = op["action"], names[i]
  112. # create and full-rewrite patch (content) always hit SKILL.md.
  113. full_rewrite = act == "patch" and bool(op.get("content"))
  114. fp = (op.get("file_path") or "").strip()
  115. target = ("SKILL.md" if (act == "create" or full_rewrite or not fp)
  116. else posixpath.normpath(fp.lstrip("/")))
  117. key = (nm, target)
  118. if (act in ("create", "write_file", "remove_file") or full_rewrite) and key in touched_files:
  119. return fail(i, f": {act} on '{target}' of skill '{nm}' — an earlier op in this "
  120. f"batch already touched that file, and this op would silently discard its work. "
  121. f"One destructive op (write_file/remove_file/full rewrite) per file per batch; put "
  122. f"it first, or fold the change in. Patch chains are fine.")
  123. touched_files.add(key)
  124. return names, None
  125. def _snapshot_skills(names, snap_root, find_skill, create_targets):
  126. """Copy every touched skill aside. Returns (snapshots, None) or (None, error_text).
  127. ``create_targets`` maps a name with no skill yet to the dir its ``create`` op will use.
  128. An EMPTY pre-existing dir there has no SKILL.md to snapshot, yet create adopts it (see
  129. ``_create_skill``): record that it pre-dated the batch so rollback never rmtree()s it."""
  130. snapshots = {} # skill name -> (pre_dir or None, snapshot_dir or None, dir_pre_existed)
  131. for nm in dict.fromkeys(names): # ordered unique
  132. pre = find_skill(nm)
  133. pre_dir = Path(pre["path"]) if pre else None
  134. snap = snap_root / nm if pre_dir is not None and pre_dir.is_dir() else None
  135. if snap is not None:
  136. try:
  137. shutil.copytree(pre_dir, snap)
  138. except Exception as exc:
  139. return None, f"Could not snapshot '{nm}' for atomic batch: {exc}"
  140. target = create_targets.get(nm) if pre is None else None
  141. snapshots[nm] = (pre_dir, snap, target is not None and target.is_dir())
  142. return snapshots, None
  143. def _restore_snapshot(pre_dir, snap, post_dir, dir_pre_existed=False, written=()) -> None:
  144. post_exists = post_dir is not None and post_dir.is_dir()
  145. if snap is None:
  146. if not post_exists:
  147. return
  148. if not dir_pre_existed: # Batch created this skill: remove the partial result.
  149. shutil.rmtree(post_dir)
  150. return
  151. # The dir predates the batch (adopted empty leftover): unlink exactly the files the
  152. # batch wrote there, then rmdir() the now-empty dirs up to and including the skill dir.
  153. # rmdir() fails on anything left, so a file that landed out-of-band survives.
  154. for rel in ("SKILL.md", *written):
  155. target = post_dir / rel
  156. with suppress(OSError):
  157. target.unlink()
  158. for parent in target.parents:
  159. if parent == post_dir or not parent.is_relative_to(post_dir):
  160. break
  161. with suppress(OSError):
  162. parent.rmdir()
  163. with suppress(OSError):
  164. post_dir.rmdir()
  165. return
  166. if not post_exists:
  167. shutil.copytree(snap, pre_dir)
  168. return
  169. # Move the broken state aside and delete it only after the snapshot is
  170. # back, so a failed copytree (disk full, locked file) can't mean total loss.
  171. aside = post_dir.with_name(post_dir.name + ".rollback-broken")
  172. shutil.rmtree(aside, ignore_errors=True)
  173. post_dir.rename(aside)
  174. try:
  175. shutil.copytree(snap, pre_dir)
  176. except Exception:
  177. # Restore failed: put the half-applied state back rather than nothing.
  178. shutil.rmtree(pre_dir, ignore_errors=True)
  179. aside.rename(pre_dir)
  180. raise
  181. shutil.rmtree(aside, ignore_errors=True)
  182. def _rollback(snapshots, find_skill, results):
  183. """Restore every snapshot. ``results`` are the ops applied so far (their name/file_path
  184. tell an adopted dir's rollback which files were the batch's). Returns (note, failed)."""
  185. notes = []
  186. for nm, (pre_dir, snap, dir_pre_existed) in snapshots.items():
  187. written = [posixpath.normpath(r["file_path"].lstrip("/")) for r in results
  188. if r["name"] == nm and r["action"] == "write_file" and r["file_path"]]
  189. try:
  190. post = find_skill(nm)
  191. _restore_snapshot(pre_dir, snap, Path(post["path"]) if post else None,
  192. dir_pre_existed, written)
  193. except Exception as exc:
  194. notes.append(f"ROLLBACK FAILED for '{nm}' ({exc})"
  195. + (f"; snapshot preserved at '{snap}'" if snap is not None else ""))
  196. return ("; ".join(notes) if notes else "all touched skills rolled back"), bool(notes)
  197. _ADVISORY_KEYS = ("lint_warnings", "lint_hint")
  198. def _skill_manage_batch(operations, default_name: str | None = None, task_id: str | None = None,
  199. session_id: str | None = None) -> str:
  200. """Apply operations atomically: every touched skill is snapshotted first and any
  201. failure rolls ALL of them back (batch-created skills are removed). ``delete`` is
  202. only legal as the SOLE op (its recoverable-archive path doesn't compose with
  203. rollback) and routes to the single-op handler. ``default_name`` is the legacy
  204. top-level ``name`` fallback (staged replay)."""
  205. from tools import skill_manager_tool as _smt
  206. from tools.registry import tool_error
  207. if not isinstance(operations, list) or not operations:
  208. return tool_error("operations must be a non-empty array.", success=False)
  209. if len(operations) > _BATCH_MAX_OPS:
  210. return tool_error(f"operations is capped at {_BATCH_MAX_OPS} ops per call.", success=False)
  211. if any(isinstance(op, dict) and op.get("action") == "delete" for op in operations):
  212. if len(operations) != 1:
  213. return tool_error("delete must be the SOLE op in its call — it doesn't "
  214. "compose with other ops' rollback.", success=False)
  215. nm = operations[0].get("name") or default_name
  216. if not nm:
  217. return tool_error("operations[0] (delete) needs a 'name'.", success=False)
  218. return _smt.skill_manage(action="delete", name=nm, task_id=task_id, session_id=session_id,
  219. absorbed_into=operations[0].get("absorbed_into"))
  220. names, err = _validate_batch_ops(operations, default_name, tool_error)
  221. if err is not None:
  222. return err
  223. if not _smt._skill_gate_bypass.get():
  224. # Approval gate for the WHOLE batch as one pending write.
  225. def _staging(wa):
  226. acts = ", ".join(op["action"] for op in operations)
  227. gist = f"batch({len(operations)} ops: {acts}) on {', '.join(sorted(set(names)))}"
  228. return {"action": "batch", "operations": operations}, gist
  229. staged = _smt._run_write_gate(_staging)
  230. if staged is not None:
  231. return staged
  232. # Every target's lock is held from the snapshot through commit or rollback; the per-op
  233. # skill_manage() calls re-enter them. Without the outer fence a concurrent writer landing
  234. # between the snapshot and a rollback would be silently reverted.
  235. with _smt._skill_mutation_locks(names):
  236. snap_root = Path(tempfile.mkdtemp(prefix="skill_batch_"))
  237. create_targets = {names[i]: _smt._resolve_skill_dir(names[i], op.get("category"))
  238. for i, op in enumerate(operations) if op.get("action") == "create"}
  239. snapshots, snap_err = _snapshot_skills(names, snap_root, _smt._find_skill, create_targets)
  240. if snap_err is not None:
  241. shutil.rmtree(snap_root, ignore_errors=True)
  242. return tool_error(snap_err, success=False)
  243. # Single-op path with the gate bypassed (the batch already cleared/staged it).
  244. results = []
  245. rollback_failed = False
  246. token = _smt._skill_gate_bypass.set(True)
  247. try:
  248. for i, op in enumerate(operations):
  249. raw = _smt._skill_manage_from({**op, "name": names[i], "operations": None},
  250. task_id=task_id, session_id=session_id)
  251. try:
  252. parsed = json.loads(raw)
  253. except Exception:
  254. parsed = {"success": False, "error": "unparseable op result"}
  255. if not parsed.get("success"):
  256. note, rollback_failed = _rollback(snapshots, _smt._find_skill, results)
  257. fail = { # key order is wire-visible
  258. "success": False,
  259. "error": (f"operations[{i}] ({op['action']} on '{names[i]}') failed: "
  260. f"{parsed.get('error', 'unknown error')} — batch aborted, {note}."),
  261. "failed_index": i, "completed_before_failure": i}
  262. # Carry the failing op's teaching payload (patch's file_preview /
  263. # fuzzy-match hints) through — without it the model recovers blind.
  264. for k, v in parsed.items():
  265. if k not in ("success", "error") and v is not None:
  266. fail.setdefault(k, v)
  267. return json.dumps(fail, ensure_ascii=False)
  268. entry = {"name": names[i], "action": op["action"],
  269. "file_path": op.get("file_path"), "success": True}
  270. # Advisory payloads (linter findings, org-sharing note) ride on the op result; the
  271. # compact success row otherwise hides them and the model never sees a finding.
  272. entry.update({k: parsed[k] for k in _ADVISORY_KEYS if parsed.get(k) is not None})
  273. results.append(entry)
  274. finally:
  275. _smt._skill_gate_bypass.reset(token)
  276. if rollback_failed:
  277. # Keep the snapshots so the operator can still recover by hand.
  278. logger.warning("skill_manage batch rollback failed, snapshots kept at %s", snap_root)
  279. else:
  280. shutil.rmtree(snap_root, ignore_errors=True)
  281. # utf-8-sig + errors="replace": SKILL.md files are user-authored and sometimes carry a Notepad BOM or
  282. # stray non-UTF-8 bytes. Pinning UTF-8 with replacement keeps skill_view deterministic across platforms
  283. # — falling back to the machine locale (cp1252/GBK) would make the same skill render differently per
  284. # host (see PR #51701).
  285. return json.dumps(
  286. {"success": True, "operations_applied": len(results), "results": results},
  287. ensure_ascii=False)