skill_linter.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215
  1. """Structural + convention linter for SKILL.md files.
  2. The hard validator (``skill_manager_tool._validate_frontmatter``) blocks the
  3. non-negotiables; this is the advisory companion encoding the CONTRIBUTING.md
  4. "Skill authoring standards" a human reviewer would otherwise catch. Findings
  5. never block by themselves — ``lint_skill`` returns ``LintFinding`` rows and the
  6. caller decides. Frontmatter parsing is delegated to ``agent.skill_utils`` so
  7. BOM handling and the prompt description budget stay in one place.
  8. """
  9. from __future__ import annotations
  10. import re
  11. from dataclasses import dataclass
  12. from pathlib import Path
  13. from typing import Any, Dict, Iterator, List, Optional
  14. from agent.skill_utils import SKILL_PROMPT_DESC_LIMIT, parse_frontmatter
  15. # Shell utilities already wrapped as native tools; naming them in prose steers
  16. # the model to a raw shell call. banned token -> native tool to name instead.
  17. _SHELL_UTIL_TO_TOOL: dict[str, str] = {
  18. "grep": "search_files", "rg": "search_files", "cat": "read_file", "head": "read_file",
  19. "tail": "read_file", "sed": "patch", "awk": "patch",
  20. "find": "search_files (target='files')", "ls": "search_files (target='files')"}
  21. _MARKETING_WORDS = (
  22. "powerful", "comprehensive", "seamless", "advanced", "cutting-edge", "state-of-the-art",
  23. "revolutionary", "robust")
  24. # POSIX-only primitives that require ``platforms:`` when a bundled script uses
  25. # them. Detected in scripts/, not in prose.
  26. _POSIX_PRIMITIVES = (
  27. "fcntl",
  28. "termios",
  29. "os.setsid", # windows-footgun: ok (search-pattern string, not a call)
  30. "signal.SIGKILL", # windows-footgun: ok (search-pattern string, not a call)
  31. "osascript",
  32. "/proc/",
  33. "apt-get",
  34. "systemctl")
  35. # Scaffolding files a skill should not ship (noise, not skill content).
  36. _FORBIDDEN_FILES = ("README.md", "CHANGELOG.md", "install.sh", ".env", ".env.example", ".gitignore")
  37. # Presence of the load-bearing section is checked, not exact ordering, so the
  38. # linter is not a change-detector.
  39. _EXPECTED_SECTIONS = ("When to Use", "When to use")
  40. # incident-log-shape: at least this many PR/issue refs AND this density (per 1k chars of prose).
  41. _INCIDENT_REF_MIN = 4
  42. _INCIDENT_REF_PER_KCHAR = 0.5 # the 100k incident-log SKILL.md this targets sat at ~0.7
  43. # references-sprawl: a skill carrying more reference files than this is hoarding per-session notes.
  44. # Calibration: a deliberately curated large workflow skill sits near 50 topical files; the hoarding
  45. # shape this catches was 443 one-per-session files.
  46. _MAX_REFERENCE_FILES = 60
  47. # oversized-body: SKILL.md is loaded whole by skill_view and then rides in context for every later
  48. # call of the session, so body size is paid per turn, not once. The authoring standard is ~200 lines;
  49. # this budget is ~3x that (bundled skills average ~20k chars). The hard cap in skill_manager_tool
  50. # (100k) is a safety stop, not a target — agent-authored skills grew to sit right under it.
  51. _BODY_SOFT_BUDGET_CHARS = 24_000
  52. ERROR = "error"
  53. WARNING = "warning"
  54. @dataclass
  55. class LintFinding:
  56. """A single lint result. ``severity`` is advisory metadata for the caller."""
  57. severity: str # ERROR | WARNING
  58. rule: str
  59. message: str
  60. def _err(rule: str, message: str) -> LintFinding:
  61. return LintFinding(ERROR, rule, message)
  62. def _warn(rule: str, message: str) -> LintFinding:
  63. return LintFinding(WARNING, rule, message)
  64. def _strip_code_blocks(body: str) -> str:
  65. """Remove fenced code blocks so prose-only checks don't fire on examples."""
  66. return re.sub(r"```.*?```", "", body, flags=re.DOTALL)
  67. def _check_frontmatter(frontmatter: dict[str, Any], skill_dir: Optional[Path]) -> Iterator[LintFinding]:
  68. name = str(frontmatter.get("name", "")).strip()
  69. if name and not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", name):
  70. yield _err("name-format", f"name '{name}' must be lowercase letters, digits, hyphens, "
  71. f"and underscores only.")
  72. if skill_dir is not None and name and name != skill_dir.name:
  73. yield _err("name-dir-mismatch", f"frontmatter name '{name}' does not match directory "
  74. f"'{skill_dir.name}'; they must be identical.")
  75. # Measure the raw authored value: extract_skill_description() already
  76. # truncates to the prompt budget, so it can never exceed the limit.
  77. desc = str(frontmatter.get("description", "")).strip().strip("'\"")
  78. if len(desc) > SKILL_PROMPT_DESC_LIMIT:
  79. yield _warn("description-length",
  80. f"description is {len(desc)} chars; the skill index shows "
  81. f"{SKILL_PROMPT_DESC_LIMIT} and clips the rest, losing routing "
  82. f"signal. Keep it to one sentence.")
  83. hits = [w for w in _MARKETING_WORDS if re.search(rf"\b{re.escape(w)}\b", desc.lower())]
  84. if hits:
  85. yield _warn("description-marketing",
  86. f"description contains marketing words {hits}; state the capability, not adjectives.")
  87. for key in ("version", "author", "license"):
  88. if key not in frontmatter:
  89. yield _warn("missing-metadata", f"frontmatter is missing '{key}'; every peer skill has it.")
  90. meta = frontmatter.get("metadata")
  91. hermes_meta = meta.get("hermes") if isinstance(meta, dict) else None
  92. if not isinstance(hermes_meta, dict):
  93. yield _warn("missing-metadata", "frontmatter is missing metadata.hermes.{tags, related_skills}.")
  94. elif "tags" not in hermes_meta:
  95. yield _warn("missing-metadata", "metadata.hermes.tags is missing.")
  96. author = str(frontmatter.get("author", ""))
  97. if author and author.strip().lower() in ("hermes", "agent", "hermes agent") and (
  98. author != "Hermes Agent"):
  99. yield _warn("author-caps", f"author '{author}' should be 'Hermes Agent' (proper caps) "
  100. f"or a real contributor name.")
  101. platforms = frontmatter.get("platforms")
  102. if platforms:
  103. valid = {"linux", "macos", "windows", "darwin"}
  104. items = platforms if isinstance(platforms, list) else [platforms]
  105. bad = [p for p in items if str(p).lower() not in valid]
  106. if bad:
  107. yield _warn("platforms-value", f"platforms contains unrecognized value(s) {bad}; "
  108. f"expected a subset of {sorted(valid)}.")
  109. def _check_body(body: str, skill_dir: Optional[Path]) -> Iterator[LintFinding]:
  110. if len(body) > _BODY_SOFT_BUDGET_CHARS:
  111. yield _warn("oversized-body",
  112. f"SKILL.md body is {len(body):,} chars (~{len(body) // 4:,} tokens); skill_view loads "
  113. f"all of it and it stays in context for every later call of the session. Keep the "
  114. f"always-on rules here (~200 lines) and move topic depth into references/<topic>.md, "
  115. f"linked from the body.")
  116. # Only backtick-wrapped mentions in PROSE (not fenced code): bare words are too noisy.
  117. prose = _strip_code_blocks(body)
  118. for util, tool in _SHELL_UTIL_TO_TOOL.items():
  119. if re.search(rf"`{re.escape(util)}`", prose):
  120. yield _warn("shell-utility-reference",
  121. f"prose references `{util}`; name the native tool `{tool}` instead.")
  122. if not any(re.search(rf"^#+\s+{re.escape(s)}", body, re.MULTILINE) for s in _EXPECTED_SECTIONS):
  123. yield _warn("missing-section", "no '## When to Use' section found; skills need explicit "
  124. "trigger conditions near the top.")
  125. # Incident-log shape: a skill body dense in PR/issue numbers is narrating history instead of
  126. # stating rules. Threshold is per 1k chars so a long body with one citation is fine.
  127. refs = len(re.findall(r"(?<![\w/])#\d{3,6}\b|\b(?:PR|issue)\s*#?\d{3,6}\b", prose))
  128. if refs >= _INCIDENT_REF_MIN and refs / max(len(prose), 1) * 1000 >= _INCIDENT_REF_PER_KCHAR:
  129. yield _warn("incident-log-shape", f"{refs} PR/issue references in prose; write the generalizable "
  130. "rule + why and drop the incident numbers — the rule must stand without the story.")
  131. if skill_dir is None:
  132. return
  133. # Dangling links. Only references/, templates/, assets/ are reliably skill-owned;
  134. # `scripts/` is excluded because dev skills legitimately cite repo-root scripts.
  135. seen: set[str] = set()
  136. for match in re.finditer(r"(references|templates|assets)/[\w./-]+", body):
  137. rel = match.group(0)
  138. if rel in seen or "*" in rel or rel.endswith("/"): # dupes, placeholders, globs
  139. continue
  140. seen.add(rel)
  141. if not (skill_dir / rel).exists():
  142. yield _warn("dangling-reference", f"body references '{rel}' but that file "
  143. f"does not exist in the skill directory.")
  144. def _check_files(frontmatter: dict[str, Any], skill_dir: Path) -> Iterator[LintFinding]:
  145. # Bundled scripts using POSIX-only primitives require a platforms: declaration.
  146. scripts_dir = skill_dir / "scripts"
  147. offenders: dict[str, list[str]] = {}
  148. if not frontmatter.get("platforms") and scripts_dir.is_dir():
  149. for script in scripts_dir.rglob("*"):
  150. if not script.is_file() or script.suffix not in (".py", ".sh", ".bash"):
  151. continue
  152. try:
  153. text = script.read_text(encoding="utf-8-sig", errors="ignore")
  154. except OSError:
  155. continue
  156. hit = [p for p in _POSIX_PRIMITIVES if p in text]
  157. if hit:
  158. offenders[script.name] = hit
  159. if offenders:
  160. detail = "; ".join(f"{k}: {v}" for k, v in offenders.items())
  161. yield _warn("platforms-gating",
  162. f"scripts use POSIX-only primitives ({detail}) but no 'platforms:' frontmatter is "
  163. f"declared. Fix cross-platform or gate with platforms: [linux, macos].")
  164. for fname in _FORBIDDEN_FILES:
  165. if (skill_dir / fname).exists():
  166. yield _warn("forbidden-file",
  167. f"skill ships '{fname}'; skills should not include scaffolding/config files.")
  168. refs_dir = skill_dir / "references"
  169. if refs_dir.is_dir():
  170. n_refs = sum(1 for p in refs_dir.rglob("*.md") if not any(part.startswith("_") for part in p.parts))
  171. if n_refs > _MAX_REFERENCE_FILES:
  172. yield _warn("references-sprawl",
  173. f"{n_refs} files under references/; that is a per-session log, not topical depth. "
  174. "Merge same-topic files into one rule set and drop incident narration.")
  175. def lint_content(content: str, *, skill_dir: Optional[Path] = None) -> list[LintFinding]:
  176. """Lint raw SKILL.md *content*.
  177. ``skill_dir`` enables on-disk checks (name/dir match, dangling links, POSIX
  178. gating, forbidden files); without it only content checks run, which is what
  179. the create path needs before the file exists.
  180. """
  181. frontmatter, body = parse_frontmatter(content)
  182. findings = list(_check_frontmatter(frontmatter, skill_dir)) + list(_check_body(body, skill_dir))
  183. if skill_dir is not None:
  184. findings += _check_files(frontmatter, skill_dir)
  185. return findings
  186. def lint_skill(skill_md_path: Path) -> list[LintFinding]:
  187. """Lint a SKILL.md file on disk, with all on-disk checks enabled."""
  188. skill_md_path = Path(skill_md_path)
  189. content = skill_md_path.read_text(encoding="utf-8-sig", errors="ignore")
  190. return lint_content(content, skill_dir=skill_md_path.parent)