skillevaluator_scan.py 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141
  1. #!/usr/bin/env python3
  2. """Advisory NVIDIA SkillEvaluator Tier 1 scan for skill installs — alongside (never instead of) skills_guard,
  3. the enforcement layer. Contract: warn, don't block (the upstream PII scanner false-positives on
  4. ``git@github.com`` / ``op://``); prompt only for secrets-class criticals (``--force`` / non-interactive proceed
  5. with a loud warning); a missing/crashing/timed-out/unparseable scanner is a no-op. Toggle
  6. ``skills.tier1_advisory`` (default on). Binary: ``uv tool install --python 3.13
  7. "skillevaluator @ git+https://github.com/NVIDIA/SkillEvaluator.git@v0.1.0"``."""
  8. from __future__ import annotations
  9. import json
  10. import logging
  11. import shutil
  12. import subprocess
  13. import tempfile
  14. from dataclasses import dataclass, field
  15. from pathlib import Path
  16. from typing import List
  17. logger = logging.getLogger(__name__)
  18. SCANNER_BIN = "skillevaluator"
  19. # Keyless, deterministic checks (schema/quality are index-pipeline hygiene, not install-time signal). `security`
  20. # invokes NVIDIA SkillSpector (static rules, no LLM); when absent it reports status="incomplete".
  21. TIER1_CHECKS = "pii,unicode,lint,license,security"
  22. SCAN_TIMEOUT_SECONDS = 120
  23. # pii_patterns.yaml categories indicating a possible REAL credential (not PII hygiene) — the only prompt-worthy ones.
  24. SECRETS_CLASS_CHECKS = frozenset({"database_credentials", "hardcoded_secrets", "jwt_tokens", "webhook_urls",
  25. "aws_identifiers", "github_tokens", "private_keys"})
  26. @dataclass
  27. class Tier1Finding:
  28. check: str # e.g. "emails", "database_credentials"
  29. validator: str # e.g. "PII Scan"
  30. severity: str # "critical" | "high" | "medium" | "low" | "info"
  31. message: str
  32. file: str = ""
  33. line: int = 0
  34. suggestion: str = ""
  35. @property
  36. def is_secrets_class(self) -> bool:
  37. return self.check in SECRETS_CLASS_CHECKS
  38. def location(self) -> str:
  39. return f"{self.file}:{self.line}" if self.file and self.line else self.file or "?"
  40. @dataclass
  41. class Tier1Report:
  42. available: bool # scanner ran and produced a report
  43. passed: bool = True
  44. findings: list[Tier1Finding] = field(default_factory=list)
  45. incomplete_checks: list[str] = field(default_factory=list)
  46. error: str = "" # why the scan is unavailable (debug only)
  47. @property
  48. def advisory_findings(self) -> list[Tier1Finding]:
  49. return [f for f in self.findings if not f.is_secrets_class]
  50. @property
  51. def secrets_findings(self) -> list[Tier1Finding]:
  52. return [f for f in self.findings if f.is_secrets_class]
  53. def tier1_advisory_enabled() -> bool:
  54. """``skills.tier1_advisory`` (default True; safe because the scan is a no-op without the binary)."""
  55. try:
  56. from hermes_cli.config import load_config
  57. skills_cfg = load_config().get("skills") or {}
  58. value = skills_cfg.get("tier1_advisory", True) if isinstance(skills_cfg, dict) else True
  59. return value.strip().lower() not in ("false", "0", "no", "off") if isinstance(value, str) else bool(value)
  60. except Exception:
  61. return True
  62. def _parse_report(report: dict) -> Tier1Report:
  63. """Reduce a SkillEvaluator JSON report to install-relevant findings. Findings from ``status == "incomplete"``
  64. validators are kept (partial evidence is evidence) but excluded from the pass/fail signal."""
  65. findings: list[Tier1Finding] = []
  66. incomplete: list[str] = []
  67. failed = False
  68. for res in report.get("results", []) or []:
  69. validator = str(res.get("validator", "unknown"))
  70. if str(res.get("status", "")).lower() == "incomplete":
  71. incomplete.append(validator)
  72. else:
  73. failed = failed or not res.get("passed", True)
  74. findings.extend(Tier1Finding(
  75. check=str(f.get("check_name", "")), validator=validator, severity=str(f.get("severity", "info")).lower(),
  76. message=str(f.get("message", ""))[:200], file=str(f.get("file_path", "")),
  77. line=int(f.get("line_number") or 0), suggestion=str(f.get("suggestion", ""))[:200])
  78. for f in res.get("findings", []) or [] if isinstance(f, dict))
  79. return Tier1Report(available=True, passed=not failed and not findings, findings=findings,
  80. incomplete_checks=incomplete)
  81. def run_tier1_scan(skill_dir: Path, timeout: int = SCAN_TIMEOUT_SECONDS) -> Tier1Report:
  82. """Run SkillEvaluator Tier 1 over one skill dir; any failure returns ``available=False``, never raises."""
  83. unavailable = lambda why: Tier1Report(available=False, error=why)
  84. if shutil.which(SCANNER_BIN) is None:
  85. return unavailable("scanner not on PATH")
  86. with tempfile.TemporaryDirectory(prefix="se-tier1-") as outdir:
  87. try:
  88. subprocess.run([SCANNER_BIN, "validate", str(skill_dir), "--checks", TIER1_CHECKS, "--no-dedup",
  89. "-r", "json", "-o", outdir], capture_output=True, text=True, encoding="utf-8", errors="replace",
  90. stdin=subprocess.DEVNULL, timeout=timeout, check=False)
  91. except subprocess.TimeoutExpired:
  92. return unavailable(f"scan timed out after {timeout}s")
  93. except OSError as exc:
  94. return unavailable(f"scanner failed to launch: {exc}")
  95. if not (reports := sorted(Path(outdir).glob("skillevaluator-output-*.json"))):
  96. return unavailable("scanner produced no JSON report")
  97. try:
  98. parsed = json.loads(reports[-1].read_text(encoding="utf-8-sig"))
  99. except (json.JSONDecodeError, OSError) as exc:
  100. return unavailable(f"unparseable report: {exc}")
  101. return _parse_report(parsed) if isinstance(parsed, dict) else unavailable("unexpected report shape")
  102. def format_tier1_report(report: Tier1Report, limit: int = 10) -> str:
  103. """Plain-text advisory summary for console display ("" when unavailable)."""
  104. if not report.available:
  105. return ""
  106. lines: list[str] = []
  107. if not report.findings:
  108. lines.append("SkillEvaluator Tier 1: no findings from completed checks." if report.incomplete_checks
  109. else "SkillEvaluator Tier 1: no findings.")
  110. else:
  111. lines.append(f"SkillEvaluator Tier 1 (advisory): {len(report.findings)} finding(s) — informational, "
  112. "verify before relying on this skill.")
  113. shown = report.secrets_findings + report.advisory_findings
  114. lines.extend(f" [{'SECRETS' if f.is_secrets_class else f.severity.upper()}] {f.location()} — {f.message}"
  115. for f in shown[:limit])
  116. if len(shown) > limit:
  117. lines.append(f" … and {len(shown) - limit} more")
  118. if report.incomplete_checks:
  119. lines.append(f" (not run: {', '.join(report.incomplete_checks)} — no opinion from these checks)")
  120. return "\n".join(lines)