osv_check.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269
  1. """OSV malware check for MCP extension packages.
  2. Before launching an MCP server via npx/uvx, queries Google's free public OSV API for
  3. known malware advisories (MAL-* IDs). Regular CVEs are ignored — only confirmed malware
  4. is blocked. Fail-open: network errors allow the package to proceed (~300ms typical).
  5. Inspired by Block/goose's extension malware check.
  6. """
  7. import json
  8. import logging
  9. import os
  10. import re
  11. import threading
  12. import time
  13. import urllib.request
  14. from pathlib import Path
  15. from typing import Optional, Tuple
  16. logger = logging.getLogger(__name__)
  17. _OSV_ENDPOINT = os.getenv("OSV_ENDPOINT", "https://api.osv.dev/v1/query")
  18. _TIMEOUT = 10 # seconds
  19. # Result cache: (ecosystem, package, version) -> (expiry_wallclock, result). Reconnect
  20. # ladders, parked-server self-probes and repeated `hermes mcp test` runs re-run the preflight
  21. # for the SAME package on every spawn; uncached, a flapping server becomes a sustained OSV/DNS
  22. # query stream. Clean AND blocked verdicts are reusable; network failures are NOT cached
  23. # (fail-open covers them and caching one could mask a real advisory later).
  24. # The cache is also persisted under the Hermes home so separate processes and gateway
  25. # restarts reuse warm verdicts; expiry is absolute wall-clock time so it survives restarts.
  26. # Trade-off: a MAL advisory published right after a clean verdict is noticed at TTL expiry
  27. # (<= 1h by default) rather than at next process start — lower OSV_CHECK_CACHE_TTL to tighten.
  28. # Without a cache, a flapping server turns into a sustained OSV query/DNS stream — the #75485 incident
  29. # logged 779K api.osv.dev DNS queries in 16h from revival loops. Malware advisories don't appear or vanish
  30. # on second-to-second timescales, so a successful verdict (clean OR blocked) is reusable. The window is the
  31. # same one the in-process cache already accepted; it just now spans restarts.
  32. _CACHE_TTL_S = float(os.getenv("OSV_CHECK_CACHE_TTL", "3600"))
  33. _CACHE_MAX_ENTRIES = 256
  34. _cache: dict = {}
  35. _cache_lock = threading.Lock()
  36. _disk_cache_loaded = False
  37. _DISK_CACHE_VERSION = 1
  38. def _disk_cache_path() -> Optional[Path]:
  39. """Return the path for the persistent OSV verdict cache.
  40. Uses ``hermes_constants.get_hermes_home()`` so the cache follows the
  41. active profile and is isolated across Hermes homes. The cache directory
  42. is created on demand. Returns ``None`` when Hermes home cannot be
  43. resolved, in which case only the in-process cache is used.
  44. """
  45. try:
  46. from hermes_constants import get_hermes_home
  47. home = get_hermes_home()
  48. except Exception:
  49. return None
  50. try:
  51. cache_dir = home / "cache"
  52. cache_dir.mkdir(parents=True, exist_ok=True)
  53. return cache_dir / "osv_check.json"
  54. except Exception:
  55. return None
  56. def _load_disk_cache() -> None:
  57. """Load persistent cache entries from disk into the in-process cache.
  58. Invoked under ``_cache_lock`` from every get/put but does real work only
  59. once per process (``_disk_cache_loaded`` latch); a transient ``OSError``
  60. leaves the latch unset so the next call retries. Skips expired or
  61. malformed entries. Only adds missing keys so an in-memory overwrite
  62. (e.g. a test forcing expiry) is not silently reversed by the disk copy.
  63. """
  64. global _disk_cache_loaded
  65. if _disk_cache_loaded:
  66. return
  67. path = _disk_cache_path()
  68. if path is None:
  69. _disk_cache_loaded = True
  70. return
  71. try:
  72. with open(path, "r", encoding="utf-8-sig") as f:
  73. data = json.load(f)
  74. except FileNotFoundError:
  75. data = None
  76. except OSError:
  77. # Transient I/O (file busy, brief permission flap). Retry next call.
  78. return
  79. except Exception:
  80. # Malformed JSON or anything else: unrecoverable, don't spin on it.
  81. data = None
  82. _disk_cache_loaded = True
  83. if not isinstance(data, dict) or data.get("version") != _DISK_CACHE_VERSION:
  84. return
  85. now = time.time()
  86. for key_str, entry in data.get("entries", {}).items():
  87. if not isinstance(entry, dict):
  88. continue
  89. expiry = entry.get("expiry")
  90. result = entry.get("result")
  91. if expiry is None or expiry <= now:
  92. continue
  93. parts = key_str.split("|", 2)
  94. if len(parts) != 3:
  95. continue
  96. key = (parts[0], parts[1], parts[2] or None)
  97. if key not in _cache:
  98. _cache[key] = (expiry, result)
  99. def _save_disk_cache() -> None:
  100. """Persist the in-process cache to disk.
  101. Caller must hold ``_cache_lock`` for consistency. Writes atomically to
  102. a sibling file then renames into place.
  103. """
  104. path = _disk_cache_path()
  105. if path is None:
  106. return
  107. entries: dict = {}
  108. for key, (expiry, result) in _cache.items():
  109. key_str = "|".join(str(k) if k is not None else "" for k in key)
  110. entries[key_str] = {"expiry": expiry, "result": result}
  111. data = {"version": _DISK_CACHE_VERSION, "entries": entries}
  112. try:
  113. # Shared atomic writer (temp file + fsync + rename); mkstemp's 0600
  114. # is kept on create, so verdicts never sit in a world-readable file.
  115. from utils import atomic_write_text
  116. atomic_write_text(path, json.dumps(data))
  117. except Exception as exc:
  118. logger.debug("Failed to save OSV disk cache to %s: %s", path, exc)
  119. def _cache_get(key) -> tuple[bool, Optional[str]]:
  120. """Return (hit, result) for a fresh cache entry."""
  121. with _cache_lock:
  122. _load_disk_cache()
  123. entry = _cache.get(key)
  124. if entry is not None and time.time() < entry[0]:
  125. return True, entry[1]
  126. _cache.pop(key, None) # absent or expired
  127. return False, None
  128. def _cache_put(key, result: Optional[str]) -> None:
  129. with _cache_lock:
  130. _load_disk_cache()
  131. if len(_cache) >= _CACHE_MAX_ENTRIES:
  132. now = time.time()
  133. for k in [k for k, (exp, _) in _cache.items() if exp <= now]:
  134. del _cache[k]
  135. if len(_cache) >= _CACHE_MAX_ENTRIES:
  136. _cache.clear() # tiny working set in practice; safe reset
  137. _cache[key] = (time.time() + _CACHE_TTL_S, result)
  138. _save_disk_cache()
  139. def check_package_for_malware(command: str, args: list) -> Optional[str]:
  140. """Check an MCP server package (inferred from ``command``/``args``) for MAL-* advisories.
  141. Returns a BLOCKED message, else None — also on network errors/unknown commands (fail-open)."""
  142. ecosystem = _infer_ecosystem(command)
  143. if not ecosystem:
  144. return None # not npx/uvx — skip
  145. package, version = _parse_package_from_args(args, ecosystem)
  146. if not package:
  147. return None
  148. cache_key = (ecosystem, package, version)
  149. hit, cached = _cache_get(cache_key)
  150. if hit:
  151. return cached
  152. try:
  153. malware = _query_osv(package, ecosystem, version)
  154. except Exception as exc:
  155. # Fail-open; deliberately NOT cached — see _CACHE_TTL_S comment.
  156. logger.debug("OSV check failed for %s/%s (allowing): %s", ecosystem, package, exc)
  157. return None
  158. result = None
  159. if malware:
  160. ids = ", ".join(m["id"] for m in malware[:3])
  161. summaries = "; ".join(m.get("summary", m["id"])[:100] for m in malware[:3])
  162. result = (f"BLOCKED: Package '{package}' ({ecosystem}) has known malware "
  163. f"advisories: {ids}. Details: {summaries}")
  164. _cache_put(cache_key, result)
  165. return result
  166. _ECOSYSTEM_BY_COMMAND = {
  167. "npx": "npm", "npx.cmd": "npm",
  168. "uvx": "PyPI", "uvx.cmd": "PyPI", "uvx.exe": "PyPI",
  169. "pipx": "PyPI", "pipx.exe": "PyPI",
  170. }
  171. def _infer_ecosystem(command: str) -> Optional[str]:
  172. # Split on BOTH separators: os.path.basename leaves ``C:\...\uvx.exe`` intact on POSIX
  173. # (config authored for Windows) and the preflight would silently skip. Only the shim
  174. # names each runner actually installs are listed; lookalikes stay fail-open.
  175. return _ECOSYSTEM_BY_COMMAND.get(re.split(r"[\\/]", command)[-1].lower())
  176. def _parse_package_from_args(args: list, ecosystem: str) -> tuple[Optional[str], Optional[str]]:
  177. """Extract (package_name, version) from command args, or (None, None) if not parseable."""
  178. # Skip flags to find the package token. npx's explicit install target (--package=NAME /
  179. # --package NAME / -p NAME) names a package distinct from the executed binary.
  180. package_token = None
  181. take_next = False
  182. for arg in args or ():
  183. if not isinstance(arg, str):
  184. continue
  185. if take_next:
  186. package_token = arg
  187. break
  188. if arg in ("--package", "-p"):
  189. take_next = True
  190. continue
  191. if arg.startswith("--package="):
  192. package_token = arg[len("--package="):]
  193. break
  194. if arg.startswith("-"):
  195. continue
  196. package_token = arg
  197. break
  198. if not package_token:
  199. return None, None
  200. parser = _PACKAGE_PARSERS.get(ecosystem)
  201. return parser(package_token) if parser else (package_token, None)
  202. def _parse_npm_package(token: str) -> tuple[Optional[str], Optional[str]]:
  203. """Parse npm package: @scope/name@version or name@version."""
  204. if token.startswith("@"):
  205. match = re.match(r"^(@[^/]+/[^@]+)(?:@(.+))?$", token)
  206. return (match.group(1), match.group(2)) if match else (token, None)
  207. if "@" in token:
  208. name, version = token.rsplit("@", 1)
  209. return name, version if version != "latest" else None
  210. return token, None
  211. def _parse_pypi_package(token: str) -> tuple[Optional[str], Optional[str]]:
  212. """Parse PyPI package: name==version or name[extras]==version."""
  213. match = re.match(r"^([a-zA-Z0-9._-]+)(?:\[[^\]]*\])?(?:==(.+))?$", token)
  214. return (match.group(1), match.group(2)) if match else (token, None)
  215. _PACKAGE_PARSERS = {"npm": _parse_npm_package, "PyPI": _parse_pypi_package}
  216. def _query_osv(package: str, ecosystem: str, version: Optional[str] = None) -> list:
  217. """Query the OSV API; return only MAL-* advisories (regular CVEs ignored)."""
  218. payload = {"package": {"name": package, "ecosystem": ecosystem}}
  219. if version:
  220. payload["version"] = version
  221. req = urllib.request.Request(
  222. _OSV_ENDPOINT,
  223. data=json.dumps(payload).encode("utf-8"),
  224. headers={"Content-Type": "application/json", "User-Agent": "hermes-agent-osv-check/1.0"},
  225. method="POST")
  226. with urllib.request.urlopen(req, timeout=_TIMEOUT) as resp:
  227. result = json.loads(resp.read())
  228. return [v for v in result.get("vulns", []) if v.get("id", "").startswith("MAL-")]