website_policy.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198
  1. """Website access policy helpers for URL-capable tools.
  2. Loads a user-managed website blocklist (``security.website_blocklist`` in ~/.hermes/config.yaml plus
  3. optional shared list files) without the heavier CLI config stack. The parsed policy is cached with a
  4. short TTL so config edits take effect quickly without re-parsing YAML on every URL check.
  5. """
  6. from __future__ import annotations
  7. import fnmatch
  8. import logging
  9. import threading
  10. import time
  11. from pathlib import Path
  12. from typing import Any, Dict, List, Optional, Tuple
  13. from urllib.parse import urlparse
  14. from hermes_constants import get_hermes_home
  15. from tools.url_safety import _normalize_hostname as _normalize_host
  16. logger = logging.getLogger(__name__)
  17. _DEFAULT_WEBSITE_BLOCKLIST = {"enabled": False, "domains": [], "shared_files": []}
  18. # Without this cache a 50-URL extract would mean 51 YAML parses of config.yaml.
  19. _CACHE_TTL_SECONDS = 30.0
  20. _cache_lock = threading.Lock()
  21. _cached_policy: Optional[dict[str, Any]] = None
  22. _cached_policy_path: Optional[str] = None
  23. _cached_policy_time: float = 0.0
  24. def _fresh_cached_policy(resolved_path: str, now: float) -> Optional[dict[str, Any]]:
  25. """Return the cached policy only when it belongs to this config path and is still within TTL."""
  26. with _cache_lock:
  27. if (
  28. _cached_policy is not None
  29. and _cached_policy_path == resolved_path
  30. and (now - _cached_policy_time) < _CACHE_TTL_SECONDS
  31. ):
  32. return _cached_policy
  33. return None
  34. class WebsitePolicyError(Exception):
  35. """Raised when a website policy file is malformed."""
  36. def _normalize_rule(rule: Any) -> Optional[str]:
  37. """Reduce a rule (bare host, URL, or ``host/path``) to a lowercase host; None for blanks/comments."""
  38. if not isinstance(rule, str) or not (value := rule.strip().lower()) or value.startswith("#"):
  39. return None
  40. if "://" in value:
  41. parsed = urlparse(value)
  42. value = parsed.netloc or parsed.path
  43. return value.split("/", 1)[0].strip().rstrip(".").removeprefix("www.") or None
  44. def _iter_blocklist_file_rules(path: Path) -> list[str]:
  45. """Rules from a shared blocklist file; missing/unreadable files warn and yield nothing rather than
  46. raising — a bad file path must not disable all web tools."""
  47. try:
  48. raw = path.read_text(encoding="utf-8-sig")
  49. except FileNotFoundError:
  50. logger.warning("Shared blocklist file not found (skipping): %s", path)
  51. return []
  52. except (OSError, UnicodeDecodeError) as exc:
  53. logger.warning("Failed to read shared blocklist file %s (skipping): %s", path, exc)
  54. return []
  55. return [rule for rule in map(_normalize_rule, raw.splitlines()) if rule]
  56. def _require_mapping(value: Any, label: str) -> dict[str, Any]:
  57. """``None`` (empty YAML section) counts as an empty mapping; other non-dicts are errors."""
  58. if value is not None and not isinstance(value, dict):
  59. raise WebsitePolicyError(f"{label} must be a mapping")
  60. return value or {}
  61. def _load_policy_config(config_path: Path) -> dict[str, Any]:
  62. from hermes_cli.config_backend import config_exists, read_config_doc
  63. if not config_exists(config_path):
  64. return dict(_DEFAULT_WEBSITE_BLOCKLIST)
  65. try:
  66. import hermes_yaml as yaml
  67. except ImportError:
  68. logger.debug("ruamel.yaml not installed — website blocklist disabled")
  69. return dict(_DEFAULT_WEBSITE_BLOCKLIST)
  70. try:
  71. config = read_config_doc(config_path) or {}
  72. except yaml.YAMLError as exc:
  73. raise WebsitePolicyError(f"Invalid config YAML at {config_path}: {exc}") from exc
  74. except OSError as exc:
  75. raise WebsitePolicyError(f"Failed to read config file {config_path}: {exc}") from exc
  76. if not isinstance(config, dict):
  77. raise WebsitePolicyError("config root must be a mapping")
  78. security = _require_mapping(config.get("security", {}), "security")
  79. website_blocklist = _require_mapping(security.get("website_blocklist", {}), "security.website_blocklist")
  80. return {**_DEFAULT_WEBSITE_BLOCKLIST, **website_blocklist}
  81. def _require_type(policy: dict[str, Any], key: str, kind: type, default: Any) -> Any:
  82. """Typed policy field; ``None``/empty list values are coerced to ``[]`` for lists only."""
  83. value = policy.get(key, default)
  84. if kind is list:
  85. value = value or []
  86. if not isinstance(value, kind):
  87. kind_name = "boolean" if kind is bool else "list"
  88. raise WebsitePolicyError(f"security.website_blocklist.{key} must be a {kind_name}")
  89. return value
  90. def load_website_blocklist(config_path: Optional[Path] = None) -> dict[str, Any]:
  91. """Parsed website blocklist policy (``{"enabled", "rules"}``); cached for ``_CACHE_TTL_SECONDS`` for
  92. the default config path only — an explicit ``config_path`` (tests) bypasses and never populates it."""
  93. global _cached_policy, _cached_policy_path, _cached_policy_time
  94. default_path = get_hermes_home() / "config.yaml"
  95. resolved_path = str(config_path or default_path)
  96. now = time.monotonic()
  97. if config_path is None:
  98. cached = _fresh_cached_policy(resolved_path, now)
  99. if cached is not None:
  100. return cached
  101. config_path = config_path or default_path
  102. policy = _load_policy_config(config_path)
  103. domains = map(_normalize_rule, _require_type(policy, "domains", list, []))
  104. pairs: list[tuple[str, str]] = [(p, "config") for p in domains if p]
  105. shared_files = _require_type(policy, "shared_files", list, [])
  106. enabled = _require_type(policy, "enabled", bool, True)
  107. for shared_file in shared_files:
  108. if not isinstance(shared_file, str) or not shared_file.strip():
  109. continue
  110. path = Path(shared_file).expanduser()
  111. path = path if path.is_absolute() else (get_hermes_home() / path).resolve()
  112. pairs += [(normalized, str(path)) for normalized in _iter_blocklist_file_rules(path)]
  113. # dict.fromkeys dedupes (pattern, source) while keeping first-seen order.
  114. result = {"enabled": enabled, "rules": [{"pattern": p, "source": s} for p, s in dict.fromkeys(pairs)]}
  115. if config_path == default_path: # explicit paths are tests — never cache them
  116. with _cache_lock:
  117. _cached_policy, _cached_policy_path, _cached_policy_time = result, resolved_path, now
  118. return result
  119. def _match_host_against_rule(host: str, pattern: str) -> bool:
  120. """``*.example.com`` rules glob-match; bare hosts match exactly or as a parent domain."""
  121. if not host or not pattern:
  122. return False
  123. if pattern.startswith("*."):
  124. return fnmatch.fnmatch(host, pattern)
  125. return host == pattern or host.endswith(f".{pattern}")
  126. def _extract_host_from_urlish(url: str) -> str:
  127. """Host of ``url``; schemeless inputs (``example.com/x``) are retried as ``//url``."""
  128. parsed = urlparse(url)
  129. host = _normalize_host(parsed.hostname or parsed.netloc)
  130. if not host and "://" not in url:
  131. parsed = urlparse(f"//{url}")
  132. host = _normalize_host(parsed.hostname or parsed.netloc)
  133. return host
  134. def check_website_access(url: str, config_path: Optional[Path] = None) -> Optional[dict[str, str]]:
  135. """``None`` if the URL is allowed by the blocklist policy, else block metadata (host/rule/source/message).
  136. Fails open on policy errors (warn + ``None``) so a config typo can't break all web tools — except with
  137. an explicit ``config_path`` (tests), where errors propagate.
  138. """
  139. # Fast path: only a fresh disabled policy for THIS profile may skip loading/host parsing.
  140. if config_path is None:
  141. default_path = get_hermes_home() / "config.yaml"
  142. cached = _fresh_cached_policy(str(default_path), time.monotonic())
  143. if cached is not None and not cached.get("enabled"):
  144. return None
  145. host = _extract_host_from_urlish(url)
  146. if not host:
  147. return None
  148. try:
  149. policy = load_website_blocklist(config_path)
  150. except WebsitePolicyError as exc:
  151. if config_path is not None:
  152. raise
  153. logger.warning("Website policy config error (failing open): %s", exc)
  154. return None
  155. except Exception as exc:
  156. logger.warning("Unexpected error loading website policy (failing open): %s", exc)
  157. return None
  158. if not policy.get("enabled"):
  159. return None
  160. for rule in policy.get("rules", []):
  161. pattern, source = rule.get("pattern", ""), rule.get("source", "config")
  162. if _match_host_against_rule(host, pattern):
  163. logger.info("Blocked URL %s — matched rule '%s' from %s", url, pattern, source)
  164. return {
  165. "url": url, "host": host, "rule": pattern, "source": source,
  166. "message": f"Blocked by website policy: '{host}' matched rule '{pattern}' from {source}",
  167. }
  168. return None