| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198 |
- """Website access policy helpers for URL-capable tools.
- Loads a user-managed website blocklist (``security.website_blocklist`` in ~/.hermes/config.yaml plus
- optional shared list files) without the heavier CLI config stack. The parsed policy is cached with a
- short TTL so config edits take effect quickly without re-parsing YAML on every URL check.
- """
- from __future__ import annotations
- import fnmatch
- import logging
- import threading
- import time
- from pathlib import Path
- from typing import Any, Dict, List, Optional, Tuple
- from urllib.parse import urlparse
- from hermes_constants import get_hermes_home
- from tools.url_safety import _normalize_hostname as _normalize_host
- logger = logging.getLogger(__name__)
- _DEFAULT_WEBSITE_BLOCKLIST = {"enabled": False, "domains": [], "shared_files": []}
- # Without this cache a 50-URL extract would mean 51 YAML parses of config.yaml.
- _CACHE_TTL_SECONDS = 30.0
- _cache_lock = threading.Lock()
- _cached_policy: Optional[dict[str, Any]] = None
- _cached_policy_path: Optional[str] = None
- _cached_policy_time: float = 0.0
- def _fresh_cached_policy(resolved_path: str, now: float) -> Optional[dict[str, Any]]:
- """Return the cached policy only when it belongs to this config path and is still within TTL."""
- with _cache_lock:
- if (
- _cached_policy is not None
- and _cached_policy_path == resolved_path
- and (now - _cached_policy_time) < _CACHE_TTL_SECONDS
- ):
- return _cached_policy
- return None
- class WebsitePolicyError(Exception):
- """Raised when a website policy file is malformed."""
- def _normalize_rule(rule: Any) -> Optional[str]:
- """Reduce a rule (bare host, URL, or ``host/path``) to a lowercase host; None for blanks/comments."""
- if not isinstance(rule, str) or not (value := rule.strip().lower()) or value.startswith("#"):
- return None
- if "://" in value:
- parsed = urlparse(value)
- value = parsed.netloc or parsed.path
- return value.split("/", 1)[0].strip().rstrip(".").removeprefix("www.") or None
- def _iter_blocklist_file_rules(path: Path) -> list[str]:
- """Rules from a shared blocklist file; missing/unreadable files warn and yield nothing rather than
- raising — a bad file path must not disable all web tools."""
- try:
- raw = path.read_text(encoding="utf-8-sig")
- except FileNotFoundError:
- logger.warning("Shared blocklist file not found (skipping): %s", path)
- return []
- except (OSError, UnicodeDecodeError) as exc:
- logger.warning("Failed to read shared blocklist file %s (skipping): %s", path, exc)
- return []
- return [rule for rule in map(_normalize_rule, raw.splitlines()) if rule]
- def _require_mapping(value: Any, label: str) -> dict[str, Any]:
- """``None`` (empty YAML section) counts as an empty mapping; other non-dicts are errors."""
- if value is not None and not isinstance(value, dict):
- raise WebsitePolicyError(f"{label} must be a mapping")
- return value or {}
- def _load_policy_config(config_path: Path) -> dict[str, Any]:
- from hermes_cli.config_backend import config_exists, read_config_doc
- if not config_exists(config_path):
- return dict(_DEFAULT_WEBSITE_BLOCKLIST)
- try:
- import hermes_yaml as yaml
- except ImportError:
- logger.debug("ruamel.yaml not installed — website blocklist disabled")
- return dict(_DEFAULT_WEBSITE_BLOCKLIST)
- try:
- config = read_config_doc(config_path) or {}
- except yaml.YAMLError as exc:
- raise WebsitePolicyError(f"Invalid config YAML at {config_path}: {exc}") from exc
- except OSError as exc:
- raise WebsitePolicyError(f"Failed to read config file {config_path}: {exc}") from exc
- if not isinstance(config, dict):
- raise WebsitePolicyError("config root must be a mapping")
- security = _require_mapping(config.get("security", {}), "security")
- website_blocklist = _require_mapping(security.get("website_blocklist", {}), "security.website_blocklist")
- return {**_DEFAULT_WEBSITE_BLOCKLIST, **website_blocklist}
- def _require_type(policy: dict[str, Any], key: str, kind: type, default: Any) -> Any:
- """Typed policy field; ``None``/empty list values are coerced to ``[]`` for lists only."""
- value = policy.get(key, default)
- if kind is list:
- value = value or []
- if not isinstance(value, kind):
- kind_name = "boolean" if kind is bool else "list"
- raise WebsitePolicyError(f"security.website_blocklist.{key} must be a {kind_name}")
- return value
- def load_website_blocklist(config_path: Optional[Path] = None) -> dict[str, Any]:
- """Parsed website blocklist policy (``{"enabled", "rules"}``); cached for ``_CACHE_TTL_SECONDS`` for
- the default config path only — an explicit ``config_path`` (tests) bypasses and never populates it."""
- global _cached_policy, _cached_policy_path, _cached_policy_time
- default_path = get_hermes_home() / "config.yaml"
- resolved_path = str(config_path or default_path)
- now = time.monotonic()
- if config_path is None:
- cached = _fresh_cached_policy(resolved_path, now)
- if cached is not None:
- return cached
- config_path = config_path or default_path
- policy = _load_policy_config(config_path)
- domains = map(_normalize_rule, _require_type(policy, "domains", list, []))
- pairs: list[tuple[str, str]] = [(p, "config") for p in domains if p]
- shared_files = _require_type(policy, "shared_files", list, [])
- enabled = _require_type(policy, "enabled", bool, True)
- for shared_file in shared_files:
- if not isinstance(shared_file, str) or not shared_file.strip():
- continue
- path = Path(shared_file).expanduser()
- path = path if path.is_absolute() else (get_hermes_home() / path).resolve()
- pairs += [(normalized, str(path)) for normalized in _iter_blocklist_file_rules(path)]
- # dict.fromkeys dedupes (pattern, source) while keeping first-seen order.
- result = {"enabled": enabled, "rules": [{"pattern": p, "source": s} for p, s in dict.fromkeys(pairs)]}
- if config_path == default_path: # explicit paths are tests — never cache them
- with _cache_lock:
- _cached_policy, _cached_policy_path, _cached_policy_time = result, resolved_path, now
- return result
- def _match_host_against_rule(host: str, pattern: str) -> bool:
- """``*.example.com`` rules glob-match; bare hosts match exactly or as a parent domain."""
- if not host or not pattern:
- return False
- if pattern.startswith("*."):
- return fnmatch.fnmatch(host, pattern)
- return host == pattern or host.endswith(f".{pattern}")
- def _extract_host_from_urlish(url: str) -> str:
- """Host of ``url``; schemeless inputs (``example.com/x``) are retried as ``//url``."""
- parsed = urlparse(url)
- host = _normalize_host(parsed.hostname or parsed.netloc)
- if not host and "://" not in url:
- parsed = urlparse(f"//{url}")
- host = _normalize_host(parsed.hostname or parsed.netloc)
- return host
- def check_website_access(url: str, config_path: Optional[Path] = None) -> Optional[dict[str, str]]:
- """``None`` if the URL is allowed by the blocklist policy, else block metadata (host/rule/source/message).
- Fails open on policy errors (warn + ``None``) so a config typo can't break all web tools — except with
- an explicit ``config_path`` (tests), where errors propagate.
- """
- # Fast path: only a fresh disabled policy for THIS profile may skip loading/host parsing.
- if config_path is None:
- default_path = get_hermes_home() / "config.yaml"
- cached = _fresh_cached_policy(str(default_path), time.monotonic())
- if cached is not None and not cached.get("enabled"):
- return None
- host = _extract_host_from_urlish(url)
- if not host:
- return None
- try:
- policy = load_website_blocklist(config_path)
- except WebsitePolicyError as exc:
- if config_path is not None:
- raise
- logger.warning("Website policy config error (failing open): %s", exc)
- return None
- except Exception as exc:
- logger.warning("Unexpected error loading website policy (failing open): %s", exc)
- return None
- if not policy.get("enabled"):
- return None
- for rule in policy.get("rules", []):
- pattern, source = rule.get("pattern", ""), rule.get("source", "config")
- if _match_host_against_rule(host, pattern):
- logger.info("Blocked URL %s — matched rule '%s' from %s", url, pattern, source)
- return {
- "url": url, "host": host, "rule": pattern, "source": source,
- "message": f"Blocked by website policy: '{host}' matched rule '{pattern}' from {source}",
- }
- return None
|