outbound_webhooks.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346
  1. """Outbound webhooks: ``hooks.outbound`` entries (url, events, secret_env|secret, matcher for
  2. pre/post_tool_call, timeout clamped to [1, 60], name) -> notify-only callbacks on the plugin hook
  3. manager, so every ``invoke_hook()`` site can POST lifecycle events (mirror of
  4. ``gateway/platforms/webhook.py``). Fire-and-forget through a bounded queue + one daemon worker,
  5. so a target can never block a tool call or influence agent flow. HMAC-SHA256 signed
  6. (``X-Hermes-Signature-256: sha256=<hex>`` over the raw body) when a secret is configured;
  7. ``HERMES_SAFE_MODE=1`` skips registration; registration is idempotent.
  8. """
  9. from __future__ import annotations
  10. import atexit
  11. import hashlib
  12. import hmac
  13. import json
  14. import logging
  15. import queue
  16. import re
  17. import threading
  18. import time
  19. import uuid
  20. from dataclasses import dataclass, field
  21. from typing import Any, Dict, List, Optional, Set, Tuple
  22. from urllib import error as urlerror
  23. from urllib import request as urlrequest
  24. from agent.deadline import poll_until
  25. from agent.shell_hooks import (
  26. _TOOL_EVENTS as _TOOL_SCOPED_EVENTS,
  27. _ToolMatcherMixin,
  28. _forget_home_registrations,
  29. _home_key,
  30. _payload_fields,
  31. _utc_now_iso,
  32. )
  33. logger = logging.getLogger(__name__)
  34. DEFAULT_TIMEOUT_SECONDS = 10
  35. MAX_TIMEOUT_SECONDS = 60
  36. MAX_DELIVERY_ATTEMPTS = 2
  37. RETRY_BACKOFF_SECONDS = 1.0
  38. QUEUE_MAX_SIZE = 256
  39. # (home, event, url) triples already wired in this process. Home is part of the key so a
  40. # multiplexed gateway's secondary profiles (own plugin managers) can register identical targets.
  41. _registered: set[tuple[str, str, str]] = set()
  42. _registered_lock = threading.Lock()
  43. _delivery_queue: queue.Queue[Optional[dict[str, Any]]] = queue.Queue(maxsize=QUEUE_MAX_SIZE)
  44. _worker_lock = threading.Lock()
  45. _worker: Optional[threading.Thread] = None
  46. @dataclass
  47. class WebhookTarget(_ToolMatcherMixin):
  48. """Parsed and validated representation of one ``hooks.outbound`` entry."""
  49. _MATCHER_KIND = "outbound webhook"
  50. url: str
  51. events: list[str]
  52. name: str = ""
  53. secret: Optional[str] = None
  54. matcher: Optional[str] = None
  55. timeout: int = DEFAULT_TIMEOUT_SECONDS
  56. compiled_matcher: Optional[re.Pattern] = field(default=None, repr=False)
  57. @property
  58. def label(self) -> str:
  59. return self.name or self.url
  60. def register_from_config(cfg: Optional[dict[str, Any]]) -> list[WebhookTarget]:
  61. """Register every configured outbound webhook on the plugin manager. Malformed ``hooks.outbound``
  62. means zero targets — never raises. Returns the targets that ended up wired (deduplicated)."""
  63. from agent.safe_worker_policy import safe_worker_enabled
  64. if safe_worker_enabled():
  65. return []
  66. if not isinstance(cfg, dict):
  67. return []
  68. from utils import env_var_enabled
  69. if env_var_enabled("HERMES_SAFE_MODE"):
  70. logger.info("HERMES_SAFE_MODE=1 — outbound webhook registration skipped")
  71. return []
  72. targets = iter_configured_targets(cfg)
  73. if not targets:
  74. return []
  75. from hermes_cli.plugins import get_plugin_manager
  76. manager = get_plugin_manager()
  77. home_key = _home_key()
  78. registered: list[WebhookTarget] = []
  79. with _registered_lock:
  80. for target in targets:
  81. wired_any = False
  82. for event in target.events:
  83. key = (home_key, event, target.url)
  84. if key in _registered:
  85. continue
  86. manager._hooks.setdefault(event, []).append(_make_callback(event, target))
  87. _registered.add(key)
  88. wired_any = True
  89. logger.info(
  90. "outbound webhook registered: %s -> %s (matcher=%s, timeout=%ds)",
  91. event, target.label, target.matcher, target.timeout,
  92. )
  93. if wired_any:
  94. registered.append(target)
  95. return registered
  96. def iter_configured_targets(cfg: Optional[dict[str, Any]]) -> list[WebhookTarget]:
  97. """Parse ``hooks.outbound`` without registering anything (``hermes hooks list``)."""
  98. if not isinstance(cfg, dict):
  99. return []
  100. hooks_cfg = cfg.get("hooks")
  101. raw = hooks_cfg.get("outbound") if isinstance(hooks_cfg, dict) else None
  102. if raw is None:
  103. return []
  104. if not isinstance(raw, list):
  105. logger.warning("hooks.outbound must be a list of webhook targets; got %s", type(raw).__name__)
  106. return []
  107. return [t for t in (_parse_single_target(i, entry) for i, entry in enumerate(raw)) if t is not None]
  108. def flush(timeout: float = 5.0) -> bool:
  109. """Block until all queued deliveries are done (or *timeout* elapses); True if drained."""
  110. def _drained() -> bool:
  111. with _delivery_queue.all_tasks_done:
  112. return _delivery_queue.unfinished_tasks == 0
  113. return poll_until(_drained, timeout, 0.02)
  114. def re_register_config_hooks() -> None:
  115. """Re-register outbound webhooks after a plugin force-reload cleared ``_hooks``. Only the
  116. current home's idempotence keys are cleared so a force-reload in one profile cannot
  117. invalidate another profile's still-live registration.
  118. Mirrors ``agent.shell_hooks.re_register_config_hooks``: config-owned outbound-webhook callbacks live in
  119. the same ``_hooks`` dict that ``PluginManager.discover_and_load(force=True)`` clears via ``unload()``,
  120. so without this the force-reloaded profile's outbound webhooks go silently inert (#92682 review).
  121. """
  122. from hermes_cli.config import load_config
  123. _forget_home_registrations(_registered, _registered_lock)
  124. register_from_config(load_config())
  125. def reset_for_tests() -> None:
  126. """Clear the idempotence set and drain the queue. Test-only helper."""
  127. with _registered_lock:
  128. _registered.clear()
  129. try:
  130. while True:
  131. _delivery_queue.get_nowait()
  132. _delivery_queue.task_done()
  133. except queue.Empty:
  134. pass
  135. def _parse_single_target(index: int, raw: Any) -> Optional[WebhookTarget]:
  136. from hermes_cli.plugins import VALID_HOOKS
  137. def warn(msg: str, *args: Any) -> None:
  138. logger.warning("hooks.outbound[%d]" + msg, index, *args)
  139. if not isinstance(raw, dict):
  140. warn(" must be a mapping with 'url' and 'events' keys; got %s", type(raw).__name__)
  141. return None
  142. url = raw.get("url")
  143. if not isinstance(url, str) or not url.strip():
  144. warn(" is missing a non-empty 'url'")
  145. return None
  146. url = url.strip()
  147. if not url.lower().startswith(("http://", "https://")):
  148. warn(".url must be http(s); got %r — skipped", url)
  149. return None
  150. if url.lower().startswith("http://"):
  151. warn(".url uses plain http:// — payloads (including tool inputs) travel unencrypted. Prefer https.")
  152. events_raw = raw.get("events")
  153. valid_list = ", ".join(sorted(VALID_HOOKS))
  154. if not isinstance(events_raw, list) or not events_raw:
  155. warn(" needs a non-empty 'events' list (valid: %s)", valid_list)
  156. return None
  157. events: list[str] = [ev for ev in events_raw if ev in VALID_HOOKS]
  158. for ev in events_raw:
  159. if ev not in VALID_HOOKS:
  160. warn(": unknown event %r ignored (valid: %s)", ev, valid_list)
  161. if not events:
  162. warn(" has no valid events — skipped")
  163. return None
  164. matcher = raw.get("matcher")
  165. if matcher is not None and not isinstance(matcher, str):
  166. warn(".matcher must be a string regex; ignoring")
  167. matcher = None
  168. if matcher is not None and not any(e in _TOOL_SCOPED_EVENTS for e in events):
  169. warn(".matcher=%r will be ignored — matcher is only honored for pre_tool_call / post_tool_call.", matcher)
  170. matcher = None
  171. timeout_raw = raw.get("timeout", DEFAULT_TIMEOUT_SECONDS)
  172. try:
  173. timeout = int(timeout_raw)
  174. except (TypeError, ValueError):
  175. warn(".timeout must be an int (got %r); using default %ds", timeout_raw, DEFAULT_TIMEOUT_SECONDS)
  176. timeout = DEFAULT_TIMEOUT_SECONDS
  177. name = raw.get("name")
  178. # ``secret_env`` (env var name, preferred) wins over inline ``secret``. Read through the profile
  179. # secret scope: the gateway registers each multiplexed profile's targets inside that profile's
  180. # scope, and a raw environ read would sign a secondary's deliveries with the DEFAULT profile's
  181. # secret (or leave them unsigned when the var lives only in the secondary's .env).
  182. secret_env = raw.get("secret_env")
  183. if isinstance(secret_env, str) and secret_env.strip():
  184. from agent.secret_scope import get_secret
  185. secret = get_secret(secret_env.strip(), "") or None
  186. if secret is None:
  187. warn(".secret_env=%r is not set in the environment — deliveries will be UNSIGNED", secret_env.strip())
  188. else:
  189. secret = raw.get("secret")
  190. secret = secret if isinstance(secret, str) and secret else None
  191. return WebhookTarget(
  192. url=url, events=events, name=name.strip() if isinstance(name, str) else "", secret=secret,
  193. matcher=matcher, timeout=max(1, min(timeout, MAX_TIMEOUT_SECONDS)),
  194. )
  195. def _make_callback(event: str, target: WebhookTarget):
  196. """Build the notify-only closure ``invoke_hook()`` calls per firing."""
  197. def _callback(**kwargs: Any) -> None:
  198. from agent.safe_worker_policy import safe_worker_enabled
  199. if safe_worker_enabled():
  200. return
  201. if event in _TOOL_SCOPED_EVENTS and not target.matches_tool(kwargs.get("tool_name")):
  202. return
  203. delivery_id = uuid.uuid4().hex
  204. try:
  205. body = _serialize_payload(event, kwargs, delivery_id)
  206. except Exception: # a bad payload must not hurt the loop
  207. logger.warning("outbound webhook payload serialization failed (event=%s target=%s)", event, target.label, exc_info=True)
  208. return
  209. _enqueue(_build_delivery(event, target, body, delivery_id))
  210. _callback.__name__ = f"outbound_webhook[{event}:{target.label}]"
  211. _callback.__qualname__ = _callback.__name__
  212. return _callback
  213. def _serialize_payload(event: str, kwargs: dict[str, Any], delivery_id: str) -> bytes:
  214. """Render the POST body: shell-hooks stdin shape plus delivery metadata. ``delivery_id``
  215. (also the ``X-Hermes-Delivery`` header) and ``timestamp`` live inside the HMAC-signed
  216. body, so they double as replay protection."""
  217. # Profile resolved at fire time so a multiplexed gateway's receivers can tell which profile emitted.
  218. # See #92674.
  219. from hermes_cli.profiles import get_active_profile_name
  220. payload = {
  221. "hook_event_name": event, "profile": get_active_profile_name(), **_payload_fields(kwargs),
  222. "delivery_id": delivery_id, "timestamp": _utc_now_iso(),
  223. }
  224. return json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8")
  225. def _build_delivery(event: str, target: WebhookTarget, body: bytes, delivery_id: str) -> dict[str, Any]:
  226. headers = {
  227. "Content-Type": "application/json", "User-Agent": "Hermes-Agent-Outbound-Webhook",
  228. "X-Hermes-Event": event, "X-Hermes-Delivery": delivery_id,
  229. }
  230. if target.secret:
  231. digest = hmac.new(target.secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
  232. headers["X-Hermes-Signature-256"] = f"sha256={digest}"
  233. return {"url": target.url, "label": target.label, "event": event, "body": body, "headers": headers, "timeout": target.timeout}
  234. def _enqueue(delivery: dict[str, Any]) -> None:
  235. global _worker
  236. if _worker is None or not _worker.is_alive():
  237. with _worker_lock:
  238. if _worker is None or not _worker.is_alive():
  239. _worker = threading.Thread(target=_worker_loop, name="outbound-webhooks", daemon=True)
  240. _worker.start()
  241. # Daemon worker: a short-lived process could exit right after enqueuing on_session_end.
  242. # Drain at interpreter shutdown, bounded so a dead endpoint can only delay exit, never hang it.
  243. atexit.register(flush, timeout=5.0)
  244. try:
  245. _delivery_queue.put_nowait(delivery)
  246. except queue.Full:
  247. logger.warning(
  248. "outbound webhook queue full (%d pending) — dropping %s event for %s",
  249. QUEUE_MAX_SIZE, delivery["event"], delivery["label"],
  250. )
  251. def _worker_loop() -> None:
  252. while True:
  253. delivery = _delivery_queue.get()
  254. try:
  255. if delivery is not None:
  256. _deliver(delivery)
  257. except Exception: # pragma: no cover — defensive
  258. label = delivery.get("label") if isinstance(delivery, dict) else "?"
  259. logger.warning("outbound webhook delivery crashed (target=%s)", label, exc_info=True)
  260. finally:
  261. _delivery_queue.task_done()
  262. class _NoRedirectHandler(urlrequest.HTTPRedirectHandler):
  263. """Refuse redirects: urllib would turn a redirected POST into a body-less GET,
  264. silently dropping the signed payload. Any 3xx surfaces as HTTPError instead."""
  265. def redirect_request(self, req, fp, code, msg, headers, newurl):
  266. return None
  267. _opener = urlrequest.build_opener(_NoRedirectHandler)
  268. def _deliver(delivery: dict[str, Any]) -> None:
  269. """POST with bounded retries: retry on connection errors and 5xx; 4xx and 3xx are final."""
  270. event, label = delivery["event"], delivery["label"]
  271. last_error = ""
  272. for attempt in range(1, MAX_DELIVERY_ATTEMPTS + 1):
  273. req = urlrequest.Request(delivery["url"], data=delivery["body"], headers=delivery["headers"], method="POST")
  274. try:
  275. with _opener.open(req, timeout=delivery["timeout"]) as resp:
  276. status = getattr(resp, "status", 200)
  277. if 200 <= status < 300:
  278. logger.debug("outbound webhook delivered: %s -> %s (HTTP %d)", event, label, status)
  279. return
  280. last_error = f"HTTP {status}"
  281. except urlerror.HTTPError as exc:
  282. last_error = f"HTTP {exc.code}"
  283. if 300 <= exc.code < 400:
  284. logger.warning(
  285. "outbound webhook target redirected (event=%s target=%s): %s -> %s — redirects are not "
  286. "followed; update the configured url", event, label, last_error, exc.headers.get("Location", "?"),
  287. )
  288. return
  289. if 400 <= exc.code < 500:
  290. logger.warning("outbound webhook rejected (event=%s target=%s): %s — not retrying", event, label, last_error)
  291. return
  292. except Exception as exc:
  293. last_error = str(exc) or type(exc).__name__
  294. if attempt < MAX_DELIVERY_ATTEMPTS:
  295. time.sleep(RETRY_BACKOFF_SECONDS * attempt)
  296. logger.warning(
  297. "outbound webhook delivery failed after %d attempt(s) (event=%s target=%s): %s",
  298. MAX_DELIVERY_ATTEMPTS, event, label, last_error,
  299. )