msgraph_webhook.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359
  1. """Microsoft Graph webhook adapter for change-notification ingress."""
  2. from __future__ import annotations
  3. import asyncio
  4. import hmac
  5. import ipaddress
  6. import json
  7. import logging
  8. import re
  9. from collections import deque
  10. from hashlib import sha1
  11. from typing import Any, Awaitable, Callable, Dict, Optional
  12. try:
  13. from aiohttp import web
  14. AIOHTTP_AVAILABLE = True
  15. except ImportError:
  16. AIOHTTP_AVAILABLE = False
  17. web = None # type: ignore[assignment]
  18. from gateway.config import Platform, PlatformConfig
  19. from gateway.platforms.base import (
  20. BasePlatformAdapter, SendResult, is_network_accessible,
  21. )
  22. from gateway.platforms.event import MessageEvent, MessageType
  23. from datetime import UTC
  24. logger = logging.getLogger(__name__)
  25. # ``None`` → aiohttp binds one socket per address family (IPv4 + IPv6); the old "0.0.0.0" default was
  26. # unreachable over IPv6-only private networks. Pin a host via extra.host. The all-interfaces default
  27. # still requires extra.allowed_source_cidrs (see _source_allowlist_required_but_missing).
  28. DEFAULT_HOST = None
  29. DEFAULT_PORT = 8646
  30. DEFAULT_WEBHOOK_PATH = "/msgraph/webhook"
  31. DEFAULT_MAX_SEEN_RECEIPTS = 5000
  32. DEFAULT_MAX_BODY_BYTES = 1_048_576
  33. NotificationScheduler = Callable[[dict[str, Any], MessageEvent], Awaitable[None] | None]
  34. _TEMPLATE_KEY_RE = re.compile(r"\{([a-zA-Z0-9_.]+)\}")
  35. def check_msgraph_webhook_requirements() -> bool:
  36. """Return whether required webhook dependencies are available."""
  37. return AIOHTTP_AVAILABLE
  38. def _string_or_none(value: Any) -> Optional[str]:
  39. return None if value is None else (str(value).strip() or None)
  40. def _normalize_path(path: Any) -> str:
  41. raw = str(path or "").strip() or "/"
  42. return raw if raw.startswith("/") else f"/{raw}"
  43. def _parse_allowed_source_cidrs(raw: Any) -> list[ipaddress._BaseNetwork]:
  44. """Parse the optional CIDR allowlist; empty/missing means "allow everything". When populated, source
  45. IPs outside every listed CIDR get 403 before the body is parsed (restrict to Microsoft Graph's
  46. published webhook source ranges in production)."""
  47. if isinstance(raw, str):
  48. candidates = raw.split(",")
  49. elif isinstance(raw, (list, tuple, set)):
  50. candidates = [str(chunk) for chunk in raw]
  51. else:
  52. return []
  53. networks: list[ipaddress._BaseNetwork] = []
  54. for chunk in (c.strip() for c in candidates):
  55. if not chunk:
  56. continue
  57. try:
  58. networks.append(ipaddress.ip_network(chunk, strict=False))
  59. except ValueError:
  60. logger.warning("[msgraph_webhook] Ignoring invalid allowed_source_cidrs entry: %r", chunk)
  61. return networks
  62. def _prefix_match(resource: str, prefix: str) -> bool:
  63. return resource == prefix or resource.startswith(f"{prefix}/")
  64. def _render_template(template: str, payload: dict[str, Any]) -> str:
  65. """Substitute ``{dotted.key}`` placeholders from *payload*; unknown keys stay literal."""
  66. def _resolve(match: re.Match[str]) -> str:
  67. key = match.group(1)
  68. value: Any = payload
  69. for part in key.split("."):
  70. if not isinstance(value, dict):
  71. return f"{{{key}}}"
  72. value = value.get(part, f"{{{key}}}")
  73. if isinstance(value, (dict, list)):
  74. return json.dumps(value, sort_keys=True)[:2000]
  75. return str(value)
  76. return _TEMPLATE_KEY_RE.sub(_resolve, template)
  77. class MSGraphWebhookAdapter(BasePlatformAdapter):
  78. """Receive Microsoft Graph change notifications and surface them internally."""
  79. # Answers /p/<profile>/... on the default listener for a served secondary (shared_ingress).
  80. serves_profile_prefix: bool = True
  81. def __init__(self, config: PlatformConfig):
  82. super().__init__(config, Platform.MSGRAPH_WEBHOOK)
  83. extra = config.extra or {}
  84. # Falsy host (None/"") collapses to the dual-stack default.
  85. _raw_host = extra.get("host", DEFAULT_HOST) or DEFAULT_HOST
  86. self._host: Optional[str] = str(_raw_host) if _raw_host else None
  87. self._port: int = int(extra.get("port", DEFAULT_PORT))
  88. self._webhook_path: str = _normalize_path(extra.get("webhook_path", DEFAULT_WEBHOOK_PATH))
  89. self._health_path: str = _normalize_path(extra.get("health_path", "/health"))
  90. self._accepted_resources: list[str] = [
  91. str(value).strip() for value in (extra.get("accepted_resources") or []) if str(value).strip()]
  92. self._client_state: Optional[str] = _string_or_none(extra.get("client_state"))
  93. self._max_seen_receipts = max(1, int(extra.get("max_seen_receipts", DEFAULT_MAX_SEEN_RECEIPTS)))
  94. self._max_body_bytes = max(1, int(extra.get("max_body_bytes", DEFAULT_MAX_BODY_BYTES)))
  95. self._allowed_source_networks = _parse_allowed_source_cidrs(extra.get("allowed_source_cidrs"))
  96. self._runner = None
  97. self._notification_scheduler: Optional[NotificationScheduler] = None
  98. self._seen_receipts: set[str] = set()
  99. self._seen_receipt_order: deque[str] = deque()
  100. self._accepted_count = self._duplicate_count = 0
  101. def set_notification_scheduler(self, scheduler: Optional[NotificationScheduler]) -> None:
  102. self._notification_scheduler = scheduler
  103. def _source_allowlist_required_but_missing(self) -> bool:
  104. # host=None binds all interfaces (both families) — network-accessible.
  105. host_is_public = self._host is None or is_network_accessible(self._host)
  106. return host_is_public and not self._allowed_source_networks
  107. async def connect(self, *, is_reconnect: bool = False) -> bool:
  108. if self._client_state is None:
  109. logger.error("[msgraph_webhook] Refusing to start without extra.client_state configured")
  110. return False
  111. if self._source_allowlist_required_but_missing():
  112. logger.error("[msgraph_webhook] Refusing to start: binding to %s requires extra.allowed_source_cidrs. "
  113. "Configure the Microsoft Graph source CIDRs or bind to loopback (127.0.0.1/::1) behind a "
  114. "tunnel or reverse proxy.", self._host)
  115. return False
  116. app = web.Application(client_max_size=self._max_body_bytes)
  117. app.router.add_get(self._health_path, self._handle_health)
  118. app.router.add_get(self._webhook_path, self._handle_validation)
  119. app.router.add_post(self._webhook_path, self._handle_notification)
  120. # Plugin-registered native routes; wired before AppRunner.setup() freezes the router.
  121. self._wire_plugin_handlers(app)
  122. # Shared-listener mode (multiplex secondary): no bind; served at /p/<profile>/<webhook_path>.
  123. from gateway.platforms.shared_ingress import bind_listener
  124. self._runner = await bind_listener(self, app, self._host, self._port, self._webhook_path)
  125. self._mark_connected()
  126. if self._runner is not None:
  127. logger.info("[msgraph_webhook] Listening on %s:%d%s", self._host, self._port, self._webhook_path)
  128. return True
  129. async def disconnect(self) -> None:
  130. if self._runner is not None:
  131. await self._runner.cleanup()
  132. self._runner = None
  133. self._mark_disconnected()
  134. async def send(self, chat_id: str, content: str, reply_to: Optional[str] = None,
  135. metadata: Optional[dict[str, Any]] = None) -> SendResult:
  136. logger.info("[msgraph_webhook] Response for %s: %s", chat_id, content[:200])
  137. return SendResult(success=True)
  138. async def get_chat_info(self, chat_id: str) -> dict[str, Any]:
  139. return {"name": chat_id, "type": "webhook"}
  140. async def _handle_health(self, request: web.Request) -> web.Response:
  141. if not self._source_ip_allowed(request):
  142. return web.Response(status=403)
  143. return web.json_response({
  144. "status": "ok", "platform": self.platform.value, "webhook_path": self._webhook_path,
  145. "accepted": self._accepted_count, "duplicates": self._duplicate_count})
  146. async def _handle_validation(self, request: web.Request) -> web.Response:
  147. """Graph subscription validation handshake: echo ``validationToken`` verbatim as text/plain. Bare GETs
  148. are rejected so the endpoint can't be enumerated."""
  149. if not self._source_ip_allowed(request):
  150. return web.Response(status=403)
  151. if not (validation_token := request.query.get("validationToken", "")):
  152. return web.Response(status=400)
  153. return web.Response(text=validation_token, content_type="text/plain")
  154. async def _ingest_notification(self, raw_notification: Any) -> str:
  155. """Classify + schedule one notification: 'accepted' | 'duplicate' | 'auth' | 'other'."""
  156. if not isinstance(raw_notification, dict):
  157. return "other"
  158. notification = dict(raw_notification)
  159. if not self._resource_accepted(str(notification.get("resource") or "")):
  160. return "other"
  161. if not self._verify_client_state(notification):
  162. # Bad clientState is an auth failure: a fully forged batch gets 403 so the sender stops
  163. # retrying; legitimate Graph retries carry a valid clientState → accepted/duplicate paths.
  164. return "auth"
  165. receipt_key = f"id:{explicit_id}" if (explicit_id := str(notification.get("id") or "").strip()) else None
  166. if self._notification_scheduler is not None and receipt_key in self._seen_receipts:
  167. return "duplicate"
  168. await self._schedule_notification(notification, self._build_message_event(notification, receipt_key))
  169. if receipt_key is not None:
  170. self._remember_receipt(receipt_key)
  171. self._accepted_count += 1
  172. return "accepted"
  173. async def _handle_notification(self, request: web.Request) -> web.Response:
  174. if not self._source_ip_allowed(request):
  175. return web.Response(status=403)
  176. # Graph never sends validationToken on POST, but tolerate clients replaying it in-band.
  177. if validation_token := request.query.get("validationToken", ""):
  178. return web.Response(text=validation_token, content_type="text/plain")
  179. status, notifications = await self._read_notifications(request)
  180. if status:
  181. return web.Response(status=status)
  182. counts = {"accepted": 0, "duplicate": 0, "auth": 0, "other": 0}
  183. for raw_notification in notifications:
  184. try:
  185. counts[await self._ingest_notification(raw_notification)] += 1
  186. except Exception:
  187. logger.exception("[msgraph_webhook] Notification admission failed")
  188. return web.Response(status=503)
  189. self._duplicate_count += counts["duplicate"]
  190. # Anything ingested OR deduped → 202 with empty body (Graph acks; no counter leak). Every item
  191. # failed auth → 403 so forged POSTs get a clear reject. Otherwise (malformed / not accepted) → 400.
  192. if counts["accepted"] or counts["duplicate"]:
  193. return web.Response(status=202)
  194. if counts["auth"] and not counts["other"]:
  195. return web.Response(status=403)
  196. return web.Response(status=400)
  197. async def _read_notifications(self, request: web.Request) -> tuple[int, list]:
  198. """Read and validate the POST body; returns (error_status, []) or (0, notifications)."""
  199. try:
  200. content_length = request.content_length
  201. except Exception:
  202. content_length = None
  203. if content_length is not None and content_length > self._max_body_bytes:
  204. return 413, []
  205. try:
  206. raw_body = await request.read()
  207. except Exception:
  208. return 400, []
  209. if len(raw_body) > self._max_body_bytes:
  210. return 413, []
  211. try:
  212. body = json.loads(raw_body.decode("utf-8"))
  213. except (json.JSONDecodeError, UnicodeDecodeError):
  214. return 400, []
  215. notifications = body.get("value") if isinstance(body, dict) else None
  216. return (0, notifications) if isinstance(notifications, list) else (400, [])
  217. def _source_ip_allowed(self, request: web.Request) -> bool:
  218. """Loopback-only binds may omit ``allowed_source_cidrs`` (local proxies, dev tunnels);
  219. network-accessible binds fail closed without one."""
  220. if self._source_allowlist_required_but_missing():
  221. return False
  222. if not self._allowed_source_networks:
  223. return True
  224. try:
  225. peer_addr = ipaddress.ip_address(request.remote or "")
  226. except ValueError:
  227. return False
  228. return any(peer_addr in network for network in self._allowed_source_networks)
  229. def _resource_accepted(self, resource: str) -> bool:
  230. if not self._accepted_resources:
  231. return True
  232. resource = resource.strip().strip("/")
  233. for pattern in (p.strip().strip("/") for p in self._accepted_resources):
  234. if pattern.endswith("*"):
  235. pattern = pattern[:-1].rstrip("/")
  236. if pattern and _prefix_match(resource, pattern):
  237. return True
  238. return False
  239. def _verify_client_state(self, notification: dict[str, Any]) -> bool:
  240. """Timing-safe compare of the Graph-supplied clientState against the configured shared secret
  241. (``openssl rand -hex 32`` in the setup guide)."""
  242. expected = self._client_state
  243. provided = _string_or_none(notification.get("clientState"))
  244. if expected is None or provided is None:
  245. return False
  246. # Compare as bytes: compare_digest raises TypeError on non-ASCII str (clientState is request-controlled).
  247. return hmac.compare_digest(provided.encode(), expected.encode())
  248. def _remember_receipt(self, receipt_key: str) -> None:
  249. self._seen_receipts.add(receipt_key)
  250. self._seen_receipt_order.append(receipt_key)
  251. while len(self._seen_receipt_order) > self._max_seen_receipts:
  252. self._seen_receipts.discard(self._seen_receipt_order.popleft())
  253. def _build_message_event(self, notification: dict[str, Any], receipt_key: Optional[str]) -> MessageEvent:
  254. message_id = receipt_key or f"sha1:{sha1(json.dumps(notification, sort_keys=True).encode('utf-8')).hexdigest()}"
  255. source = self.build_source(
  256. chat_id=f"msgraph:{notification.get('subscriptionId', 'unknown')}", chat_name="msgraph/webhook",
  257. chat_type="webhook", user_id="msgraph", user_name="Microsoft Graph")
  258. return MessageEvent(
  259. text=self._render_prompt(notification), message_type=MessageType.TEXT, source=source,
  260. raw_message=notification, message_id=message_id, internal=True)
  261. def _render_prompt(self, notification: dict[str, Any]) -> str:
  262. template = self.config.extra.get("prompt", "")
  263. if template:
  264. return _render_template(template, {
  265. "notification": notification, "resource": notification.get("resource", ""),
  266. "change_type": notification.get("changeType", ""),
  267. "subscription_id": notification.get("subscriptionId", "")})
  268. rendered = json.dumps(notification, indent=2, sort_keys=True)[:4000]
  269. return f"Microsoft Graph change notification:\n\n```json\n{rendered}\n```"
  270. async def _schedule_notification(self, notification: dict[str, Any], event: MessageEvent) -> None:
  271. scheduler = self._notification_scheduler
  272. if scheduler is not None:
  273. result = scheduler(notification, event)
  274. if asyncio.iscoroutine(result):
  275. await result
  276. return # Plugin-owned pipeline; never also start a gateway inference.
  277. await self._admit_notification(event)
  278. @property
  279. def token(self):
  280. return self._client_state
  281. async def _admit_notification(self, event):
  282. # Only _ingest_notification reaches this after resource + clientState gates;
  283. # the HTTP handler additionally owns source-IP and body-size authorization.
  284. # Reuse the existing private automation envelope: public native admission
  285. # continues to reject internal events and arbitrary metadata.
  286. from datetime import datetime, timezone
  287. from gateway.platforms.webhook_ingress import producer_scope
  288. from gateway.session_ingress_context import capture_provenance
  289. from hermes_state_runtime import admit_session_input, RuntimeStoreError
  290. async with producer_scope(self, event) as authority:
  291. authority._require_admission_open()
  292. provenance = capture_provenance(authority.runner, event)
  293. if provenance is None:
  294. raise RuntimeStoreError('permission_denied')
  295. # The routing reservation writes too: off the owner loop like the admission below.
  296. ref = authority._publish_route(event.source, *await asyncio.to_thread(authority._reserve_route, event.source))
  297. route = authority.sessions[ref.session_id].route
  298. source = event.source.to_dict()
  299. source['is_bot'] = event.source.is_bot
  300. envelope = {'source': source, 'route': route, 'provenance': provenance,
  301. 'timestamp': datetime.fromtimestamp(0, UTC).isoformat(),
  302. 'event': {'message_id': event.message_id},
  303. 'automation': {'identity': event.message_id, 'owner': ref.session_id}}
  304. # Off the owner loop and in admission order (``tracked_write``), committed before the
  305. # 202: a held SQLite writer must not freeze every session while Graph waits.
  306. from functools import partial
  307. from gateway.session_runtime_workers import tracked_write
  308. return await tracked_write(authority, partial(
  309. admit_session_input, authority.db, epoch=authority.epoch,
  310. principal_id='msgraph:' + route, session_id=ref.session_id, request_id=event.message_id,
  311. payload={'text': event.text, 'native_text_v1': envelope},
  312. _authorize_write=authority._admission_gate()),
  313. then=partial(authority._admitted, ref, event), ordered=True,
  314. after=partial(authority._schedule_admitted, ref))