__init__.py 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852
  1. """Relay/connector support package for the Hermes gateway.
  2. EXPERIMENTAL gateway side of the "Gateway Gateway" relay design: a generic
  3. ``RelayAdapter`` plus the wire-serializable ``CapabilityDescriptor`` the connector
  4. hands it at handshake, and the production ``WebSocketRelayTransport``. The public
  5. API MAY CHANGE without a deprecation cycle until >=2 real Class-1 platforms have
  6. shaken out the schema (``website/docs/developer-guide/relay-connector-contract.md``). Activation is
  7. config-driven: the relay platform is registered when a connector relay URL is set
  8. (``GATEWAY_RELAY_URL`` env or ``gateway.relay_url``), like ``gateway.proxy_url``,
  9. unless the effective relay platform configuration explicitly disables it.
  10. """
  11. from __future__ import annotations
  12. import asyncio
  13. import contextlib
  14. import contextvars
  15. import json
  16. import logging
  17. import os
  18. import re
  19. import socket
  20. import urllib.error
  21. import urllib.parse
  22. import urllib.request
  23. from typing import Callable, Optional
  24. logger = logging.getLogger("gateway.relay")
  25. # Shape gate for ambient-endpoint token bodies (mode 1b in
  26. # _resolve_relay_identity_token): a JWT (3+ dotted segments) or a long opaque
  27. # base64url token (>= 32 chars). Short bare words — 'unauthorized', 'error',
  28. # 'null' — match the alphabet but are plain-text error bodies and must fail closed.
  29. _AMBIENT_TOKEN_SHAPE = re.compile(
  30. r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+){2,}" # JWT-like: 3+ dotted segments
  31. r"|[A-Za-z0-9_-]{32,}" # long opaque bearer token
  32. )
  33. _HTTP_TIMEOUT_S = 15.0
  34. # Boot self-provision retry (a relay configured but no credentials obtained). Delays
  35. # before each retry; the last one repeats until credentials exist. The first retry
  36. # lands quickly because the common cause is a blip (proxy, DNS, connector restart);
  37. # the cap matches the reconnect watcher's 300s cap so recovery is never slower than
  38. # the 4401 loop it replaces.
  39. _PROVISION_RETRY_DELAYS_S = (5.0, 30.0, 120.0, 300.0)
  40. # A refusal the connector authored WITH a reason (identity not confirmed, binding owned
  41. # elsewhere, withheld secret) changes only when an operator acts, so it is retried
  42. # less often than a blip: bounded log noise and NAS introspection load, yet the
  43. # gateway still heals on its own within minutes of the fix.
  44. _PROVISION_REFUSAL_RETRY_S = 600.0
  45. # self-provision outcomes (``_self_provision``)
  46. _PROVISIONED, _SKIPPED, _REFUSED, _FAILED = "provisioned", "skipped", "refused", "failed"
  47. class ProvisionRefused(RuntimeError):
  48. """The connector refused ``/relay/provision`` with its own reason (a 4xx JSON
  49. ``{"error"}``): definitive until something changes on the connector/NAS side."""
  50. # gateway.idp.* keys (and GATEWAY_RELAY_IDP_<KEY> env mirrors) read by the identity resolver.
  51. _IDP_KEYS = ("token_url", "client_id", "client_secret", "scope")
  52. # ─────────────────────────── config access ───────────────────────────
  53. def _load_cfg() -> dict:
  54. """The full gateway config, or ``{}`` — config absence/parse must never crash boot."""
  55. try:
  56. from gateway.run import _load_gateway_config # late import to avoid cycle
  57. cfg = _load_gateway_config()
  58. except Exception:
  59. return {}
  60. return cfg if isinstance(cfg, dict) else {}
  61. def _gateway_cfg() -> dict:
  62. """The ``gateway:`` block of config.yaml, or ``{}``."""
  63. block = _load_cfg().get("gateway")
  64. return block if isinstance(block, dict) else {}
  65. def _launch_profile_scope():
  66. """The launch profile's runtime scope once the gateway multiplexes, ``nullcontext`` before.
  67. Self-provision runs at startup, before any turn binds a profile, and under multiplexing the
  68. Nous Portal and inference routing overrides only resolve inside a profile scope. The scope is
  69. built from the profile's files like a turn's (``gateway.run._profile_runtime_scope``), not via
  70. ``launch_profile_scope_if_multiplexed``, whose first entry freezes the launch env snapshot
  71. before startup has written ``GATEWAY_RELAY_*`` into ``os.environ``.
  72. """
  73. from agent.secret_scope import is_multiplex_active
  74. if not is_multiplex_active():
  75. return contextlib.nullcontext()
  76. from gateway.run import _profile_runtime_scope # late import to avoid cycle
  77. from hermes_constants import get_process_hermes_home
  78. return _profile_runtime_scope(get_process_hermes_home())
  79. def _env_or_cfg(env_var: str, cfg_key: str) -> str:
  80. """Env var first (Docker/NAS stamp), then ``gateway.<cfg_key>``; stripped, ``""`` when unset."""
  81. value = os.environ.get(env_var, "").strip()
  82. return value or str(_gateway_cfg().get(cfg_key, "") or "").strip()
  83. def _env_or_cfg_url(env_var: str, cfg_key: str) -> Optional[str]:
  84. """``_env_or_cfg`` for URL-shaped values: trailing slash stripped, None when unset."""
  85. return _env_or_cfg(env_var, cfg_key).rstrip("/") or None
  86. def relay_explicitly_disabled() -> bool:
  87. """``platforms.relay.enabled: false`` in the profile's YAML (user or managed).
  88. Same files, merge and boolean normalization as the gateway loader (a malformed user
  89. file drops the whole YAML layer there too), minus its env/plugin side effects, so a
  90. standalone scheduler can ask without bootstrapping the gateway.
  91. Mirrors the loader's ``_enabled_explicit`` rule: only a YAML ``enabled`` key is
  92. authoritative — a legacy ``gateway.json`` block is advisory for relay exactly as it
  93. is for every other platform, and an absent key keeps URL-only activation.
  94. """
  95. from gateway.config import Platform, PlatformConfig
  96. from gateway.config_loader import bridge_platform_shared_keys, merge_platform_sections, read_yaml_layers
  97. from hermes_constants import get_hermes_home
  98. try:
  99. cfg = read_yaml_layers(get_hermes_home())
  100. except Exception:
  101. return False
  102. if not isinstance(cfg, dict):
  103. return False
  104. gateway = cfg.get("gateway") if isinstance(cfg.get("gateway"), dict) else {}
  105. platforms = merge_platform_sections(cfg, gateway, {})
  106. bridge_platform_shared_keys(cfg, gateway.get("platforms"), {}, platforms, [Platform.RELAY])
  107. block = platforms.get("relay")
  108. if not isinstance(block, dict) or not block.get("extra", {}).get("_enabled_explicit"):
  109. return False
  110. return not PlatformConfig.from_dict(block).enabled
  111. def relay_url() -> Optional[str]:
  112. """Effective connector URL; an explicit platform disable vetoes even an env URL."""
  113. if relay_explicitly_disabled():
  114. return None
  115. return _env_or_cfg_url("GATEWAY_RELAY_URL", "relay_url")
  116. def relay_platform_identities() -> list[tuple[str, str]]:
  117. """The ordered (platform, bot_id) pairs this gateway fronts on one WS connection,
  118. from ``GATEWAY_RELAY_PLATFORMS`` (comma-sep) and ``GATEWAY_RELAY_BOT_IDS`` (JSON
  119. ``{"discord": {"botId": "..."}}``). The FIRST pair is the handshake/descriptor
  120. default; a platform absent from the ids map gets an empty bot_id (the connector
  121. rejects it with a structured failure). ``[("relay", "")]`` when nothing is set."""
  122. platforms = [p.strip() for p in os.environ.get("GATEWAY_RELAY_PLATFORMS", "").split(",") if p.strip()]
  123. if not platforms:
  124. return [("relay", "")]
  125. ids = _relay_bot_ids_map()
  126. out: list[tuple[str, str]] = []
  127. for platform in platforms:
  128. entry = ids.get(platform) or {}
  129. bot_id = str(entry.get("botId", "")).strip() if isinstance(entry, dict) else ""
  130. out.append((platform, bot_id))
  131. return out
  132. def relay_fronted_platforms() -> set[str]:
  133. """Logical platform names the relay fronts, minus the generic ``relay`` fallback.
  134. Same env source the live adapter's identity set comes from, so config-time
  135. validation (cron delivery preflight) and fire-time routing can never disagree —
  136. and it needs no live adapter, so a standalone scheduler can use it."""
  137. if relay_explicitly_disabled():
  138. return set()
  139. return {p for p, _ in relay_platform_identities() if p != "relay"}
  140. def _relay_bot_ids_map() -> dict:
  141. """Parse ``GATEWAY_RELAY_BOT_IDS``; a malformed map yields ``{}`` rather than crashing boot."""
  142. raw = os.environ.get("GATEWAY_RELAY_BOT_IDS", "").strip()
  143. if not raw:
  144. return {}
  145. try:
  146. parsed = json.loads(raw)
  147. return parsed if isinstance(parsed, dict) else {}
  148. except Exception:
  149. logger.warning("GATEWAY_RELAY_BOT_IDS is not valid JSON; treating as empty")
  150. return {}
  151. def relay_platform_identity() -> tuple[str, str]:
  152. """The PRIMARY (platform, bot_id) — first of ``relay_platform_identities()``."""
  153. return relay_platform_identities()[0]
  154. def relay_connection_auth() -> tuple[Optional[str], Optional[str]]:
  155. """The (gateway_id, upgrade_secret) from enrollment (``GATEWAY_RELAY_ID`` /
  156. ``GATEWAY_RELAY_SECRET``, then ``gateway.relay_id`` / ``gateway.relay_secret``).
  157. Either absent -> ``(None, None)`` and the transport dials unauthenticated."""
  158. gateway_id = _env_or_cfg("GATEWAY_RELAY_ID", "relay_id")
  159. secret = _env_or_cfg("GATEWAY_RELAY_SECRET", "relay_secret")
  160. return (gateway_id or None, secret or None)
  161. def relay_endpoint() -> Optional[str]:
  162. """The gateway's own PUBLIC inbound URL, asserted to the connector at provision
  163. (``GATEWAY_RELAY_ENDPOINT`` / ``gateway.relay_endpoint``). Stored on the tenant's
  164. route rows; gateway-asserted but tenant-scoped, so a dishonest gateway can only
  165. misdirect its OWN inbound. Absent -> outbound-only (no inbound routes written)."""
  166. return _env_or_cfg_url("GATEWAY_RELAY_ENDPOINT", "relay_endpoint")
  167. def relay_route_keys() -> list[str]:
  168. """Discriminators (scope_ids / chat_ids / paths) this gateway's tenant owns. The
  169. connector writes one route row per (routeKey -> tenant, endpoint), so keys only
  170. take effect alongside ``relay_endpoint()``. ``GATEWAY_RELAY_ROUTE_KEYS`` is
  171. comma-separated; ``gateway.relay_route_keys`` may be a list or a comma string."""
  172. raw = os.environ.get("GATEWAY_RELAY_ROUTE_KEYS", "").strip()
  173. if not raw:
  174. val = _gateway_cfg().get("relay_route_keys", "")
  175. if isinstance(val, (list, tuple)):
  176. return [str(k).strip() for k in val if str(k).strip()]
  177. raw = str(val or "").strip()
  178. return [k.strip() for k in raw.split(",") if k.strip()]
  179. def relay_instance_id() -> Optional[str]:
  180. """Stable per-instance id forwarded at provision (``GATEWAY_RELAY_INSTANCE_ID`` /
  181. ``gateway.relay_instance_id``): binds the connector's ``gatewayId -> instanceId``
  182. so inbound routes per-instance rather than tenant-broadcast (NAS stamps its
  183. ``AgentInstance.id``). Tenant-scoped like ``relay_endpoint()``; absent -> null."""
  184. return _env_or_cfg("GATEWAY_RELAY_INSTANCE_ID", "relay_instance_id") or None
  185. def relay_wake_url() -> Optional[str]:
  186. """The gateway's WAKE URL forwarded at provision (``GATEWAY_RELAY_WAKE_URL`` /
  187. ``gateway.relay_wake_url``): a payload-free poke the connector GETs when a
  188. going-idle destination receives its first buffered event, so a suspended gateway
  189. wakes, reconnects and drains. Absent -> no wake (buffering still works)."""
  190. return _env_or_cfg_url("GATEWAY_RELAY_WAKE_URL", "relay_wake_url")
  191. def relay_display_name() -> Optional[str]:
  192. """The human-facing agent display name forwarded at provision — the connector's
  193. multi-agent reply-attribution prefix (``**<displayName>:** ``).
  194. ``GATEWAY_RELAY_DISPLAY_NAME`` env, then the skin's branded agent name (a skin
  195. rename propagates on the next boot). Absent -> connector's linked-owner fallback.
  196. The PRIMARY source for the connector's multi-agent reply-attribution prefix (gateway-gateway #171): in a
  197. multi-agent scope the shared bot prepends ``**<displayName>:** `` to this instance's replies.
  198. Gateway-asserted but safely scoped exactly like ``relay_instance_id()`` / ``relay_wake_url()`` — the
  199. tenant stays token-verified, so a dishonest gateway can only label its OWN instance. Absent -> the
  200. connector stores null and attribution falls back to the instance's linked-owner identity, else skips the
  201. prefix.
  202. """
  203. value = os.environ.get("GATEWAY_RELAY_DISPLAY_NAME", "").strip()
  204. if not value:
  205. try:
  206. from hermes_cli.skin_engine import get_active_skin # late import: boot-safe
  207. value = str(get_active_skin().get_branding("agent_name", "") or "").strip()
  208. except Exception:
  209. value = ""
  210. # The stock brand is identical on every default install: forwarding it would
  211. # prefix every reply "**Hermes Agent:**" and shadow the connector's
  212. # linked-owner fallback, which actually disambiguates. Only a customized
  213. # name is forwarded.
  214. if value == "Hermes Agent":
  215. value = ""
  216. # Mirror the connector's ingest sanitization (trim + 64-char cap).
  217. return value[:64] or None
  218. # ─────────────────────────── connector HTTP ───────────────────────────
  219. def _connector_url(relay_dial_url: str, path: str) -> str:
  220. """``ws(s)://…/relay`` dial URL -> ``http(s)://…{path}`` connector route."""
  221. from gateway.relay.media import media_base_url
  222. return f"{media_base_url(relay_dial_url)}{path}"
  223. def _provision_url(relay_dial_url: str) -> str:
  224. return _connector_url(relay_dial_url, "/relay/provision")
  225. def _json_post(url: str, token: str, body: dict, timeout: float):
  226. """Bearer-authenticated JSON POST; returns the open response (caller reads it)."""
  227. req = urllib.request.Request(
  228. url,
  229. data=json.dumps(body).encode("utf-8"),
  230. method="POST",
  231. headers={
  232. "Authorization": f"Bearer {token}",
  233. "Content-Type": "application/json",
  234. "Accept": "application/json",
  235. },
  236. )
  237. return urllib.request.urlopen(req, timeout=timeout)
  238. def relay_relevance_policy(platform: Optional[str] = None) -> Optional[dict]:
  239. """Project a fronted platform's RELEVANCE config into the connector's generic vocabulary.
  240. The connector's relevance gate reasons over a platform-agnostic policy keyed by
  241. ``(tenant, platform, instanceId)``: ``requireAddress`` <- ``require_mention``,
  242. ``freeResponseScopes`` <- ``free_response_channels``, ``allowOtherBots`` <-
  243. ``{PLATFORM}_ALLOW_BOTS`` in {"mentions","all"}. Read from the platform's config
  244. block (``discord:``), falling back to the bridged top-level keys, then env.
  245. ``platform`` defaults to the PRIMARY fronted platform. Returns None when relay
  246. isn't configured or nothing is CONFIGURED to declare, so the connector's default
  247. (mention-gated) applies. The condition is "require_mention is unset", NOT "falsy":
  248. an EXPLICIT ``require_mention: false`` is a non-default choice that MUST be
  249. declared or the connector would mention-gate an agent configured to free-respond.
  250. """
  251. if platform is None:
  252. platform, _bot_id = relay_platform_identity()
  253. if not platform or platform == "relay":
  254. return None
  255. require_mention = None
  256. free_response: list[str] = []
  257. try:
  258. cfg = _load_cfg()
  259. # Platform block lookup order: top-level ``<platform>:``, then
  260. # ``gateway.platforms.<platform>``, then ``platforms.<platform>``.
  261. def _candidates():
  262. yield cfg.get(platform)
  263. gw_platforms = (cfg.get("gateway") or {}).get("platforms") or {}
  264. yield gw_platforms.get(platform) if isinstance(gw_platforms, dict) else None
  265. yield (cfg.get("platforms") or {}).get(platform)
  266. plat_cfg = next((c for c in _candidates() if isinstance(c, dict)), {})
  267. if "require_mention" in plat_cfg:
  268. require_mention = plat_cfg.get("require_mention")
  269. elif cfg.get("require_mention") is not None:
  270. require_mention = cfg.get("require_mention")
  271. frc = plat_cfg.get("free_response_channels")
  272. if frc is None:
  273. frc = cfg.get("free_response_channels")
  274. if isinstance(frc, (list, tuple)):
  275. free_response = [str(c).strip() for c in frc if str(c).strip()]
  276. elif isinstance(frc, str) and frc.strip():
  277. free_response = [c.strip() for c in frc.split(",") if c.strip()]
  278. except Exception:
  279. pass
  280. # Same gate as the gateway's own authz_mixin DISCORD_ALLOW_BOTS bypass.
  281. allow_bots_env = os.environ.get(f"{platform.upper()}_ALLOW_BOTS", "").lower().strip()
  282. allow_other_bots = allow_bots_env in {"mentions", "all"}
  283. if require_mention is None and not free_response and not allow_other_bots:
  284. return None
  285. return {
  286. "platform": platform,
  287. "requireAddress": bool(require_mention),
  288. "freeResponseScopes": free_response,
  289. "allowOtherBots": allow_other_bots,
  290. }
  291. def _post_provision(
  292. *,
  293. provision_url: str,
  294. access_token: str,
  295. gateway_id: str,
  296. platform: str,
  297. bot_id: str,
  298. gateway_endpoint: Optional[str],
  299. route_keys: list[str],
  300. instance_id: Optional[str] = None,
  301. wake_url: Optional[str] = None,
  302. display_name: Optional[str] = None,
  303. timeout: float = _HTTP_TIMEOUT_S,
  304. ) -> dict:
  305. """POST ``/relay/provision``; return ``{secret, deliveryKey, tenant, gatewayId,
  306. routeKeys}`` (the connector validates ``access_token`` against NAS, derives the
  307. tenant, mints the per-gateway secret and upserts route rows). Raises RuntimeError
  308. with a user-facing message on any non-2xx / transport failure."""
  309. body: dict = {
  310. "gatewayId": gateway_id,
  311. "platform": platform,
  312. "botId": bot_id,
  313. "gatewayEndpoint": gateway_endpoint or "",
  314. "routeKeys": route_keys,
  315. }
  316. # Optional fields are OMITTED when absent so the connector stores null
  317. # (back-compat) rather than binding an empty string.
  318. for key, value in (("instanceId", instance_id), ("wakeUrl", wake_url), ("displayName", display_name)):
  319. if value:
  320. body[key] = value
  321. try:
  322. with _json_post(provision_url, access_token, body, timeout) as resp:
  323. payload = json.loads(resp.read().decode())
  324. except urllib.error.HTTPError as exc:
  325. from gateway.relay import http_diagnostics as diag
  326. err_body = diag.read_error_body(exc)
  327. # Never log the identity token we sent, even if a hop echoes it back bare.
  328. detail = diag.redact_excerpt(diag.connector_reason(err_body), (access_token,))
  329. if detail and 400 <= exc.code < 500 and exc.code not in (408, 429) and not diag.answered_by_proxy(exc):
  330. # The connector itself refused, with a reason: retrying soon will not change it.
  331. raise ProvisionRefused(f"connector returned HTTP {exc.code}: {detail}") from exc
  332. # No connector reason (a proxy's page, JSON or not) or a 5xx/408/429: name whoever answered.
  333. described = diag.describe_http_error(exc, err_body, secrets=(access_token,))
  334. raise RuntimeError(described + (f": {detail}" if detail else "")) from exc
  335. except urllib.error.URLError as exc:
  336. raise RuntimeError(f"could not reach connector: {exc.reason}") from exc
  337. if not isinstance(payload, dict):
  338. raise RuntimeError("connector returned an unexpected response")
  339. # A response WITHOUT a secret is valid, not an error (connector F-004).
  340. #
  341. # `/relay/provision` used to hand the stored per-gateway secret back on every
  342. # replay, to anyone who reached the endpoint with the right gatewayId. The
  343. # connector now returns credential material only to a caller that proves
  344. # ownership or possession, and answers `secretIssued: false` otherwise — with
  345. # `secret`/`deliveryKey` absent rather than empty.
  346. #
  347. # Two legitimate shapes reach here, and neither is a failure:
  348. # * the SECOND and later platforms of a multi-platform boot. All platforms
  349. # share one gatewayId, so the first POST mints the secret and the rest are
  350. # replays of an unchanged binding. Raising here aborted every platform
  351. # after the first.
  352. # * a re-provision by a caller that legitimately no longer holds the secret;
  353. # `POST /relay/rotate` is the supported recovery path.
  354. #
  355. # Treat "no secret" as a value, and let the caller decide — it knows whether
  356. # it already holds credentials. Only a malformed body is an error.
  357. if not payload.get("secret") and payload.get("secretIssued") is not False:
  358. # Fail closed on the discriminator itself. The ONLY credential-less shape
  359. # the connector emits is literal JSON `false`. No key at all is a
  360. # pre-F-004 connector (the old unexpected-response case); `true` with no
  361. # secret, or any non-boolean value, is a malformed body. Accepting those
  362. # would mark the platform provisioned with no credential in hand.
  363. raise RuntimeError("connector returned an unexpected response (no secret)")
  364. return payload
  365. def _resolve_relay_identity_token() -> str:
  366. """Resolve the caller-identity bearer token the connector introspects to a tenant.
  367. Canonical resolver shared by runtime self-provision and ``hermes gateway enroll``.
  368. Modes, in precedence order:
  369. 1. Generic OIDC client-credentials (self-hosted IdP): ``gateway.idp.token_url``
  370. (``GATEWAY_RELAY_IDP_TOKEN_URL``) set together with client id + secret ->
  371. POST the OAuth2 ``client_credentials`` grant.
  372. 1b. Ambient token endpoint: ``token_url`` with NEITHER client_id nor
  373. client_secret is a metadata-server-style endpoint (e.g. Domino's
  374. ``$DOMINO_API_PROXY/access-token``): a plain GET whose body IS the token,
  375. raw JWT or a JSON envelope with ``access_token``. Possession of the
  376. (typically loopback) endpoint is the credential.
  377. 2. Nous Portal (default): ``resolve_nous_access_token()``.
  378. Raises on failure; callers decide whether that's fatal (enroll CLI) or a graceful
  379. boot no-op (self-provision).
  380. """
  381. env = {k: os.environ.get(f"GATEWAY_RELAY_IDP_{k.upper()}", "").strip() for k in _IDP_KEYS}
  382. if not env["token_url"]:
  383. # Env token_url absent -> the whole idp block comes from config (env id/secret
  384. # still win per key). A malformed gateway.idp degrades to "no token_url".
  385. idp = _gateway_cfg().get("idp")
  386. idp = idp if isinstance(idp, dict) else {}
  387. for k in _IDP_KEYS:
  388. env[k] = env[k] or str(idp.get(k, "") or "").strip()
  389. token_url, client_id, client_secret, scope = (env[k] for k in _IDP_KEYS)
  390. if not token_url:
  391. from hermes_cli.auth import resolve_nous_access_token
  392. # Only the Nous refresh is scoped: the gateway config read above and the IdP modes below
  393. # keep resolving ``${VAR}`` refs from the launch env, as they always have.
  394. with _launch_profile_scope():
  395. return resolve_nous_access_token()
  396. if not client_id and not client_secret:
  397. # Mode 1b — plain GET; the body is the token, raw or JSON-enveloped.
  398. req = urllib.request.Request(
  399. token_url, method="GET", headers={"Accept": "application/json, text/plain"}
  400. )
  401. with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT_S) as resp:
  402. body = resp.read().decode().strip()
  403. token = ""
  404. if body.startswith("{"):
  405. try:
  406. envelope_token = (json.loads(body) or {}).get("access_token")
  407. except ValueError:
  408. envelope_token = None
  409. # No shape gate on an envelope: it is a deliberate token response, and
  410. # opaque tokens may use the standard-base64 alphabet the raw gate rejects.
  411. if isinstance(envelope_token, str):
  412. token = envelope_token.strip()
  413. elif _AMBIENT_TOKEN_SHAPE.fullmatch(body):
  414. token = body
  415. if not token:
  416. raise RuntimeError(
  417. "no client_id/client_secret configured, so gateway.idp.token_url was "
  418. "treated as an ambient token endpoint (GET), but the response body "
  419. "was not a token. For the OAuth2 client_credentials grant, configure "
  420. "client_id and client_secret alongside token_url."
  421. )
  422. return token
  423. if not client_id or not client_secret:
  424. # Exactly one credential: a mistyped client_credentials setup, not an
  425. # ambient endpoint. Fail loud; never GET the IdP.
  426. missing = "client_secret" if client_id else "client_id"
  427. raise RuntimeError(
  428. f"gateway.idp.token_url is configured with a partial client credential "
  429. f"({missing} missing). Configure both client_id and client_secret for "
  430. f"the OAuth2 client_credentials grant, or neither to treat token_url "
  431. f"as an ambient token endpoint (plain GET returning the token)."
  432. )
  433. # Mode 1 — OAuth2 client_credentials grant.
  434. form = {"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret}
  435. if scope:
  436. form["scope"] = scope
  437. req = urllib.request.Request(
  438. token_url, data=urllib.parse.urlencode(form).encode("utf-8"), method="POST",
  439. headers={"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json"},
  440. )
  441. with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT_S) as resp:
  442. payload = json.loads(resp.read().decode())
  443. access_token = (payload or {}).get("access_token")
  444. if not isinstance(access_token, str) or not access_token.strip():
  445. raise RuntimeError("IdP client_credentials response had no access_token")
  446. return access_token.strip()
  447. def self_provision_relay() -> bool:
  448. """Boot-time relay self-provision: mint relay creds in-process, no human, no disk.
  449. Fires when ``relay_url()`` is set and NO per-gateway secret is pinned: resolves the
  450. identity token, POSTs ``/relay/provision`` for EACH fronted platform and sets
  451. ``GATEWAY_RELAY_ID`` / ``_SECRET`` / ``_DELIVERY_KEY`` in ``os.environ`` for
  452. ``register_relay_adapter()``. Creds live ONLY in process memory (never
  453. ``~/.hermes/.env``), so a hosted container re-provisions every boot; the
  454. connector's rotation window covers a still-connected prior instance. The trigger
  455. is deliberately NOT ``is_managed()`` (False on a NAS-hosted Fly agent): "pointed
  456. at a connector without a pinned secret" is the real signal and self-guards (an
  457. enrolled gateway has a PINNED secret -> skipped; no resolvable identity -> no-op).
  458. Returns True iff it provisioned. NEVER raises: a failure logs and returns False so
  459. the gateway still boots; ``retry_self_provision`` then keeps trying in the
  460. background (the adapter meanwhile dials unauthenticated / is rejected).
  461. """
  462. return _self_provision() == _PROVISIONED
  463. def relay_provision_pending() -> bool:
  464. """A relay is configured but this process holds no per-gateway secret yet."""
  465. if not relay_url():
  466. return False
  467. existing_id, existing_secret = relay_connection_auth()
  468. return not (existing_id and existing_secret)
  469. def _self_provision() -> str:
  470. """``self_provision_relay`` with the outcome kept: provisioned / skipped (nothing to
  471. do) / refused (the connector said no, with a reason) / failed (anything else)."""
  472. dial_url = relay_url()
  473. if not dial_url:
  474. return _SKIPPED
  475. existing_id, existing_secret = relay_connection_auth()
  476. if existing_id and existing_secret:
  477. logger.info("relay self-provision skipped: GATEWAY_RELAY_SECRET already set")
  478. return _SKIPPED
  479. try:
  480. access_token = _resolve_relay_identity_token()
  481. except Exception as exc: # health: allow BLE001 -- any resolver failure is a soft boot no-op; retried
  482. logger.warning("relay self-provision skipped: could not resolve identity token (%s)", exc)
  483. return _FAILED
  484. identities = relay_platform_identities()
  485. # gatewayId default mirrors the enroll CLI's hostname-based slug.
  486. try:
  487. host = socket.gethostname().strip()
  488. except OSError:
  489. host = ""
  490. gateway_id = os.environ.get("GATEWAY_RELAY_ID", "").strip() or f"gw-{host or 'hermes'}"
  491. endpoint = relay_endpoint()
  492. route_keys = relay_route_keys()
  493. instance_id = relay_instance_id()
  494. wake_url = relay_wake_url()
  495. provisioned, refused, result = _provision_each_platform(
  496. identities, provision_url=_provision_url(dial_url), access_token=access_token,
  497. gateway_id=gateway_id, gateway_endpoint=endpoint, route_keys=route_keys,
  498. instance_id=instance_id, wake_url=wake_url, display_name=relay_display_name(),
  499. )
  500. if not provisioned:
  501. logger.warning(
  502. "relay self-provision failed for ALL platforms (%s); gateway will boot without relay auth"
  503. " and retry in the background",
  504. ",".join(p for p, _ in identities),
  505. )
  506. return _REFUSED if refused == len(identities) else _FAILED
  507. if not os.environ.get("GATEWAY_RELAY_SECRET"):
  508. # Every platform answered, none issued a credential: the connector holds a
  509. # binding for this gatewayId that this caller could prove neither ownership
  510. # of nor possession of (F-004). The routes are bound but the WS upgrade
  511. # will be refused, so this is NOT a provision — do not report one. The
  512. # operator's recovery path is POST /relay/rotate (or pinning
  513. # GATEWAY_RELAY_SECRET).
  514. logger.warning(
  515. "relay self-provision withheld credentials for ALL platforms (%s): the connector "
  516. "holds a binding for gateway_id=%s this caller could not prove ownership of; "
  517. "gateway will boot without relay auth. Recover with POST /relay/rotate or pin "
  518. "GATEWAY_RELAY_SECRET",
  519. ",".join(provisioned), gateway_id,
  520. )
  521. return _REFUSED
  522. logger.info(
  523. "relay self-provisioned (gateway_id=%s tenant=%s platforms=%s routes=%d inbound=%s instance=%s wake=%s)",
  524. os.environ.get("GATEWAY_RELAY_ID", gateway_id),
  525. str(result.get("tenant") or "") or "?",
  526. ",".join(provisioned),
  527. len(route_keys),
  528. "yes" if endpoint else "outbound-only",
  529. instance_id or "unbound",
  530. "yes" if wake_url else "none",
  531. )
  532. return _PROVISIONED
  533. def _provision_each_platform(identities, *, gateway_id: str, **post_kwargs) -> tuple[list[str], int, dict]:
  534. """POST ``/relay/provision`` per fronted platform; ``(provisioned, refused_count, last_result)``.
  535. Provision EACH fronted platform under the SAME gatewayId + the SAME per-gateway
  536. secret: the connector's secret record is (gatewayId -> tenant) only, platform/
  537. botId live on per-platform route rows, so N POSTs with one gatewayId are
  538. idempotent on the secret and additive on routes. PARTIAL-FAILURE-TOLERANT: a
  539. platform that fails is logged and skipped (just not fronted); the others come up.
  540. """
  541. provisioned: list[str] = []
  542. refused = 0
  543. result: dict = {}
  544. for platform, bot_id in identities:
  545. try:
  546. result = _post_provision(platform=platform, bot_id=bot_id, gateway_id=gateway_id, **post_kwargs)
  547. except RuntimeError as exc:
  548. refused += isinstance(exc, ProvisionRefused)
  549. logger.warning(
  550. "relay self-provision failed for platform=%s (%s); continuing with the rest",
  551. platform, exc,
  552. )
  553. continue
  554. provisioned.append(platform)
  555. # Set creds in-process on the first response that ACTUALLY CARRIES them.
  556. # Never logged.
  557. #
  558. # `secret` may legitimately be absent (connector F-004 — see _post_provision):
  559. # only a caller that proves ownership or possession is handed credential
  560. # material, and every platform after the first in a multi-platform boot is
  561. # a replay of an unchanged binding. Guard on the secret being PRESENT
  562. # rather than on the response having arrived, or the first withheld
  563. # response writes empty strings over the real values — and because the
  564. # `if` tests the same env var it writes, an empty write looks "unset" on
  565. # the next pass while deliveryKey has already been clobbered.
  566. if result.get("secret") and not os.environ.get("GATEWAY_RELAY_SECRET"):
  567. os.environ["GATEWAY_RELAY_ID"] = str(result.get("gatewayId") or gateway_id)
  568. os.environ["GATEWAY_RELAY_SECRET"] = str(result["secret"])
  569. os.environ["GATEWAY_RELAY_DELIVERY_KEY"] = str(result.get("deliveryKey") or "")
  570. return provisioned, refused, result
  571. async def retry_self_provision(
  572. on_provisioned: Optional[Callable[[], None]] = None, *, sleep=asyncio.sleep,
  573. ) -> bool:
  574. """Keep provisioning a relay that booted without credentials until it has them.
  575. Boot provisioning is one attempt, and a relay without credentials is closed 4401 on
  576. every dial, so one transient failure (an egress proxy's 403, a DNS blip, a connector
  577. restart) used to last until the next restart. This retries on
  578. ``_PROVISION_RETRY_DELAYS_S`` (``_PROVISION_REFUSAL_RETRY_S`` after a reasoned
  579. refusal) and stops as soon as credentials exist, from here or from an operator
  580. pinning them, or when the relay is no longer configured. On success it re-declares
  581. the relevance policy (skipped at boot without credentials) and calls
  582. ``on_provisioned`` so the caller can reconnect the relay now. Returns True iff it
  583. provisioned. Never raises (cancellation aside): a failed attempt is logged and paced.
  584. """
  585. loop = asyncio.get_running_loop()
  586. attempt = 0
  587. delay = _PROVISION_RETRY_DELAYS_S[0]
  588. while True:
  589. await sleep(delay)
  590. if not relay_provision_pending():
  591. return False
  592. attempt += 1
  593. try:
  594. outcome = await loop.run_in_executor(None, contextvars.copy_context().run, _self_provision)
  595. except Exception:
  596. logger.warning("relay self-provision retry %d raised", attempt, exc_info=True)
  597. outcome = _FAILED
  598. if outcome == _PROVISIONED:
  599. logger.info("relay credentials obtained on self-provision retry %d; reconnecting the relay", attempt)
  600. with contextlib.suppress(Exception):
  601. await loop.run_in_executor(None, contextvars.copy_context().run, send_relay_policy)
  602. if on_provisioned is not None:
  603. on_provisioned()
  604. return True
  605. if outcome == _SKIPPED:
  606. return False
  607. delay = (_PROVISION_REFUSAL_RETRY_S if outcome == _REFUSED
  608. else _PROVISION_RETRY_DELAYS_S[min(attempt, len(_PROVISION_RETRY_DELAYS_S) - 1)])
  609. logger.info("relay self-provision retry %d %s; next attempt in %ds", attempt, outcome, delay)
  610. def _post_policy(*, policy_url: str, token: str, policy: dict, timeout: float = _HTTP_TIMEOUT_S) -> int:
  611. """POST the relevance policy to ``/relay/policy``; return the HTTP status.
  612. Authenticated with the gateway's own upgrade token, so the connector resolves
  613. ``{tenant, instanceId}`` from its stored secret record, never the body. Raises
  614. RuntimeError on transport failure."""
  615. try:
  616. with _json_post(policy_url, token, policy, timeout) as resp:
  617. return int(resp.status)
  618. except urllib.error.HTTPError as exc:
  619. return int(exc.code)
  620. except urllib.error.URLError as exc:
  621. raise RuntimeError(f"could not reach connector: {exc.reason}") from exc
  622. def send_relay_policy() -> bool:
  623. """Declare this gateway's relevance policy to the connector, per fronted platform,
  624. at boot AFTER the per-gateway secret is resolved. The connector enforces it on
  625. delivery, so the SAME mention-gating / free-response / allow-bots behavior the
  626. agent applies directly also governs relay delivery, and excluded traffic never
  627. wakes a scaled-to-zero agent. Re-declared every boot (idempotent full replace); a
  628. platform with nothing non-default is skipped; one failed POST doesn't block the
  629. others. NEVER raises / blocks boot: relevance is an optimization layered on the
  630. authorization gate, so a failure just leaves the connector's prior policy.
  631. Returns True iff the connector accepted at least one policy (HTTP 200)."""
  632. dial_url = relay_url()
  633. if not dial_url:
  634. return False
  635. gateway_id, secret = relay_connection_auth()
  636. if not gateway_id or not secret:
  637. # Can't authenticate the POST (and there's no instance to attach a policy to).
  638. return False
  639. try:
  640. from gateway.relay.auth import make_upgrade_token
  641. token = make_upgrade_token(gateway_id, secret)
  642. except Exception as exc:
  643. logger.warning("relay policy declaration failed to build token (%s); connector keeps prior policy", exc)
  644. return False
  645. policy_url = _connector_url(dial_url, "/relay/policy")
  646. any_declared = False
  647. for platform, _bot_id in relay_platform_identities():
  648. policy = relay_relevance_policy(platform)
  649. if policy is None:
  650. continue
  651. try:
  652. status = _post_policy(policy_url=policy_url, token=token, policy=policy)
  653. except Exception as exc:
  654. logger.warning(
  655. "relay policy declaration failed for platform=%s (%s); continuing", platform, exc
  656. )
  657. continue
  658. if status == 200:
  659. any_declared = True
  660. logger.info(
  661. "relay policy declared (platform=%s require_address=%s free_scopes=%d allow_bots=%s)",
  662. policy.get("platform"),
  663. policy.get("requireAddress"),
  664. len(policy.get("freeResponseScopes") or []),
  665. policy.get("allowOtherBots"),
  666. )
  667. else:
  668. logger.warning(
  669. "relay policy declaration for platform=%s returned HTTP %s; connector keeps prior/default policy",
  670. platform,
  671. status,
  672. )
  673. return any_declared
  674. def register_relay_adapter(force: bool = False, url: Optional[str] = None) -> bool:
  675. """Register the generic ``relay`` platform when a relay URL is configured (or
  676. ``force=True`` for tests: transport-less adapter). Neither overrides an
  677. explicit profile disable. Returns True if registered.
  678. With a URL the factory builds a live ``WebSocketRelayTransport``; the adapter
  679. negotiates the real ``CapabilityDescriptor`` at ``connect()``."""
  680. if relay_explicitly_disabled():
  681. return False
  682. resolved_url = url if url is not None else relay_url()
  683. if not (force or resolved_url):
  684. return False
  685. from gateway.platform_registry import PlatformEntry, platform_registry
  686. from gateway.relay.adapter import RelayAdapter
  687. from gateway.relay.descriptor import CONTRACT_VERSION, CapabilityDescriptor
  688. platform, bot_id = relay_platform_identity()
  689. def _factory(config):
  690. # Placeholder descriptor; replaced by the negotiated one at connect time.
  691. # With no URL (force/test) the adapter is transport-less and keeps it.
  692. placeholder = CapabilityDescriptor(
  693. contract_version=CONTRACT_VERSION,
  694. platform=platform,
  695. label="Relay",
  696. max_message_length=4096,
  697. supports_draft_streaming=False,
  698. supports_edit=True,
  699. supports_threads=False,
  700. markdown_dialect="plain",
  701. len_unit="chars",
  702. )
  703. transport = None
  704. if resolved_url:
  705. from gateway.relay.ws_transport import WebSocketRelayTransport
  706. gateway_id, upgrade_secret = relay_connection_auth()
  707. transport = WebSocketRelayTransport(
  708. resolved_url,
  709. platform,
  710. bot_id,
  711. # The full SET of identities: one hello per identity (the connector
  712. # accumulates them) and the per-frame egress botId resolves from it.
  713. identities=relay_platform_identities(),
  714. gateway_id=gateway_id,
  715. upgrade_secret=upgrade_secret,
  716. # Re-dial + re-handshake after an unexpected close so a gateway that
  717. # went idle re-establishes its socket, which triggers the connector's
  718. # buffered-flip drain on the new handshake.
  719. reconnect=True,
  720. )
  721. return RelayAdapter(config, placeholder, transport=transport)
  722. platform_registry.register(
  723. PlatformEntry(
  724. name="relay",
  725. label="Relay",
  726. adapter_factory=_factory,
  727. check_fn=lambda: True,
  728. source="builtin",
  729. emoji="\U0001f50c",
  730. )
  731. )
  732. return True