| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852 |
- """Relay/connector support package for the Hermes gateway.
- EXPERIMENTAL gateway side of the "Gateway Gateway" relay design: a generic
- ``RelayAdapter`` plus the wire-serializable ``CapabilityDescriptor`` the connector
- hands it at handshake, and the production ``WebSocketRelayTransport``. The public
- API MAY CHANGE without a deprecation cycle until >=2 real Class-1 platforms have
- shaken out the schema (``website/docs/developer-guide/relay-connector-contract.md``). Activation is
- config-driven: the relay platform is registered when a connector relay URL is set
- (``GATEWAY_RELAY_URL`` env or ``gateway.relay_url``), like ``gateway.proxy_url``,
- unless the effective relay platform configuration explicitly disables it.
- """
- from __future__ import annotations
- import asyncio
- import contextlib
- import contextvars
- import json
- import logging
- import os
- import re
- import socket
- import urllib.error
- import urllib.parse
- import urllib.request
- from typing import Callable, Optional
- logger = logging.getLogger("gateway.relay")
- # Shape gate for ambient-endpoint token bodies (mode 1b in
- # _resolve_relay_identity_token): a JWT (3+ dotted segments) or a long opaque
- # base64url token (>= 32 chars). Short bare words — 'unauthorized', 'error',
- # 'null' — match the alphabet but are plain-text error bodies and must fail closed.
- _AMBIENT_TOKEN_SHAPE = re.compile(
- r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+){2,}" # JWT-like: 3+ dotted segments
- r"|[A-Za-z0-9_-]{32,}" # long opaque bearer token
- )
- _HTTP_TIMEOUT_S = 15.0
- # Boot self-provision retry (a relay configured but no credentials obtained). Delays
- # before each retry; the last one repeats until credentials exist. The first retry
- # lands quickly because the common cause is a blip (proxy, DNS, connector restart);
- # the cap matches the reconnect watcher's 300s cap so recovery is never slower than
- # the 4401 loop it replaces.
- _PROVISION_RETRY_DELAYS_S = (5.0, 30.0, 120.0, 300.0)
- # A refusal the connector authored WITH a reason (identity not confirmed, binding owned
- # elsewhere, withheld secret) changes only when an operator acts, so it is retried
- # less often than a blip: bounded log noise and NAS introspection load, yet the
- # gateway still heals on its own within minutes of the fix.
- _PROVISION_REFUSAL_RETRY_S = 600.0
- # self-provision outcomes (``_self_provision``)
- _PROVISIONED, _SKIPPED, _REFUSED, _FAILED = "provisioned", "skipped", "refused", "failed"
- class ProvisionRefused(RuntimeError):
- """The connector refused ``/relay/provision`` with its own reason (a 4xx JSON
- ``{"error"}``): definitive until something changes on the connector/NAS side."""
- # gateway.idp.* keys (and GATEWAY_RELAY_IDP_<KEY> env mirrors) read by the identity resolver.
- _IDP_KEYS = ("token_url", "client_id", "client_secret", "scope")
- # ─────────────────────────── config access ───────────────────────────
- def _load_cfg() -> dict:
- """The full gateway config, or ``{}`` — config absence/parse must never crash boot."""
- try:
- from gateway.run import _load_gateway_config # late import to avoid cycle
- cfg = _load_gateway_config()
- except Exception:
- return {}
- return cfg if isinstance(cfg, dict) else {}
- def _gateway_cfg() -> dict:
- """The ``gateway:`` block of config.yaml, or ``{}``."""
- block = _load_cfg().get("gateway")
- return block if isinstance(block, dict) else {}
- def _launch_profile_scope():
- """The launch profile's runtime scope once the gateway multiplexes, ``nullcontext`` before.
- Self-provision runs at startup, before any turn binds a profile, and under multiplexing the
- Nous Portal and inference routing overrides only resolve inside a profile scope. The scope is
- built from the profile's files like a turn's (``gateway.run._profile_runtime_scope``), not via
- ``launch_profile_scope_if_multiplexed``, whose first entry freezes the launch env snapshot
- before startup has written ``GATEWAY_RELAY_*`` into ``os.environ``.
- """
- from agent.secret_scope import is_multiplex_active
- if not is_multiplex_active():
- return contextlib.nullcontext()
- from gateway.run import _profile_runtime_scope # late import to avoid cycle
- from hermes_constants import get_process_hermes_home
- return _profile_runtime_scope(get_process_hermes_home())
- def _env_or_cfg(env_var: str, cfg_key: str) -> str:
- """Env var first (Docker/NAS stamp), then ``gateway.<cfg_key>``; stripped, ``""`` when unset."""
- value = os.environ.get(env_var, "").strip()
- return value or str(_gateway_cfg().get(cfg_key, "") or "").strip()
- def _env_or_cfg_url(env_var: str, cfg_key: str) -> Optional[str]:
- """``_env_or_cfg`` for URL-shaped values: trailing slash stripped, None when unset."""
- return _env_or_cfg(env_var, cfg_key).rstrip("/") or None
- def relay_explicitly_disabled() -> bool:
- """``platforms.relay.enabled: false`` in the profile's YAML (user or managed).
- Same files, merge and boolean normalization as the gateway loader (a malformed user
- file drops the whole YAML layer there too), minus its env/plugin side effects, so a
- standalone scheduler can ask without bootstrapping the gateway.
- Mirrors the loader's ``_enabled_explicit`` rule: only a YAML ``enabled`` key is
- authoritative — a legacy ``gateway.json`` block is advisory for relay exactly as it
- is for every other platform, and an absent key keeps URL-only activation.
- """
- from gateway.config import Platform, PlatformConfig
- from gateway.config_loader import bridge_platform_shared_keys, merge_platform_sections, read_yaml_layers
- from hermes_constants import get_hermes_home
- try:
- cfg = read_yaml_layers(get_hermes_home())
- except Exception:
- return False
- if not isinstance(cfg, dict):
- return False
- gateway = cfg.get("gateway") if isinstance(cfg.get("gateway"), dict) else {}
- platforms = merge_platform_sections(cfg, gateway, {})
- bridge_platform_shared_keys(cfg, gateway.get("platforms"), {}, platforms, [Platform.RELAY])
- block = platforms.get("relay")
- if not isinstance(block, dict) or not block.get("extra", {}).get("_enabled_explicit"):
- return False
- return not PlatformConfig.from_dict(block).enabled
- def relay_url() -> Optional[str]:
- """Effective connector URL; an explicit platform disable vetoes even an env URL."""
- if relay_explicitly_disabled():
- return None
- return _env_or_cfg_url("GATEWAY_RELAY_URL", "relay_url")
- def relay_platform_identities() -> list[tuple[str, str]]:
- """The ordered (platform, bot_id) pairs this gateway fronts on one WS connection,
- from ``GATEWAY_RELAY_PLATFORMS`` (comma-sep) and ``GATEWAY_RELAY_BOT_IDS`` (JSON
- ``{"discord": {"botId": "..."}}``). The FIRST pair is the handshake/descriptor
- default; a platform absent from the ids map gets an empty bot_id (the connector
- rejects it with a structured failure). ``[("relay", "")]`` when nothing is set."""
- platforms = [p.strip() for p in os.environ.get("GATEWAY_RELAY_PLATFORMS", "").split(",") if p.strip()]
- if not platforms:
- return [("relay", "")]
- ids = _relay_bot_ids_map()
- out: list[tuple[str, str]] = []
- for platform in platforms:
- entry = ids.get(platform) or {}
- bot_id = str(entry.get("botId", "")).strip() if isinstance(entry, dict) else ""
- out.append((platform, bot_id))
- return out
- def relay_fronted_platforms() -> set[str]:
- """Logical platform names the relay fronts, minus the generic ``relay`` fallback.
- Same env source the live adapter's identity set comes from, so config-time
- validation (cron delivery preflight) and fire-time routing can never disagree —
- and it needs no live adapter, so a standalone scheduler can use it."""
- if relay_explicitly_disabled():
- return set()
- return {p for p, _ in relay_platform_identities() if p != "relay"}
- def _relay_bot_ids_map() -> dict:
- """Parse ``GATEWAY_RELAY_BOT_IDS``; a malformed map yields ``{}`` rather than crashing boot."""
- raw = os.environ.get("GATEWAY_RELAY_BOT_IDS", "").strip()
- if not raw:
- return {}
- try:
- parsed = json.loads(raw)
- return parsed if isinstance(parsed, dict) else {}
- except Exception:
- logger.warning("GATEWAY_RELAY_BOT_IDS is not valid JSON; treating as empty")
- return {}
- def relay_platform_identity() -> tuple[str, str]:
- """The PRIMARY (platform, bot_id) — first of ``relay_platform_identities()``."""
- return relay_platform_identities()[0]
- def relay_connection_auth() -> tuple[Optional[str], Optional[str]]:
- """The (gateway_id, upgrade_secret) from enrollment (``GATEWAY_RELAY_ID`` /
- ``GATEWAY_RELAY_SECRET``, then ``gateway.relay_id`` / ``gateway.relay_secret``).
- Either absent -> ``(None, None)`` and the transport dials unauthenticated."""
- gateway_id = _env_or_cfg("GATEWAY_RELAY_ID", "relay_id")
- secret = _env_or_cfg("GATEWAY_RELAY_SECRET", "relay_secret")
- return (gateway_id or None, secret or None)
- def relay_endpoint() -> Optional[str]:
- """The gateway's own PUBLIC inbound URL, asserted to the connector at provision
- (``GATEWAY_RELAY_ENDPOINT`` / ``gateway.relay_endpoint``). Stored on the tenant's
- route rows; gateway-asserted but tenant-scoped, so a dishonest gateway can only
- misdirect its OWN inbound. Absent -> outbound-only (no inbound routes written)."""
- return _env_or_cfg_url("GATEWAY_RELAY_ENDPOINT", "relay_endpoint")
- def relay_route_keys() -> list[str]:
- """Discriminators (scope_ids / chat_ids / paths) this gateway's tenant owns. The
- connector writes one route row per (routeKey -> tenant, endpoint), so keys only
- take effect alongside ``relay_endpoint()``. ``GATEWAY_RELAY_ROUTE_KEYS`` is
- comma-separated; ``gateway.relay_route_keys`` may be a list or a comma string."""
- raw = os.environ.get("GATEWAY_RELAY_ROUTE_KEYS", "").strip()
- if not raw:
- val = _gateway_cfg().get("relay_route_keys", "")
- if isinstance(val, (list, tuple)):
- return [str(k).strip() for k in val if str(k).strip()]
- raw = str(val or "").strip()
- return [k.strip() for k in raw.split(",") if k.strip()]
- def relay_instance_id() -> Optional[str]:
- """Stable per-instance id forwarded at provision (``GATEWAY_RELAY_INSTANCE_ID`` /
- ``gateway.relay_instance_id``): binds the connector's ``gatewayId -> instanceId``
- so inbound routes per-instance rather than tenant-broadcast (NAS stamps its
- ``AgentInstance.id``). Tenant-scoped like ``relay_endpoint()``; absent -> null."""
- return _env_or_cfg("GATEWAY_RELAY_INSTANCE_ID", "relay_instance_id") or None
- def relay_wake_url() -> Optional[str]:
- """The gateway's WAKE URL forwarded at provision (``GATEWAY_RELAY_WAKE_URL`` /
- ``gateway.relay_wake_url``): a payload-free poke the connector GETs when a
- going-idle destination receives its first buffered event, so a suspended gateway
- wakes, reconnects and drains. Absent -> no wake (buffering still works)."""
- return _env_or_cfg_url("GATEWAY_RELAY_WAKE_URL", "relay_wake_url")
- def relay_display_name() -> Optional[str]:
- """The human-facing agent display name forwarded at provision — the connector's
- multi-agent reply-attribution prefix (``**<displayName>:** ``).
- ``GATEWAY_RELAY_DISPLAY_NAME`` env, then the skin's branded agent name (a skin
- rename propagates on the next boot). Absent -> connector's linked-owner fallback.
- The PRIMARY source for the connector's multi-agent reply-attribution prefix (gateway-gateway #171): in a
- multi-agent scope the shared bot prepends ``**<displayName>:** `` to this instance's replies.
- Gateway-asserted but safely scoped exactly like ``relay_instance_id()`` / ``relay_wake_url()`` — the
- tenant stays token-verified, so a dishonest gateway can only label its OWN instance. Absent -> the
- connector stores null and attribution falls back to the instance's linked-owner identity, else skips the
- prefix.
- """
- value = os.environ.get("GATEWAY_RELAY_DISPLAY_NAME", "").strip()
- if not value:
- try:
- from hermes_cli.skin_engine import get_active_skin # late import: boot-safe
- value = str(get_active_skin().get_branding("agent_name", "") or "").strip()
- except Exception:
- value = ""
- # The stock brand is identical on every default install: forwarding it would
- # prefix every reply "**Hermes Agent:**" and shadow the connector's
- # linked-owner fallback, which actually disambiguates. Only a customized
- # name is forwarded.
- if value == "Hermes Agent":
- value = ""
- # Mirror the connector's ingest sanitization (trim + 64-char cap).
- return value[:64] or None
- # ─────────────────────────── connector HTTP ───────────────────────────
- def _connector_url(relay_dial_url: str, path: str) -> str:
- """``ws(s)://…/relay`` dial URL -> ``http(s)://…{path}`` connector route."""
- from gateway.relay.media import media_base_url
- return f"{media_base_url(relay_dial_url)}{path}"
- def _provision_url(relay_dial_url: str) -> str:
- return _connector_url(relay_dial_url, "/relay/provision")
- def _json_post(url: str, token: str, body: dict, timeout: float):
- """Bearer-authenticated JSON POST; returns the open response (caller reads it)."""
- req = urllib.request.Request(
- url,
- data=json.dumps(body).encode("utf-8"),
- method="POST",
- headers={
- "Authorization": f"Bearer {token}",
- "Content-Type": "application/json",
- "Accept": "application/json",
- },
- )
- return urllib.request.urlopen(req, timeout=timeout)
- def relay_relevance_policy(platform: Optional[str] = None) -> Optional[dict]:
- """Project a fronted platform's RELEVANCE config into the connector's generic vocabulary.
- The connector's relevance gate reasons over a platform-agnostic policy keyed by
- ``(tenant, platform, instanceId)``: ``requireAddress`` <- ``require_mention``,
- ``freeResponseScopes`` <- ``free_response_channels``, ``allowOtherBots`` <-
- ``{PLATFORM}_ALLOW_BOTS`` in {"mentions","all"}. Read from the platform's config
- block (``discord:``), falling back to the bridged top-level keys, then env.
- ``platform`` defaults to the PRIMARY fronted platform. Returns None when relay
- isn't configured or nothing is CONFIGURED to declare, so the connector's default
- (mention-gated) applies. The condition is "require_mention is unset", NOT "falsy":
- an EXPLICIT ``require_mention: false`` is a non-default choice that MUST be
- declared or the connector would mention-gate an agent configured to free-respond.
- """
- if platform is None:
- platform, _bot_id = relay_platform_identity()
- if not platform or platform == "relay":
- return None
- require_mention = None
- free_response: list[str] = []
- try:
- cfg = _load_cfg()
- # Platform block lookup order: top-level ``<platform>:``, then
- # ``gateway.platforms.<platform>``, then ``platforms.<platform>``.
- def _candidates():
- yield cfg.get(platform)
- gw_platforms = (cfg.get("gateway") or {}).get("platforms") or {}
- yield gw_platforms.get(platform) if isinstance(gw_platforms, dict) else None
- yield (cfg.get("platforms") or {}).get(platform)
- plat_cfg = next((c for c in _candidates() if isinstance(c, dict)), {})
- if "require_mention" in plat_cfg:
- require_mention = plat_cfg.get("require_mention")
- elif cfg.get("require_mention") is not None:
- require_mention = cfg.get("require_mention")
- frc = plat_cfg.get("free_response_channels")
- if frc is None:
- frc = cfg.get("free_response_channels")
- if isinstance(frc, (list, tuple)):
- free_response = [str(c).strip() for c in frc if str(c).strip()]
- elif isinstance(frc, str) and frc.strip():
- free_response = [c.strip() for c in frc.split(",") if c.strip()]
- except Exception:
- pass
- # Same gate as the gateway's own authz_mixin DISCORD_ALLOW_BOTS bypass.
- allow_bots_env = os.environ.get(f"{platform.upper()}_ALLOW_BOTS", "").lower().strip()
- allow_other_bots = allow_bots_env in {"mentions", "all"}
- if require_mention is None and not free_response and not allow_other_bots:
- return None
- return {
- "platform": platform,
- "requireAddress": bool(require_mention),
- "freeResponseScopes": free_response,
- "allowOtherBots": allow_other_bots,
- }
- def _post_provision(
- *,
- provision_url: str,
- access_token: str,
- gateway_id: str,
- platform: str,
- bot_id: str,
- gateway_endpoint: Optional[str],
- route_keys: list[str],
- instance_id: Optional[str] = None,
- wake_url: Optional[str] = None,
- display_name: Optional[str] = None,
- timeout: float = _HTTP_TIMEOUT_S,
- ) -> dict:
- """POST ``/relay/provision``; return ``{secret, deliveryKey, tenant, gatewayId,
- routeKeys}`` (the connector validates ``access_token`` against NAS, derives the
- tenant, mints the per-gateway secret and upserts route rows). Raises RuntimeError
- with a user-facing message on any non-2xx / transport failure."""
- body: dict = {
- "gatewayId": gateway_id,
- "platform": platform,
- "botId": bot_id,
- "gatewayEndpoint": gateway_endpoint or "",
- "routeKeys": route_keys,
- }
- # Optional fields are OMITTED when absent so the connector stores null
- # (back-compat) rather than binding an empty string.
- for key, value in (("instanceId", instance_id), ("wakeUrl", wake_url), ("displayName", display_name)):
- if value:
- body[key] = value
- try:
- with _json_post(provision_url, access_token, body, timeout) as resp:
- payload = json.loads(resp.read().decode())
- except urllib.error.HTTPError as exc:
- from gateway.relay import http_diagnostics as diag
- err_body = diag.read_error_body(exc)
- # Never log the identity token we sent, even if a hop echoes it back bare.
- detail = diag.redact_excerpt(diag.connector_reason(err_body), (access_token,))
- if detail and 400 <= exc.code < 500 and exc.code not in (408, 429) and not diag.answered_by_proxy(exc):
- # The connector itself refused, with a reason: retrying soon will not change it.
- raise ProvisionRefused(f"connector returned HTTP {exc.code}: {detail}") from exc
- # No connector reason (a proxy's page, JSON or not) or a 5xx/408/429: name whoever answered.
- described = diag.describe_http_error(exc, err_body, secrets=(access_token,))
- raise RuntimeError(described + (f": {detail}" if detail else "")) from exc
- except urllib.error.URLError as exc:
- raise RuntimeError(f"could not reach connector: {exc.reason}") from exc
- if not isinstance(payload, dict):
- raise RuntimeError("connector returned an unexpected response")
- # A response WITHOUT a secret is valid, not an error (connector F-004).
- #
- # `/relay/provision` used to hand the stored per-gateway secret back on every
- # replay, to anyone who reached the endpoint with the right gatewayId. The
- # connector now returns credential material only to a caller that proves
- # ownership or possession, and answers `secretIssued: false` otherwise — with
- # `secret`/`deliveryKey` absent rather than empty.
- #
- # Two legitimate shapes reach here, and neither is a failure:
- # * the SECOND and later platforms of a multi-platform boot. All platforms
- # share one gatewayId, so the first POST mints the secret and the rest are
- # replays of an unchanged binding. Raising here aborted every platform
- # after the first.
- # * a re-provision by a caller that legitimately no longer holds the secret;
- # `POST /relay/rotate` is the supported recovery path.
- #
- # Treat "no secret" as a value, and let the caller decide — it knows whether
- # it already holds credentials. Only a malformed body is an error.
- if not payload.get("secret") and payload.get("secretIssued") is not False:
- # Fail closed on the discriminator itself. The ONLY credential-less shape
- # the connector emits is literal JSON `false`. No key at all is a
- # pre-F-004 connector (the old unexpected-response case); `true` with no
- # secret, or any non-boolean value, is a malformed body. Accepting those
- # would mark the platform provisioned with no credential in hand.
- raise RuntimeError("connector returned an unexpected response (no secret)")
- return payload
- def _resolve_relay_identity_token() -> str:
- """Resolve the caller-identity bearer token the connector introspects to a tenant.
- Canonical resolver shared by runtime self-provision and ``hermes gateway enroll``.
- Modes, in precedence order:
- 1. Generic OIDC client-credentials (self-hosted IdP): ``gateway.idp.token_url``
- (``GATEWAY_RELAY_IDP_TOKEN_URL``) set together with client id + secret ->
- POST the OAuth2 ``client_credentials`` grant.
- 1b. Ambient token endpoint: ``token_url`` with NEITHER client_id nor
- client_secret is a metadata-server-style endpoint (e.g. Domino's
- ``$DOMINO_API_PROXY/access-token``): a plain GET whose body IS the token,
- raw JWT or a JSON envelope with ``access_token``. Possession of the
- (typically loopback) endpoint is the credential.
- 2. Nous Portal (default): ``resolve_nous_access_token()``.
- Raises on failure; callers decide whether that's fatal (enroll CLI) or a graceful
- boot no-op (self-provision).
- """
- env = {k: os.environ.get(f"GATEWAY_RELAY_IDP_{k.upper()}", "").strip() for k in _IDP_KEYS}
- if not env["token_url"]:
- # Env token_url absent -> the whole idp block comes from config (env id/secret
- # still win per key). A malformed gateway.idp degrades to "no token_url".
- idp = _gateway_cfg().get("idp")
- idp = idp if isinstance(idp, dict) else {}
- for k in _IDP_KEYS:
- env[k] = env[k] or str(idp.get(k, "") or "").strip()
- token_url, client_id, client_secret, scope = (env[k] for k in _IDP_KEYS)
- if not token_url:
- from hermes_cli.auth import resolve_nous_access_token
- # Only the Nous refresh is scoped: the gateway config read above and the IdP modes below
- # keep resolving ``${VAR}`` refs from the launch env, as they always have.
- with _launch_profile_scope():
- return resolve_nous_access_token()
- if not client_id and not client_secret:
- # Mode 1b — plain GET; the body is the token, raw or JSON-enveloped.
- req = urllib.request.Request(
- token_url, method="GET", headers={"Accept": "application/json, text/plain"}
- )
- with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT_S) as resp:
- body = resp.read().decode().strip()
- token = ""
- if body.startswith("{"):
- try:
- envelope_token = (json.loads(body) or {}).get("access_token")
- except ValueError:
- envelope_token = None
- # No shape gate on an envelope: it is a deliberate token response, and
- # opaque tokens may use the standard-base64 alphabet the raw gate rejects.
- if isinstance(envelope_token, str):
- token = envelope_token.strip()
- elif _AMBIENT_TOKEN_SHAPE.fullmatch(body):
- token = body
- if not token:
- raise RuntimeError(
- "no client_id/client_secret configured, so gateway.idp.token_url was "
- "treated as an ambient token endpoint (GET), but the response body "
- "was not a token. For the OAuth2 client_credentials grant, configure "
- "client_id and client_secret alongside token_url."
- )
- return token
- if not client_id or not client_secret:
- # Exactly one credential: a mistyped client_credentials setup, not an
- # ambient endpoint. Fail loud; never GET the IdP.
- missing = "client_secret" if client_id else "client_id"
- raise RuntimeError(
- f"gateway.idp.token_url is configured with a partial client credential "
- f"({missing} missing). Configure both client_id and client_secret for "
- f"the OAuth2 client_credentials grant, or neither to treat token_url "
- f"as an ambient token endpoint (plain GET returning the token)."
- )
- # Mode 1 — OAuth2 client_credentials grant.
- form = {"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret}
- if scope:
- form["scope"] = scope
- req = urllib.request.Request(
- token_url, data=urllib.parse.urlencode(form).encode("utf-8"), method="POST",
- headers={"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json"},
- )
- with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT_S) as resp:
- payload = json.loads(resp.read().decode())
- access_token = (payload or {}).get("access_token")
- if not isinstance(access_token, str) or not access_token.strip():
- raise RuntimeError("IdP client_credentials response had no access_token")
- return access_token.strip()
- def self_provision_relay() -> bool:
- """Boot-time relay self-provision: mint relay creds in-process, no human, no disk.
- Fires when ``relay_url()`` is set and NO per-gateway secret is pinned: resolves the
- identity token, POSTs ``/relay/provision`` for EACH fronted platform and sets
- ``GATEWAY_RELAY_ID`` / ``_SECRET`` / ``_DELIVERY_KEY`` in ``os.environ`` for
- ``register_relay_adapter()``. Creds live ONLY in process memory (never
- ``~/.hermes/.env``), so a hosted container re-provisions every boot; the
- connector's rotation window covers a still-connected prior instance. The trigger
- is deliberately NOT ``is_managed()`` (False on a NAS-hosted Fly agent): "pointed
- at a connector without a pinned secret" is the real signal and self-guards (an
- enrolled gateway has a PINNED secret -> skipped; no resolvable identity -> no-op).
- Returns True iff it provisioned. NEVER raises: a failure logs and returns False so
- the gateway still boots; ``retry_self_provision`` then keeps trying in the
- background (the adapter meanwhile dials unauthenticated / is rejected).
- """
- return _self_provision() == _PROVISIONED
- def relay_provision_pending() -> bool:
- """A relay is configured but this process holds no per-gateway secret yet."""
- if not relay_url():
- return False
- existing_id, existing_secret = relay_connection_auth()
- return not (existing_id and existing_secret)
- def _self_provision() -> str:
- """``self_provision_relay`` with the outcome kept: provisioned / skipped (nothing to
- do) / refused (the connector said no, with a reason) / failed (anything else)."""
- dial_url = relay_url()
- if not dial_url:
- return _SKIPPED
- existing_id, existing_secret = relay_connection_auth()
- if existing_id and existing_secret:
- logger.info("relay self-provision skipped: GATEWAY_RELAY_SECRET already set")
- return _SKIPPED
- try:
- access_token = _resolve_relay_identity_token()
- except Exception as exc: # health: allow BLE001 -- any resolver failure is a soft boot no-op; retried
- logger.warning("relay self-provision skipped: could not resolve identity token (%s)", exc)
- return _FAILED
- identities = relay_platform_identities()
- # gatewayId default mirrors the enroll CLI's hostname-based slug.
- try:
- host = socket.gethostname().strip()
- except OSError:
- host = ""
- gateway_id = os.environ.get("GATEWAY_RELAY_ID", "").strip() or f"gw-{host or 'hermes'}"
- endpoint = relay_endpoint()
- route_keys = relay_route_keys()
- instance_id = relay_instance_id()
- wake_url = relay_wake_url()
- provisioned, refused, result = _provision_each_platform(
- identities, provision_url=_provision_url(dial_url), access_token=access_token,
- gateway_id=gateway_id, gateway_endpoint=endpoint, route_keys=route_keys,
- instance_id=instance_id, wake_url=wake_url, display_name=relay_display_name(),
- )
- if not provisioned:
- logger.warning(
- "relay self-provision failed for ALL platforms (%s); gateway will boot without relay auth"
- " and retry in the background",
- ",".join(p for p, _ in identities),
- )
- return _REFUSED if refused == len(identities) else _FAILED
- if not os.environ.get("GATEWAY_RELAY_SECRET"):
- # Every platform answered, none issued a credential: the connector holds a
- # binding for this gatewayId that this caller could prove neither ownership
- # of nor possession of (F-004). The routes are bound but the WS upgrade
- # will be refused, so this is NOT a provision — do not report one. The
- # operator's recovery path is POST /relay/rotate (or pinning
- # GATEWAY_RELAY_SECRET).
- logger.warning(
- "relay self-provision withheld credentials for ALL platforms (%s): the connector "
- "holds a binding for gateway_id=%s this caller could not prove ownership of; "
- "gateway will boot without relay auth. Recover with POST /relay/rotate or pin "
- "GATEWAY_RELAY_SECRET",
- ",".join(provisioned), gateway_id,
- )
- return _REFUSED
- logger.info(
- "relay self-provisioned (gateway_id=%s tenant=%s platforms=%s routes=%d inbound=%s instance=%s wake=%s)",
- os.environ.get("GATEWAY_RELAY_ID", gateway_id),
- str(result.get("tenant") or "") or "?",
- ",".join(provisioned),
- len(route_keys),
- "yes" if endpoint else "outbound-only",
- instance_id or "unbound",
- "yes" if wake_url else "none",
- )
- return _PROVISIONED
- def _provision_each_platform(identities, *, gateway_id: str, **post_kwargs) -> tuple[list[str], int, dict]:
- """POST ``/relay/provision`` per fronted platform; ``(provisioned, refused_count, last_result)``.
- Provision EACH fronted platform under the SAME gatewayId + the SAME per-gateway
- secret: the connector's secret record is (gatewayId -> tenant) only, platform/
- botId live on per-platform route rows, so N POSTs with one gatewayId are
- idempotent on the secret and additive on routes. PARTIAL-FAILURE-TOLERANT: a
- platform that fails is logged and skipped (just not fronted); the others come up.
- """
- provisioned: list[str] = []
- refused = 0
- result: dict = {}
- for platform, bot_id in identities:
- try:
- result = _post_provision(platform=platform, bot_id=bot_id, gateway_id=gateway_id, **post_kwargs)
- except RuntimeError as exc:
- refused += isinstance(exc, ProvisionRefused)
- logger.warning(
- "relay self-provision failed for platform=%s (%s); continuing with the rest",
- platform, exc,
- )
- continue
- provisioned.append(platform)
- # Set creds in-process on the first response that ACTUALLY CARRIES them.
- # Never logged.
- #
- # `secret` may legitimately be absent (connector F-004 — see _post_provision):
- # only a caller that proves ownership or possession is handed credential
- # material, and every platform after the first in a multi-platform boot is
- # a replay of an unchanged binding. Guard on the secret being PRESENT
- # rather than on the response having arrived, or the first withheld
- # response writes empty strings over the real values — and because the
- # `if` tests the same env var it writes, an empty write looks "unset" on
- # the next pass while deliveryKey has already been clobbered.
- if result.get("secret") and not os.environ.get("GATEWAY_RELAY_SECRET"):
- os.environ["GATEWAY_RELAY_ID"] = str(result.get("gatewayId") or gateway_id)
- os.environ["GATEWAY_RELAY_SECRET"] = str(result["secret"])
- os.environ["GATEWAY_RELAY_DELIVERY_KEY"] = str(result.get("deliveryKey") or "")
- return provisioned, refused, result
- async def retry_self_provision(
- on_provisioned: Optional[Callable[[], None]] = None, *, sleep=asyncio.sleep,
- ) -> bool:
- """Keep provisioning a relay that booted without credentials until it has them.
- Boot provisioning is one attempt, and a relay without credentials is closed 4401 on
- every dial, so one transient failure (an egress proxy's 403, a DNS blip, a connector
- restart) used to last until the next restart. This retries on
- ``_PROVISION_RETRY_DELAYS_S`` (``_PROVISION_REFUSAL_RETRY_S`` after a reasoned
- refusal) and stops as soon as credentials exist, from here or from an operator
- pinning them, or when the relay is no longer configured. On success it re-declares
- the relevance policy (skipped at boot without credentials) and calls
- ``on_provisioned`` so the caller can reconnect the relay now. Returns True iff it
- provisioned. Never raises (cancellation aside): a failed attempt is logged and paced.
- """
- loop = asyncio.get_running_loop()
- attempt = 0
- delay = _PROVISION_RETRY_DELAYS_S[0]
- while True:
- await sleep(delay)
- if not relay_provision_pending():
- return False
- attempt += 1
- try:
- outcome = await loop.run_in_executor(None, contextvars.copy_context().run, _self_provision)
- except Exception:
- logger.warning("relay self-provision retry %d raised", attempt, exc_info=True)
- outcome = _FAILED
- if outcome == _PROVISIONED:
- logger.info("relay credentials obtained on self-provision retry %d; reconnecting the relay", attempt)
- with contextlib.suppress(Exception):
- await loop.run_in_executor(None, contextvars.copy_context().run, send_relay_policy)
- if on_provisioned is not None:
- on_provisioned()
- return True
- if outcome == _SKIPPED:
- return False
- delay = (_PROVISION_REFUSAL_RETRY_S if outcome == _REFUSED
- else _PROVISION_RETRY_DELAYS_S[min(attempt, len(_PROVISION_RETRY_DELAYS_S) - 1)])
- logger.info("relay self-provision retry %d %s; next attempt in %ds", attempt, outcome, delay)
- def _post_policy(*, policy_url: str, token: str, policy: dict, timeout: float = _HTTP_TIMEOUT_S) -> int:
- """POST the relevance policy to ``/relay/policy``; return the HTTP status.
- Authenticated with the gateway's own upgrade token, so the connector resolves
- ``{tenant, instanceId}`` from its stored secret record, never the body. Raises
- RuntimeError on transport failure."""
- try:
- with _json_post(policy_url, token, policy, timeout) as resp:
- return int(resp.status)
- except urllib.error.HTTPError as exc:
- return int(exc.code)
- except urllib.error.URLError as exc:
- raise RuntimeError(f"could not reach connector: {exc.reason}") from exc
- def send_relay_policy() -> bool:
- """Declare this gateway's relevance policy to the connector, per fronted platform,
- at boot AFTER the per-gateway secret is resolved. The connector enforces it on
- delivery, so the SAME mention-gating / free-response / allow-bots behavior the
- agent applies directly also governs relay delivery, and excluded traffic never
- wakes a scaled-to-zero agent. Re-declared every boot (idempotent full replace); a
- platform with nothing non-default is skipped; one failed POST doesn't block the
- others. NEVER raises / blocks boot: relevance is an optimization layered on the
- authorization gate, so a failure just leaves the connector's prior policy.
- Returns True iff the connector accepted at least one policy (HTTP 200)."""
- dial_url = relay_url()
- if not dial_url:
- return False
- gateway_id, secret = relay_connection_auth()
- if not gateway_id or not secret:
- # Can't authenticate the POST (and there's no instance to attach a policy to).
- return False
- try:
- from gateway.relay.auth import make_upgrade_token
- token = make_upgrade_token(gateway_id, secret)
- except Exception as exc:
- logger.warning("relay policy declaration failed to build token (%s); connector keeps prior policy", exc)
- return False
- policy_url = _connector_url(dial_url, "/relay/policy")
- any_declared = False
- for platform, _bot_id in relay_platform_identities():
- policy = relay_relevance_policy(platform)
- if policy is None:
- continue
- try:
- status = _post_policy(policy_url=policy_url, token=token, policy=policy)
- except Exception as exc:
- logger.warning(
- "relay policy declaration failed for platform=%s (%s); continuing", platform, exc
- )
- continue
- if status == 200:
- any_declared = True
- logger.info(
- "relay policy declared (platform=%s require_address=%s free_scopes=%d allow_bots=%s)",
- policy.get("platform"),
- policy.get("requireAddress"),
- len(policy.get("freeResponseScopes") or []),
- policy.get("allowOtherBots"),
- )
- else:
- logger.warning(
- "relay policy declaration for platform=%s returned HTTP %s; connector keeps prior/default policy",
- platform,
- status,
- )
- return any_declared
- def register_relay_adapter(force: bool = False, url: Optional[str] = None) -> bool:
- """Register the generic ``relay`` platform when a relay URL is configured (or
- ``force=True`` for tests: transport-less adapter). Neither overrides an
- explicit profile disable. Returns True if registered.
- With a URL the factory builds a live ``WebSocketRelayTransport``; the adapter
- negotiates the real ``CapabilityDescriptor`` at ``connect()``."""
- if relay_explicitly_disabled():
- return False
- resolved_url = url if url is not None else relay_url()
- if not (force or resolved_url):
- return False
- from gateway.platform_registry import PlatformEntry, platform_registry
- from gateway.relay.adapter import RelayAdapter
- from gateway.relay.descriptor import CONTRACT_VERSION, CapabilityDescriptor
- platform, bot_id = relay_platform_identity()
- def _factory(config):
- # Placeholder descriptor; replaced by the negotiated one at connect time.
- # With no URL (force/test) the adapter is transport-less and keeps it.
- placeholder = CapabilityDescriptor(
- contract_version=CONTRACT_VERSION,
- platform=platform,
- label="Relay",
- max_message_length=4096,
- supports_draft_streaming=False,
- supports_edit=True,
- supports_threads=False,
- markdown_dialect="plain",
- len_unit="chars",
- )
- transport = None
- if resolved_url:
- from gateway.relay.ws_transport import WebSocketRelayTransport
- gateway_id, upgrade_secret = relay_connection_auth()
- transport = WebSocketRelayTransport(
- resolved_url,
- platform,
- bot_id,
- # The full SET of identities: one hello per identity (the connector
- # accumulates them) and the per-frame egress botId resolves from it.
- identities=relay_platform_identities(),
- gateway_id=gateway_id,
- upgrade_secret=upgrade_secret,
- # Re-dial + re-handshake after an unexpected close so a gateway that
- # went idle re-establishes its socket, which triggers the connector's
- # buffered-flip drain on the new handshake.
- reconnect=True,
- )
- return RelayAdapter(config, placeholder, transport=transport)
- platform_registry.register(
- PlatformEntry(
- name="relay",
- label="Relay",
- adapter_factory=_factory,
- check_fn=lambda: True,
- source="builtin",
- emoji="\U0001f50c",
- )
- )
- return True
|