| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490 |
- """The gateway's local HTTP/WS listener: the native session WebSocket plus the dashboard API.
- The caller initializes session authority before starting this listener and owns
- all runtime services and signals. This module owns only HTTP resources; it does
- not bootstrap an agent, scheduler, hosted room, or a second gateway.
- The native session WebSocket (ticketed ``/api/ws`` — every local CLI/TUI/Desktop session) is served
- without the dashboard app: importing ``hermes_cli.web_server`` (about 690 routes, ~0.6 s) before the
- listener bound stalled every cold start and the control socket's identify. The dashboard is mounted
- on its first request (imported off the event loop) or warmed after READY for a supervised gateway
- (``warm_gateway_dashboard``); a gated listener (non-loopback bind, ``webapp.public_url`` on the Webapp's,
- ``dashboard.public_url`` on the private one) and a
- process that already imported it mount it before the listener starts, exactly as before.
- """
- import asyncio
- from dataclasses import dataclass
- import logging
- import os
- import socket
- import sys
- import types
- from typing import Any
- logger = logging.getLogger(__name__)
- # One listener per process: the dashboard routers share one process-local app.
- _ACTIVE_RUNNER: Any = None
- # A fixed port (the Webapp's) has to bind again while the last gateway's closed connections sit in
- # TIME_WAIT. POSIX allows that with SO_REUSEADDR; on Windows that option lets another process bind
- # the same port, and a rebind there does not wait on TIME_WAIT anyway.
- _REUSE_FIXED_PORT = os.name != "nt"
- # On Windows a fixed port is held for exclusive use instead: otherwise another process may bind it with
- # SO_REUSEADDR, and a more specific address (127.0.0.1 under a 0.0.0.0 bind) then takes its connections.
- _EXCLUSIVE_FIXED_PORT = os.name == "nt"
- @dataclass
- class GatewayAPIHandle:
- api_origin: str
- server: Any
- task: asyncio.Task
- app: Any
- socket: socket.socket
- class GatewayDashboardMount:
- """ASGI app that imports and mounts the dashboard (``hermes_cli.web_server.app``) on first use.
- ``state`` is the listener's boundary state (``bound_host``, ``bound_port``, ``auth_required``,
- ``trusted_public_hosts``, ``gateway_runner``, ``session_authority``) — what the native WebSocket
- path checks, mirrored onto the dashboard app state once it is mounted. ``state.webapp`` (a
- ``gateway.webapp_listener.WebappMount``) makes the mount serve the browser-hosted Desktop.
- """
- def __init__(self, runner, state):
- self.runner, self._state = runner, state
- self.web = None
- self._lock: asyncio.Lock | None = None
- self._lifespan = None
- self._closed = False
- self._replaced_session_token = None
- self._undo_own_sign_in = None
- @property
- def loaded(self) -> bool:
- return self.web is not None
- @property
- def state(self):
- """Boundary state: the dashboard app's own state once mounted (operators and tests may
- change it there), the listener's pre-mount snapshot before."""
- return self.web.app.state if self.web is not None else self._state
- def _bind_state(self, web) -> None:
- webapp = getattr(self._state, "webapp", None)
- # The browser credential before any app state: one the home cannot store fails the mount
- # with nothing set, and the gateway falls back to its private listener.
- token = None
- if webapp is not None:
- from gateway.webapp_listener import browser_session_token
- token = browser_session_token(webapp.home)
- # Local clients authenticate with tickets minted on the owner-only control socket. The SPA
- # session token is a bearer credential any loopback peer (any OS user) could read from
- # GET /, so the gateway-hosted listener never publishes it.
- web.app.state.withhold_session_token = True
- own_url = _listener_public_url(webapp)
- if own_url is not None:
- web.app.state.listener_public_url = own_url
- elif hasattr(web.app.state, "listener_public_url"):
- delattr(web.app.state, "listener_public_url")
- if webapp is not None: # before the gate looks for a sign-in provider
- from gateway.webapp_listener import use_own_sign_in
- self._undo_own_sign_in = use_own_sign_in(webapp)
- web._configure_auth_gate(self._state.bound_host, False, None, None)
- web.app.state.gateway_runner = self.runner
- web.app.state.session_authority = getattr(self.runner, "session_authority", None)
- web.app.state.bound_host = self._state.bound_host
- web.app.state.bound_port = self._state.bound_port
- if webapp is not None:
- # The browser-hosted Desktop: its bundle at /, its routes (launch handoff, socket tickets,
- # host terminal) on, and the browser credential kept in the home so tabs survive a restart.
- # The page itself still never carries the credential (the webapp surface withholds it).
- web.app.state.ui_surface = "webapp"
- web.app.state.web_dist = webapp.dist
- web.app.state.webapp_home = webapp.home
- self._replaced_session_token = web._SESSION_TOKEN
- web._SESSION_TOKEN = token
- def _unbind_state(self) -> None:
- undo_own_sign_in, self._undo_own_sign_in = self._undo_own_sign_in, None
- if undo_own_sign_in is not None: # a listener mounted next signs in with the configured client
- undo_own_sign_in()
- if self.web is not None:
- self.web.app.state.gateway_runner = None
- self.web.app.state.session_authority = None
- self.web.app.state.withhold_session_token = False
- if hasattr(self.web.app.state, "listener_public_url"):
- delattr(self.web.app.state, "listener_public_url")
- if self._replaced_session_token is not None:
- # Removed, not cleared: a listener mounted next (the private one the gateway falls back
- # to) must find the app as main's listener does, with no surface, bundle or home set.
- for name in ("ui_surface", "web_dist", "webapp_home"):
- if hasattr(self.web.app.state, name):
- delattr(self.web.app.state, name)
- self.web._SESSION_TOKEN, self._replaced_session_token = self._replaced_session_token, None
- async def ensure(self):
- """Import (off the loop), bind and start the dashboard app once; returns ``web_server``."""
- if self.web is not None:
- return self.web
- if self._lock is None:
- self._lock = asyncio.Lock()
- async with self._lock:
- if self.web is not None:
- return self.web
- if self._closed:
- raise RuntimeError("gateway API stopped")
- import importlib
- web = sys.modules.get("hermes_cli.web_server") or await asyncio.to_thread(
- importlib.import_module, "hermes_cli.web_server")
- if self._closed: # the listener stopped while the import ran
- raise RuntimeError("gateway API stopped")
- if getattr(web.app.state, "gateway_runner", None) not in (None, self.runner):
- raise RuntimeError("gateway API already started")
- try:
- # A bind that fails partway (the auth gate refusing a gated bind, or the Webapp's
- # browser credential that cannot be read or written) is undone like a failed lifespan:
- # the listener the gateway falls back to finds the app as main's does.
- self._bind_state(web)
- lifespan = web.app.router.lifespan_context(web.app)
- await lifespan.__aenter__()
- if self._closed: # close() won while the lifespan started; it never saw this mount
- await lifespan.__aexit__(None, None, None)
- raise RuntimeError("gateway API stopped")
- except BaseException:
- self.web = web
- self._unbind_state()
- self.web = None
- raise
- self.web, self._lifespan = web, lifespan
- return web
- async def close(self) -> None:
- self._closed = True
- lifespan, self._lifespan = self._lifespan, None
- try:
- if lifespan is not None:
- await lifespan.__aexit__(None, None, None)
- finally:
- self._unbind_state()
- async def __call__(self, scope, receive, send):
- if scope["type"] not in {"http", "websocket"}:
- return
- try:
- web = await self.ensure()
- except Exception:
- logger.exception("Gateway dashboard API failed to load")
- if scope["type"] == "websocket":
- await send({"type": "websocket.close", "code": 1011})
- else:
- from starlette.responses import JSONResponse
- await JSONResponse({"error": "dashboard_unavailable"}, status_code=500)(scope, receive, send)
- return
- await web.app(scope, receive, send)
- def _listener_socket(family, kind, proto, port: int, *, dual_stack: bool = False) -> socket.socket:
- """The listener's unbound socket; a fixed *port* may bind again at once (``_REUSE_FIXED_PORT``)
- or is held exclusively (``_EXCLUSIVE_FIXED_PORT``). A *dual_stack* IPv6 socket (the ``::``
- wildcard) takes IPv4 connections too."""
- listener = socket.socket(family, kind, proto)
- if dual_stack:
- # "::" means every address, as Linux binds it by default. Windows binds IPv6 only otherwise:
- # IPv4 clients would never reach the Webapp, and the exclusive hold below would cover the
- # IPv6 port alone, leaving 0.0.0.0 on the same port to another program.
- listener.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
- if port and _REUSE_FIXED_PORT:
- listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- elif port and _EXCLUSIVE_FIXED_PORT:
- listener.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)
- return listener
- def _listener_public_url(webapp) -> str | None:
- """The public URL the listener answers to: the Webapp's own ``webapp.public_url`` ("" for none),
- never ``dashboard.public_url``, which describes ``hermes dashboard``. None leaves main's listener
- reading ``dashboard.public_url``."""
- return None if webapp is None else webapp.public_url
- async def start_gateway_api(runner, *, host: str = "127.0.0.1", port: int = 0, webapp=None) -> GatewayAPIHandle:
- global _ACTIVE_RUNNER
- from gateway.webapp_listener import dial_origin
- from hermes_cli.web_server_boundary import (
- _dashboard_public_hosts, listener_auth_required, public_hosts_engage_gate, public_url_hosts,
- should_require_auth,
- )
- from hermes_cli.web_server_lifecycle import build_listener_server
- # Existing routers/auth helpers share one process-local app. Never rebind it
- # underneath another listener. Each authority is bound to its own profile DB.
- loaded_web = sys.modules.get("hermes_cli.web_server")
- if _ACTIVE_RUNNER is not None or (
- loaded_web is not None and getattr(loaded_web.app.state, "gateway_runner", None) is not None):
- raise RuntimeError("gateway API already started")
- public_url = _listener_public_url(webapp)
- if public_url is None:
- trusted = _dashboard_public_hosts()
- auth_required = listener_auth_required(host, trusted)
- else:
- # What web_server._configure_auth_gate decides for a listener that declares its public URL.
- trusted = public_url_hosts(public_url)
- auth_required = should_require_auth(host) or public_hosts_engage_gate(trusted)
- state = types.SimpleNamespace(
- bound_host=host, bound_port=None, auth_required=auth_required,
- trusted_public_hosts=trusted, gateway_runner=runner,
- session_authority=getattr(runner, "session_authority", None), webapp=webapp)
- mount = GatewayDashboardMount(runner, state)
- # Lifespan off: the dashboard's own lifespan runs when it is mounted (GatewayDashboardMount).
- config, server = build_listener_server(mount, host, port, auth_required=state.auth_required,
- timeout_graceful_shutdown=5, lifespan="off")
- family, kind, proto, _, address = socket.getaddrinfo(
- host, port, type=socket.SOCK_STREAM,
- )[0]
- listener = _listener_socket(family, kind, proto, port,
- dual_stack=family == socket.AF_INET6 and address[0] == "::")
- _ACTIVE_RUNNER = runner
- try:
- listener.bind(address)
- if port:
- # Under SO_REUSEADDR another socket may bind a fixed port too, and the first to listen
- # owns it. Listening now settles that before the dashboard mounts, so a lost race falls
- # back at once instead of after the mount is up.
- listener.listen(config.backlog)
- listener.setblocking(False)
- state.bound_port = listener.getsockname()[1]
- # Loading the ASGI graph may fail too; it owns the same bound socket.
- if not config.loaded:
- config.load()
- config.loaded_app = GatewayRuntimeAPI(config.loaded_app, runner, mount)
- # A gated listener needs the dashboard's auth providers (and refuses to start without one);
- # an already-imported dashboard costs nothing to mount now; the Webapp serves its page and
- # mints launch links from the start.
- if state.auth_required or loaded_web is not None or webapp is not None:
- await mount.ensure()
- server.lifespan = config.lifespan_class(config)
- await server.startup(sockets=[listener])
- if not server.started or server.should_exit:
- raise RuntimeError("gateway API lifespan startup failed")
- except BaseException:
- listener.close()
- _ACTIVE_RUNNER = None
- await mount.close()
- raise
- async def serve():
- global _ACTIVE_RUNNER
- try:
- await server.main_loop()
- finally:
- try:
- await server.shutdown(sockets=[listener])
- finally:
- listener.close()
- try:
- await mount.close()
- finally:
- _ACTIVE_RUNNER = None
- task = asyncio.create_task(serve(), name="gateway-api")
- return GatewayAPIHandle(
- # The address the socket bound, never a name: native clients accept only a numeric loopback
- # origin, and ``localhost`` may resolve to either family.
- api_origin=dial_origin(listener.getsockname()[0], state.bound_port),
- server=server, task=task, app=mount, socket=listener,
- )
- def warm_gateway_dashboard(runner) -> None:
- """Mount the dashboard in the background once the gateway is READY (supervised gateways, whose
- clients — Desktop, the dashboard SPA — call its routes right away). Never raises."""
- handle = getattr(runner, "session_api", None)
- mount = getattr(handle, "app", None)
- if not isinstance(mount, GatewayDashboardMount) or mount.loaded:
- return
- async def _warm():
- try:
- await mount.ensure()
- except Exception:
- if not mount._closed:
- logger.warning("Gateway dashboard API warm-up failed; it loads on first request", exc_info=True)
- runner._gateway_dashboard_warmup = asyncio.get_running_loop().create_task(
- _warm(), name="gateway-dashboard-warmup")
- async def stop_gateway_api(handle: GatewayAPIHandle) -> None:
- """Drain sockets without stopping the session authority or taking signals."""
- handle.server.should_exit = True
- # The bootstrap supervisor reports listener failure. Cleanup must still
- # reach adapter/worker settlement when that listener raised or was cancelled.
- await asyncio.shield(asyncio.gather(handle.task, return_exceptions=True))
- def _embedded_chat_enabled() -> bool:
- """``web_server._DASHBOARD_EMBEDDED_CHAT_ENABLED`` without importing the dashboard (always True
- unless a loaded dashboard says otherwise)."""
- web = sys.modules.get("hermes_cli.web_server")
- return True if web is None else bool(getattr(web, "_DASHBOARD_EMBEDDED_CHAT_ENABLED", True))
- def _restart_wait_read(runner, scope) -> bool:
- from gateway.drain_control import restart_wait_read
- return scope['type'] == 'http' and restart_wait_read(runner, scope.get('method'), scope.get('path'))
- def _restart_wait_socket_ticket(runner, scope) -> bool:
- from gateway.drain_control import restart_wait_socket_ticket
- return scope['type'] == 'http' and restart_wait_socket_ticket(
- runner, scope.get('method'), scope.get('path'), scope.get('query_string', b'').decode('latin-1'))
- class GatewayRuntimeAPI:
- """Redeem private local tickets at the existing WS subprotocol boundary.
- Other credentials and HTTP routes retain the complete dashboard gate. Local
- bootstrap never grants an exposure/worker ticket interactive permissions.
- *web_app* is the dashboard mount (or any object whose ``state`` carries the listener's
- ``bound_host`` / ``auth_required`` / ``trusted_public_hosts``).
- """
- def __init__(self, app, runner, web_app):
- self.app, self.runner, self.web_app = app, runner, web_app
- async def __call__(self, scope, receive, send):
- if scope['type'] != 'websocket' or scope.get('path') != '/api/ws':
- return await self._serve(scope, receive, send)
- started = accepted = False
- async def send_watched(message):
- nonlocal started, accepted
- started = True
- accepted = accepted or message.get('type') == 'websocket.accept'
- await send(message)
- try:
- return await self._serve(scope, receive, send_watched)
- except Exception as exc:
- # Uvicorn answers an upgrade that raised before accept with HTTP 500 and logs it only to
- # its own logger, never gateway.log: the dialing cron worker or managed worker sees a bare
- # InvalidStatus (#139922). Name the cause here; uvicorn still sends the 500, but only when
- # nothing was sent yet: a handler that had closed the upgrade already got its 403 out.
- if not accepted:
- logger.error('Gateway refused a session WebSocket upgrade from %s %s: %s: %s',
- scope.get('hermes.gateway_socket_peer') or scope.get('client'),
- 'after its handler had refused it' if started else 'with HTTP 500',
- type(exc).__name__, exc, exc_info=True)
- raise
- async def _serve(self, scope, receive, send):
- descriptor = getattr(self.runner, 'session_runtime_descriptor', None)
- if scope['type'] not in {'http', 'websocket'} or descriptor is None:
- return await self.app(scope, receive, send)
- draining_socket = (descriptor['state'] in {'ready', 'draining'}
- and (self.runner._draining or descriptor['state'] == 'draining')
- and scope['type'] == 'websocket' and scope['path'] == '/api/ws')
- if ((descriptor['state'] != 'ready' or self.runner._draining) and not draining_socket
- and not _restart_wait_read(self.runner, scope)
- and not _restart_wait_socket_ticket(self.runner, scope)):
- if scope['type'] == 'websocket':
- await send({'type': 'websocket.close', 'code': 1013})
- else:
- from starlette.responses import JSONResponse
- await JSONResponse({'error': 'gateway_not_ready', 'state': descriptor['state']},
- status_code=503)(scope, receive, send)
- return
- # Capture before Uvicorn's proxy middleware rewrites scope.client (native grants, both kinds).
- scope['hermes.gateway_socket_peer'] = scope.get('client')
- from gateway.run_idle_exit import attached_client, note_client_activity
- if scope['type'] != 'websocket' or scope['path'] != '/api/ws':
- note_client_activity(self.runner) # dashboard/API traffic keeps an idle-exit gateway up
- return await self.app(scope, receive, send)
- original_receive = receive
- allow_draining = False
- admitted_in_wait = False
- from gateway.drain_control import restart_wait_open
- async def receive_admitted():
- message = await original_receive()
- # Only a socket admitted INTO the restart wait (to answer a parked prompt) outlives it. One
- # opened before the wait is cut as before: its client reconnects to the next owner, and a
- # held one (a kanban worker polling its card) would otherwise pin the drain to its cap.
- if ((descriptor['state'] != 'ready' or self.runner._draining) and not allow_draining
- and not (admitted_in_wait and restart_wait_open(self.runner))):
- return {'type': 'websocket.disconnect', 'code': 1013}
- return message
- receive = receive_admitted
- from starlette.websockets import WebSocket
- from hermes_cli.web_server_boundary import (
- _gateway_ws_ticket_from_subprotocol, ws_client_reason, ws_host_origin_reason,
- )
- state = getattr(self.web_app, 'state', None)
- scope['app'] = self.web_app
- ws = WebSocket(scope, receive, send)
- ticket, reason = _gateway_ws_ticket_from_subprotocol(ws)
- # Native grants are loopback-only, so a remote ticket is the Webapp's (the mobile app sends no
- # Origin, #147361). Origin is client-controlled, so this reaches no gate an Origin could not.
- native_peer = bool(ws.client) and ws.client.host in {'127.0.0.1', '::1'}
- if reason == 'none' or ws.headers.get('origin') or (reason == 'ok' and not native_peer):
- if draining_socket:
- # Let a Webapp tab's ticket reach its normal upgrade gate during restart wait: only a
- # surface that mints browser tickets redeems one there (elsewhere the loopback gate
- # takes ``?token=``).
- from hermes_cli.web_server_surface import policy
- admitted_in_wait = (reason == 'ok' and 'internal' not in ws.query_params
- and policy(state).gateway_ticket
- and restart_wait_open(self.runner))
- if not admitted_in_wait:
- await ws.close(code=1013)
- return
- return await self.app(scope, receive, send)
- if (reason != 'ok' or not _embedded_chat_enabled()
- or ws_host_origin_reason(ws, state) is not None or ws_client_reason(ws, state) is not None
- or ws.headers.get('origin') or not native_peer):
- await ws.close(code=4403)
- return
- operator = True
- try:
- grant = self.runner.session_ticket_store.redeem(ticket, profile_id=None, purpose='interactive')
- except PermissionError:
- operator = False
- try:
- grant = self.runner.session_ticket_store.redeem(ticket, profile_id=None, purpose='worker-adoption')
- except PermissionError:
- # Browser/OAuth tickets have a separate issuer.
- if draining_socket:
- await ws.close(code=1013)
- return
- return await self.app(scope, receive, send)
- allow_draining = not operator
- # Decided here, at admission, not at the mint: a restart wait that opened after the client's
- # discovery still refuses the machine poller that asked to stay out of it (a Kanban worker),
- # which would otherwise hold the turn the wait is waiting on.
- admitted_in_wait = restart_wait_open(self.runner) and not grant.get('refuse_restart_wait')
- # A restart's after-turn wait still owns its turns: an interactive client attaches to answer
- # a parked approval/clarify (#139041); stop() closes this window.
- if ((descriptor['state'] != 'ready' or self.runner._draining) and not allow_draining
- and not admitted_in_wait):
- await ws.close(code=1013)
- return
- from gateway.session_authorities import authority_for_profile_id
- authority = authority_for_profile_id(self.runner, grant['profile_id'])
- if authority is None:
- await ws.close(code=4403)
- return
- # The ticket names the served profile; this connection binds to that home's authority.
- scope['hermes.session_authority'] = authority
- from gateway.session_contract import CANONICAL_GATEWAY_PROTOCOL
- from tui_gateway.ws import handle_ws
- from gateway.session_authorities import owner_scope
- with owner_scope(authority), attached_client(self.runner):
- await handle_ws(ws, auth_identity={'user_id': grant['subject'], 'provider': 'local',
- 'profile_id': grant['profile_id'],
- 'instance_id': grant['instance_id'],
- 'capabilities': grant['capabilities'], 'native_bootstrap': True,
- 'profile_scope': grant.get('scope', 'profile')},
- subprotocol=CANONICAL_GATEWAY_PROTOCOL, operator=operator)
|