web_server_lifecycle.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613
  1. """Serve-process lifecycle: parent death watchdog, port-conflict preflight, READY announcement, browser open, trusted proxies.
  2. """
  3. import asyncio
  4. import logging
  5. import ipaddress
  6. import os
  7. import signal
  8. import subprocess
  9. import sys
  10. import threading
  11. import time
  12. from pathlib import Path
  13. from typing import TYPE_CHECKING, Any, Optional
  14. from utils import atomic_json_write
  15. if TYPE_CHECKING: # pragma: no cover - annotation only
  16. import uvicorn
  17. # Same logger the code used before extraction (record parity).
  18. _log = logging.getLogger("hermes_cli.web_server")
  19. def _process_start_marker(pid: int) -> str:
  20. """Return a cross-runtime marker for the current incarnation of ``pid``.
  21. ``ProcessLookupError`` means the process is absent; other failures stay
  22. distinct so callers fail safe rather than killing a healthy backend.
  23. """
  24. if sys.platform == "linux":
  25. try:
  26. stat_line = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8")
  27. except FileNotFoundError as exc:
  28. raise ProcessLookupError(pid) from exc
  29. # Field 2 (comm) may contain spaces/parens; split after its final ')'
  30. # so field 3 is index 0 and field 22 (starttime) is index 19.
  31. fields = stat_line.rsplit(")", 1)[1].strip().split()
  32. if len(fields) < 20 or not fields[19].isdigit():
  33. raise OSError(f"invalid /proc stat data for PID {pid}")
  34. return f"linux:{fields[19]}"
  35. if os.name == "nt":
  36. import ctypes
  37. from ctypes import wintypes
  38. process_query_limited_information = 0x1000
  39. kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
  40. kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
  41. kernel32.OpenProcess.restype = wintypes.HANDLE
  42. kernel32.GetProcessTimes.argtypes = [wintypes.HANDLE] + [ctypes.POINTER(wintypes.FILETIME)] * 4
  43. kernel32.GetProcessTimes.restype = wintypes.BOOL
  44. kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
  45. kernel32.CloseHandle.restype = wintypes.BOOL
  46. handle = kernel32.OpenProcess(process_query_limited_information, False, pid)
  47. if not handle:
  48. error = ctypes.get_last_error()
  49. if error in (87, 1168): # invalid parameter / not found
  50. raise ProcessLookupError(pid)
  51. raise OSError(error, f"OpenProcess failed for PID {pid}")
  52. creation, exit_time, kernel, user = (wintypes.FILETIME() for _ in range(4))
  53. try:
  54. if not kernel32.GetProcessTimes(
  55. handle, ctypes.byref(creation), ctypes.byref(exit_time), ctypes.byref(kernel), ctypes.byref(user)
  56. ):
  57. raise OSError(ctypes.get_last_error(), f"GetProcessTimes failed for PID {pid}")
  58. finally:
  59. kernel32.CloseHandle(handle)
  60. filetime = (creation.dwHighDateTime << 32) | creation.dwLowDateTime
  61. return f"win:{filetime + 504911232000000000}"
  62. result = subprocess.run(["ps", "-p", str(pid), "-o", "lstart="], capture_output=True, text=True, encoding="utf-8",
  63. errors="replace", check=False)
  64. marker = result.stdout.strip()
  65. if result.returncode == 0 and marker:
  66. return f"ps:{marker}"
  67. # Only known "missing pid" signals become ProcessLookupError; anything else stays OSError so the
  68. # watchdog degrades to pid liveness instead of exiting on a healthy backend.
  69. if (result.returncode == 1 and not marker) or "no such process" in result.stderr.lower():
  70. raise ProcessLookupError(pid)
  71. raise OSError(f"ps could not inspect PID {pid}: {result.stderr.strip()}")
  72. def _valid_parent_start_marker(marker: str) -> bool:
  73. prefix, separator, value = marker.partition(":")
  74. if not separator or not value or value != value.strip():
  75. return False
  76. if prefix in ("linux", "win", "winms"):
  77. return value.isdigit()
  78. if prefix != "ps":
  79. return False
  80. # ``ps -o lstart=`` renders as ``Sat Aug 29 15:04:31 2026``. A marker split
  81. # on whitespace somewhere in the env plumbing arrives as ``ps:Sat`` -- a
  82. # real-looking identity that can never match, so the watchdog would exit a
  83. # live backend. Require a full value (#98132).
  84. tokens = value.split()
  85. has_year = any(token.isdigit() and len(token) == 4 for token in tokens)
  86. has_time = any(":" in token for token in tokens)
  87. return len(tokens) >= 4 and has_year and has_time
  88. def _parent_start_marker_mismatch_is_conclusive(actual: str, expected: str) -> bool:
  89. """Whether a marker mismatch proves the Desktop parent was replaced.
  90. ``linux:`` jiffies and ``win:``/``winms:`` FILETIME markers are machine
  91. values; a mismatch there is PID-reuse evidence. The POSIX fallback (macOS
  92. has no ``/proc``) is ``ps -o lstart=`` -- a wall-clock string rendered in
  93. the current TZ/locale. Electron caches it once per app lifetime while the
  94. backend re-renders it per spawn, so a timezone change (or DST, or column
  95. padding) makes the SAME instant differ byte-for-byte. Any ``ps:`` mismatch
  96. is therefore inconclusive (#95693, #93958).
  97. """
  98. return not (actual.startswith("ps:") or expected.startswith("ps:"))
  99. def _parent_start_markers_match(actual: str, expected: str) -> bool:
  100. """Compare parent markers across Desktop generations.
  101. Old Windows Desktop sends .NET ticks (``win:``); new builds send Electron's
  102. creation time in Unix ms (``winms:``) to avoid launching PowerShell. The
  103. backend reads the exact FILETIME and normalizes only for ``winms``.
  104. """
  105. if actual == expected:
  106. return True
  107. if not actual.startswith("win:") or not expected.startswith("winms:"):
  108. return False
  109. try:
  110. dotnet_ticks = int(actual.removeprefix("win:"))
  111. expected_unix_ms = int(expected.removeprefix("winms:"))
  112. except ValueError:
  113. return False
  114. dotnet_ticks_at_unix_epoch = 621_355_968_000_000_000
  115. actual_unix_ms = (dotnet_ticks - dotnet_ticks_at_unix_epoch) // 10_000
  116. return actual_unix_ms == expected_unix_ms
  117. def _warm_gateway_module() -> None:
  118. """Pre-import heavy modules so the event loop is not stalled on first use.
  119. Cold Windows installs pay .pyc compilation + Defender scans (15-30s) on
  120. these chains; the first WS RPC burst (setup.status, setup.runtime_check,
  121. gateway.ready→resolve_skin, model.options) pulled them in on the loop
  122. thread (#60800). Warm them all off-loop while the socket is already open.
  123. """
  124. for mod in (
  125. "hermes_cli.gateway",
  126. "hermes_cli.auth", # provider auth state
  127. "providers", # provider plugin discovery (below) imports every profile's module chain
  128. "hermes_cli.runtime_provider",
  129. "hermes_cli.skin_engine", # resolve_skin() config + skin engine init
  130. "hermes_cli.inventory", # provider catalogs + models.dev cache
  131. "hermes_cli.model_switch",
  132. ):
  133. try:
  134. __import__(mod)
  135. except Exception: # health: allow BLE001 -- best-effort warm-up; the real import reports
  136. _log.debug("warm-up import of %s failed", mod, exc_info=True)
  137. try:
  138. # Provider plugins carry their own auth chains (token exchange, CLI probes): warm them too.
  139. import providers
  140. providers.list_providers()
  141. except Exception: # health: allow BLE001 -- best-effort warm-up; discovery reports on real use
  142. _log.debug("warm-up provider discovery failed", exc_info=True)
  143. def _resolve_restart_drain_timeout() -> float:
  144. try:
  145. from hermes_cli.gateway import _get_restart_drain_timeout
  146. return _get_restart_drain_timeout()
  147. except ImportError:
  148. from gateway.restart import DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT
  149. return DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT
  150. def _eager_reconcile_own_session_db() -> None:
  151. """Bring this process's own state.db schema current at startup — read-only first.
  152. The dashboard is a view layer; the gateway owns the writer. A healthy store
  153. must never see a second writable ``SessionDB`` from this process (its
  154. close-time checkpoint and a possible FTS rebuild in ``_init_fts`` are the
  155. two-writer corruption vector, #107688 / #100896). Access-mode semantics
  156. (no bootstrap of a missing store; one additive catch-up of a store that lags
  157. SCHEMA_SQL, only while no gateway owns it, #79531 / #140315) live in the routers' own-store opener,
  158. :func:`hermes_cli.web_server_sessions._open_session_db_at_path`. Never
  159. raises: an unfixable store still gets the per-poll read-probe heal.
  160. """
  161. try:
  162. from hermes_cli.web_server_sessions import _open_session_db_for_profile
  163. from hermes_state_registry import release_or_close
  164. release_or_close(_open_session_db_for_profile(None, read_only=True))
  165. except Exception as exc:
  166. _log.warning(
  167. "startup schema reconcile of state.db failed (%s); session "
  168. "reads will report the store unavailable until owner recovery", exc,
  169. )
  170. def _read_bound_port(server: "uvicorn.Server", fallback: int) -> int:
  171. """Read the OS-assigned port from the live uvicorn socket (ephemeral port-0 discovery)."""
  172. if server.servers and server.servers[0].sockets:
  173. return server.servers[0].sockets[0].getsockname()[1]
  174. return fallback
  175. def _write_dashboard_ready_file(actual_port: int) -> None:
  176. """Publish the port through an atomic ready file when ``HERMES_DESKTOP_READY_FILE`` is set.
  177. Windows Desktop launches via ``pythonw.exe`` (no console flash) cannot use
  178. stdout for the port announcement, so Electron waits for this JSON instead.
  179. """
  180. target = os.environ.get("HERMES_DESKTOP_READY_FILE")
  181. if not target:
  182. return
  183. try:
  184. atomic_json_write(Path(target), {"port": int(actual_port)}, indent=None, separators=(",", ":"))
  185. except Exception as exc:
  186. _log.warning("Failed to write dashboard ready file %r: %s", target, exc)
  187. def _maybe_open_browser(host: str, actual_port: int, open_browser: bool, initial_profile: str) -> None:
  188. """Open the dashboard URL in the user's browser if appropriate.
  189. Skips headless Linux (no DISPLAY/WAYLAND_DISPLAY) so a TUI browser can't
  190. SIGHUP the server; maps ``0.0.0.0``/``::`` binds to ``127.0.0.1``.
  191. """
  192. if not open_browser:
  193. return
  194. import webbrowser
  195. _has_display = sys.platform != "linux" or bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"))
  196. if not _has_display:
  197. _log.debug(
  198. "Skipping browser-open: no DISPLAY or WAYLAND_DISPLAY detected "
  199. "(headless Linux). Pass --no-open to suppress this detection."
  200. )
  201. return
  202. from hermes_cli.url_utils import format_url_host
  203. _display_host = host if host not in ("0.0.0.0", "::") else "127.0.0.1"
  204. _open_url = f"http://{format_url_host(_display_host)}:{actual_port}"
  205. if initial_profile:
  206. from urllib.parse import quote
  207. _open_url += f"/?profile={quote(initial_profile)}"
  208. def _open():
  209. try:
  210. time.sleep(1.0)
  211. webbrowser.open(_open_url)
  212. except Exception:
  213. pass
  214. threading.Thread(target=_open, daemon=True).start()
  215. def _is_serve_orphaned(
  216. desktop_pid: int,
  217. expected_start_marker: Optional[str] = None,
  218. *,
  219. pid_exists=None,
  220. process_start_marker=None,
  221. ) -> bool:
  222. """True when the exact Desktop process that owns this backend is gone.
  223. ``HERMES_PARENT_PID`` is the Electron PID, not necessarily our PPID (the
  224. Windows ``hermes.exe`` launcher adds shims), so never compare getppid().
  225. The start marker (newer Desktops) defeats PID recycling; PID-only probing
  226. stays for older ones. Any inconclusive failure keeps serving (fail-safe).
  227. """
  228. try:
  229. if expected_start_marker is not None:
  230. probe = process_start_marker or _process_start_marker
  231. try:
  232. actual_marker = probe(int(desktop_pid))
  233. except ProcessLookupError:
  234. return True
  235. except Exception:
  236. actual_marker = None
  237. if actual_marker is not None:
  238. if _parent_start_markers_match(actual_marker, expected_start_marker):
  239. return False
  240. if _parent_start_marker_mismatch_is_conclusive(actual_marker, expected_start_marker):
  241. return True
  242. # Inconclusive marker: degrade to PID liveness instead of exiting.
  243. if pid_exists is None:
  244. from gateway.status import _pid_exists
  245. pid_exists = _pid_exists
  246. return not bool(pid_exists(int(desktop_pid)))
  247. except Exception:
  248. return False
  249. #: Ceiling on the post-orphan graceful unwind before the hard exit takes over.
  250. #: The desktop is already gone, so nothing escalates for us and this is the
  251. #: whole budget: comfortably past the 5s exit-flush budget
  252. #: (``HERMES_TUI_EXIT_FLUSH_BUDGET_S``), still short enough to keep the reap prompt.
  253. _ORPHAN_EXIT_CEILING_S = 10.0
  254. def _request_orphan_shutdown() -> threading.Timer:
  255. """Ask this backend to shut itself down, with a hard-exit backstop.
  256. ``os._exit`` here skipped the flush-on-kill handlers this same process
  257. installs before serving (#96095, ``install_exit_flush_signal_handlers``)
  258. and every ``atexit`` hook, so an unclean desktop exit dropped the in-memory
  259. transcripts that a SIGTERM from a live desktop persists to ``state.db``.
  260. Raise SIGTERM instead, and keep the reap guaranteed the way ``cli.py``'s
  261. ``_arm_exit_watchdog`` does: a daemon timer that ``os._exit``\\ s if the
  262. unwind stalls (it survives ``Py_FinalizeEx``'s non-daemon thread joins).
  263. ``raise_signal`` reaches Python's handler on Windows too, where
  264. ``os.kill(pid, SIGTERM)`` would terminate the process without running it.
  265. Returns the armed ceiling timer.
  266. """
  267. ceiling = threading.Timer(_ORPHAN_EXIT_CEILING_S, os._exit, args=(0,))
  268. ceiling.daemon = True
  269. ceiling.start()
  270. signal.raise_signal(signal.SIGTERM)
  271. return ceiling
  272. def _start_parent_death_watchdog() -> None:
  273. """Exit when the exact desktop parent that spawned this backend dies.
  274. Desktop passes its PID and, in newer versions, a start marker (defeats PID
  275. reuse) plus a per-spawn nonce (makes mixed-version plumbing fail safe:
  276. marker without nonce or vice versa disables the watchdog).
  277. """
  278. raw_pid = os.environ.get("HERMES_PARENT_PID")
  279. # Empty inherited values mean "absent", not "marker present but blank".
  280. start_marker = os.environ.get("HERMES_PARENT_START_MARKER") or None
  281. nonce = os.environ.get("HERMES_PARENT_NONCE") or None
  282. try:
  283. desktop_pid = int(raw_pid or "")
  284. except (TypeError, ValueError):
  285. return
  286. if desktop_pid <= 0:
  287. return
  288. has_marker = start_marker is not None
  289. if has_marker != (nonce is not None):
  290. return
  291. if has_marker and (not _valid_parent_start_marker(start_marker or "") or not nonce or nonce != nonce.strip()):
  292. # Disarming is fail-safe (the backend keeps serving) but must not be
  293. # traceless: this backend will never reap itself if the Desktop dies.
  294. _log.warning(
  295. "Parent-death watchdog disabled: unusable HERMES_PARENT_START_MARKER=%r / nonce; "
  296. "falling back to no parent tracking for desktop PID %s.",
  297. start_marker,
  298. desktop_pid,
  299. )
  300. return
  301. try:
  302. poll = max(0.5, float(os.environ.get("HERMES_SERVE_WATCHDOG_POLL_S", "2.0")))
  303. except (TypeError, ValueError):
  304. poll = 2.0
  305. def _loop() -> None:
  306. while not _is_serve_orphaned(desktop_pid, start_marker):
  307. time.sleep(poll)
  308. try:
  309. _log.warning(
  310. "Parent-death watchdog: desktop PID %s appears orphaned (expected_start_marker=%r); shutting down.",
  311. desktop_pid,
  312. start_marker,
  313. )
  314. except Exception:
  315. pass
  316. _request_orphan_shutdown()
  317. threading.Thread(target=_loop, daemon=True, name="serve-parent-watchdog").start()
  318. # Port-conflict sentinel (#93608): uvicorn's bind_socket() turns EADDRINUSE into
  319. # a bare ERROR + exit 1, indistinguishable from a crash for the Desktop spawn.
  320. # We probe the exact bind first and emit ONE machine-readable stdout line plus a
  321. # distinct exit code. 75 == BSD EX_TEMPFAIL, the codebase's "transient
  322. # environmental condition" convention (gateway/restart.py, kanban_db.py).
  323. PORT_IN_USE_EXIT_CODE = 75
  324. _PORT_IN_USE_SENTINEL = "BACKEND_PORT_IN_USE port={port}"
  325. def _is_addr_in_use_error(exc: OSError) -> bool:
  326. """True when ``exc`` is the platform's address-in-use bind failure."""
  327. import errno
  328. # POSIX, Linux, macOS, WinSock; WSAEADDRINUSE also surfaces as winerror.
  329. return exc.errno in {errno.EADDRINUSE, 98, 48, 10048} or getattr(exc, "winerror", None) == 10048
  330. def _port_bind_conflict(host: str, port: int) -> bool:
  331. """Probe whether binding ``host:port`` would fail with EADDRINUSE.
  332. ``port == 0`` (ephemeral) never conflicts, so it is skipped. Any probe
  333. error other than address-in-use returns ``False`` so uvicorn surfaces it
  334. with its normal diagnostics (bad host, EACCES, …).
  335. """
  336. if not port:
  337. return False
  338. import socket as _socket
  339. family = _socket.AF_INET6 if ":" in host else _socket.AF_INET
  340. try:
  341. probe = _socket.socket(family, _socket.SOCK_STREAM)
  342. except OSError:
  343. return False
  344. try:
  345. _exclusive = getattr(_socket, "SO_EXCLUSIVEADDRUSE", None)
  346. if sys.platform == "win32" and _exclusive is not None:
  347. # Windows SO_REUSEADDR binds over a live LISTEN socket and can never
  348. # detect a conflict; SO_EXCLUSIVEADDRUSE fails with 10048 exactly
  349. # when another socket holds the port (#93608).
  350. probe.setsockopt(_socket.SOL_SOCKET, _exclusive, 1)
  351. else:
  352. # Match uvicorn's own bind flags so the probe conflicts exactly when
  353. # its bind would: TIME_WAIT remnants pass, a live LISTEN fails.
  354. probe.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1)
  355. probe.bind((host, port))
  356. except OSError as exc:
  357. return _is_addr_in_use_error(exc)
  358. except Exception:
  359. return False
  360. finally:
  361. probe.close()
  362. return False
  363. def _write_machine_sentinel_line(line: str) -> None:
  364. """Write a machine-parsed sentinel line to the REAL stdout (fd 1).
  365. ``tui_gateway.server`` redirects ``sys.stdout`` to stderr at import (#94724),
  366. so a ``print()`` sentinel never reaches the Desktop's stdout pipe; fd 1 is
  367. untouched. If fd 1 is unwritable (pythonw.exe) fall back to ``print()`` for
  368. humans only — pythonw spawns discover the port via the ready file. Never
  369. raises.
  370. """
  371. try:
  372. os.write(1, (line + "\n").encode())
  373. except OSError:
  374. try:
  375. print(line, flush=True)
  376. except Exception:
  377. pass
  378. def _report_port_in_use(host: str, port: int) -> None:
  379. """Print the machine sentinel + a human hint naming likely holders."""
  380. _write_machine_sentinel_line(_PORT_IN_USE_SENTINEL.format(port=port))
  381. print(
  382. f" Port {port} on {host} is already in use — likely another "
  383. "'hermes serve' / 'hermes dashboard' backend or the Hermes gateway. "
  384. "Stop the other process, or pass --port <other> "
  385. "(--port 0 picks a free ephemeral port).",
  386. flush=True,
  387. )
  388. _DEFAULT_DASHBOARD_FORWARDED_ALLOW_IPS = ("127.0.0.1", "::1")
  389. def _dashboard_forwarded_allow_ips(dashboard_config: dict[str, Any]) -> list[str]:
  390. """Return the bounded proxy addresses uvicorn may trust: loopback plus valid config entries.
  391. Invalid or unbounded (/0, '*') ``dashboard.trusted_proxies`` entries fail
  392. closed so client-supplied forwarding headers never become request metadata.
  393. """
  394. configured = dashboard_config.get("trusted_proxies", [])
  395. if configured in (None, ""):
  396. configured = []
  397. elif isinstance(configured, str):
  398. configured = [configured]
  399. elif not isinstance(configured, (list, tuple)):
  400. _log.warning(
  401. "dashboard.trusted_proxies must be a list of IP addresses or CIDR networks; "
  402. "ignoring %r",
  403. configured,
  404. )
  405. configured = []
  406. trusted = list(_DEFAULT_DASHBOARD_FORWARDED_ALLOW_IPS)
  407. for raw_entry in configured:
  408. if not isinstance(raw_entry, str) or not raw_entry.strip():
  409. _log.warning(
  410. "Ignoring invalid dashboard.trusted_proxies entry %r; expected an IP "
  411. "address or CIDR network",
  412. raw_entry,
  413. )
  414. continue
  415. entry = raw_entry.strip()
  416. try:
  417. if "/" in entry:
  418. network = ipaddress.ip_network(entry, strict=False)
  419. if network.prefixlen == 0:
  420. raise ValueError("unbounded network")
  421. normalized = str(network)
  422. else:
  423. normalized = str(ipaddress.ip_address(entry))
  424. except ValueError:
  425. _log.warning(
  426. "Ignoring unsafe dashboard.trusted_proxies entry %r; use a bounded IP "
  427. "address or CIDR network, never '*' or a /0 network",
  428. raw_entry,
  429. )
  430. continue
  431. if normalized not in trusted:
  432. trusted.append(normalized)
  433. if trusted != list(_DEFAULT_DASHBOARD_FORWARDED_ALLOW_IPS):
  434. _log.info("Dashboard trusted proxies: %s", ", ".join(trusted))
  435. return trusted
  436. def _uvicorn_console_logging() -> None:
  437. """uvicorn's default console logging, without its ``dictConfig``.
  438. ``uvicorn.Config``'s default ``log_config`` runs ``dictConfig``, which calls
  439. ``logging.shutdown()`` on every live handler: a multiplexed gateway's profile log routers are
  440. already installed when its listener is built, and a closed router stays closed. The listeners
  441. pass ``log_config=None`` and get uvicorn's handlers here instead: ``uvicorn`` to stderr,
  442. ``uvicorn.access`` to stdout, neither propagating into the Hermes log files.
  443. """
  444. from uvicorn.config import LOGGING_CONFIG
  445. from uvicorn.logging import AccessFormatter, DefaultFormatter
  446. formats = LOGGING_CONFIG["formatters"]
  447. for name, stream, formatter in (
  448. ("uvicorn", sys.stderr, DefaultFormatter(formats["default"]["fmt"])),
  449. ("uvicorn.access", sys.stdout, AccessFormatter(formats["access"]["fmt"])),
  450. ):
  451. logger = logging.getLogger(name)
  452. logger.propagate = False
  453. if not logger.handlers: # a second listener in this process keeps the first one's
  454. handler = logging.StreamHandler(stream)
  455. handler.setFormatter(formatter)
  456. logger.addHandler(handler)
  457. # LOGGING_CONFIG's level for the parent; uvicorn.Config sets only the child loggers' levels.
  458. logging.getLogger("uvicorn").setLevel(logging.INFO)
  459. def build_listener_server(served_app, host: str, port: int, *, auth_required: bool,
  460. ws_ping: Optional[tuple[float, float]] = None,
  461. timeout_graceful_shutdown: Optional[float] = 3, lifespan: str = "auto"):
  462. """uvicorn ``Config`` + ``Server`` for a dashboard or gateway-session listener (FastAPI-free, so
  463. the gateway can build its session listener without importing the dashboard app).
  464. WS keepalive ping runs ON the agent event loop; a GIL-holding worker call can starve it for
  465. minutes, so uvicorn misses the pong and drops a healthy local socket (#53773/#48445/#50005).
  466. The ping only detects half-open connections (proxy 524, dropped tunnels), impossible on
  467. loopback where a dead client sends a real FIN/RST -> WebSocketDisconnect. So: no ping on
  468. loopback; non-loopback sits behind a Cloudflare Tunnel (~100s idle) and keeps a config-driven
  469. cadence (dashboard.ws_ping_interval/_timeout, #79635) defaulting to 20/20. *ws_ping* overrides
  470. both (SSH-isolated Desktop backends).
  471. """
  472. import uvicorn
  473. from hermes_cli.config import load_config
  474. from hermes_cli.web_server_boundary import _DESKTOP_ATTACHMENT_WS_MAX_BYTES, _LOOPBACK_HOST_VALUES
  475. try:
  476. dash_cfg = load_config().get("dashboard") or {}
  477. except Exception: # health: allow BLE001 -- same fail-open read as web_server._build_uvicorn_server: a broken config.yaml must not stop the listener
  478. dash_cfg = {}
  479. def _ws_ping_setting(key: str, default: float = 20.0) -> float:
  480. try:
  481. return float(dash_cfg.get(key, default))
  482. except (TypeError, ValueError):
  483. return default
  484. # permessage-deflate runs zlib on the event loop for every outbound frame, and the gateway sends
  485. # one frame per streamed token of every running session: under load it was the largest single
  486. # cost on the loop. Loopback bytes are free, so only a listener remote peers reach (a non-loopback
  487. # bind, or a loopback bind behind a public URL / tunnel, which engages the auth gate) compresses.
  488. per_message_deflate = auth_required or host not in _LOOPBACK_HOST_VALUES
  489. if ws_ping is None:
  490. ws_ping = (None, None) if host in _LOOPBACK_HOST_VALUES else (
  491. _ws_ping_setting("ws_ping_interval"), _ws_ping_setting("ws_ping_timeout"))
  492. _uvicorn_console_logging()
  493. config = uvicorn.Config(
  494. served_app, host=host, port=port, log_level="warning", log_config=None,
  495. # Off by default so _ws_client_is_allowed sees the real peer, not
  496. # X-Forwarded-For. Gated mode runs behind a TLS terminator and needs
  497. # X-Forwarded-Proto for cookie Secure flags.
  498. proxy_headers=auth_required,
  499. # Loopback-only unless the operator trusts a bounded upstream proxy, so
  500. # spoofed X-Forwarded-* from arbitrary callers is never honoured.
  501. forwarded_allow_ips=_dashboard_forwarded_allow_ips(dash_cfg),
  502. ws_ping_interval=ws_ping[0],
  503. ws_ping_timeout=ws_ping[1],
  504. ws_per_message_deflate=per_message_deflate,
  505. ws_max_size=_DESKTOP_ATTACHMENT_WS_MAX_BYTES,
  506. # Desktop sends a single SIGTERM and escalates to SIGKILL ~5s later;
  507. # uvicorn's default (None) waits on lingering ASGI tasks forever, so a
  508. # mid-turn request orphans the backend past that budget (#76244). 3s
  509. # leaves room for lifespan shutdown + cron_stop before the kill.
  510. timeout_graceful_shutdown=timeout_graceful_shutdown,
  511. lifespan=lifespan,
  512. )
  513. return config, uvicorn.Server(config)