| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236 |
- """HTTP lifespan composition; standalone backend services remain opt-in owners."""
- from contextlib import asynccontextmanager
- import asyncio
- import logging
- import os
- import threading
- from fastapi import FastAPI
- from hermes_cli.pty_session import run_reaper
- logger = logging.getLogger("hermes_cli.web_server")
- def _unlink_pty_markers(app: FastAPI) -> None:
- """Second pass after ``PTY_REGISTRY.close_all()``: drop channel markers left in app state,
- including stale paths from sessions reaped earlier (``PtySession.close()`` only removes
- markers owned by live registry sessions)."""
- from hermes_cli.web_server import _get_pty_active_session_files
- for marker in set(_get_pty_active_session_files(app).values()):
- try:
- marker.unlink(missing_ok=True)
- except OSError:
- pass
- @asynccontextmanager
- async def standalone_lifespan(app: "FastAPI"):
- from hermes_cli.web_server import (
- _eager_reconcile_own_session_db, _warm_gateway_module,
- _start_desktop_cron_ticker, _dashboard_selftest_loop,
- _auto_archive_ticker_loop, _terminate_desktop_managed_gateway,
- )
- from hermes_cli.web_server_chat import PTY_REGISTRY
- app.state.event_channels = {} # dict[str, set]
- app.state.event_lock = asyncio.Lock()
- app.state.pty_active_session_files = {} # dict[str, Path]
- # Serializes chat-argv resolution so concurrent /api/pty connections don't
- # overlap ``npm install`` / ``npm run build``. Locks live on app.state (not
- # module globals) so they bind to the running loop, not the import-time one.
- app.state.chat_argv_lock = asyncio.Lock()
- # Bring state.db schema current BEFORE the first session-list poll
- # (#79531/#80037): a store left behind by `hermes update` otherwise 500s
- # every poll until its owner completes schema reconciliation.
- # Off-thread so a locked store never delays the socket (Desktop
- # ready-probe times out at 10s, GH-73083). NOT a daemon, and joined at
- # shutdown: its sqlite connection must be closed by the thread that is
- # stepping it. A daemon copy that outlived the lifespan had its
- # connection closed from the main thread mid-probe (pytest's leaked-DB
- # sweep) and segfaulted the interpreter. The worker is time-bounded by
- # SessionDB's lock patience, so the join cannot hang shutdown.
- eager_reconcile_thread = threading.Thread(
- target=_eager_reconcile_own_session_db,
- name="statedb-eager-reconcile",
- )
- eager_reconcile_thread.start()
- # Import hermes_cli.gateway *before* the yield: on Windows + 3.11 the
- # import holds the GIL, so run_in_executor still froze the loop 15-22s and
- # the Desktop's 10s ready-probe timed out (GH-73083).
- _warm_gateway_module()
- # Snapshot the checkout revision so lazy-import paths (model picker) can
- # refuse with "restart required" after `hermes update` replaced the code
- # (#86207); the update flow does not reliably restart the dashboard.
- from gateway.code_skew import record_boot_fingerprint
- record_boot_fingerprint()
- # Hosted Bot rooms belong to the backend process. Recovery may need a
- # contended state.db migration, so keep it off the pre-yield path: Group
- # Chat must degrade on its own rather than block every Desktop feature.
- from tui_gateway import methods_groups as _hosted_groups
- import tui_gateway.server
- try:
- tui_gateway.server.install_tui_message_injector()
- except Exception:
- logger.warning("TUI message injector did not install", exc_info=True)
- hosted_room_start_cancel = threading.Event()
- def _start_hosted_rooms() -> None:
- try:
- _hosted_groups.start_hosted_room_service()
- except Exception:
- logger.exception("Hosted Group Chat recovery failed during backend startup")
- finally:
- if hosted_room_start_cancel.is_set():
- _hosted_groups.stop_hosted_room_service(timeout=1.0)
- hosted_room_start_thread = threading.Thread(
- target=_start_hosted_rooms,
- daemon=True,
- name="hosted-room-startup",
- )
- hosted_room_start_thread.start()
- # Desktop-spawned backends fire cron jobs themselves, since the app has no
- # gateway running the scheduler. Server `hermes dashboard` is unaffected —
- # it relies on its own gateway.
- from hermes_cli.process_identity import is_desktop_owned_backend
- cron_stop: threading.Event | None = None
- cron_thread: threading.Thread | None = None
- desktop_owned = is_desktop_owned_backend()
- if desktop_owned:
- # Reap an orphaned gateway from an abnormal previous exit (reparented to
- # launchd, still holding the platform WebSocket) before forking a fresh
- # one that would race the same credential (#77276). Runs
- # unconditionally; protection of a healthy standalone gateway lives
- # INSIDE the reaper (registration probed with cleanup_stale=False).
- # Startup grace: spare a gateway still claiming gateway.pid/lock (#122533).
- try:
- from hermes_cli.dashboard_procs import _REAP_MIN_AGE_SECONDS
- from hermes_cli.gateway import _reap_unsupervised_gateway_orphans
- _reap_unsupervised_gateway_orphans(min_age_s=_REAP_MIN_AGE_SECONDS)
- except Exception:
- logger.exception("Desktop startup: orphan gateway reap failed")
- cron_stop = threading.Event()
- cron_thread = threading.Thread(
- target=_start_desktop_cron_ticker,
- args=(cron_stop,),
- daemon=True,
- name="desktop-cron-ticker",
- )
- cron_thread.start()
- # Reap idle/dead keep-alive PTY sessions (30-min TTL). The registry is process-wide and
- # close_all() refuses new spawns, so a re-entered lifespan reopens it first.
- PTY_REGISTRY.reopen()
- pty_reaper_task = asyncio.create_task(run_reaper(PTY_REGISTRY))
- # Periodic authenticated self-test feeding the ``dashboard`` component on /api/status.
- selftest_task = asyncio.create_task(_dashboard_selftest_loop())
- # Live auto-archive timer, independent of list requests.
- auto_archive_task = asyncio.create_task(_auto_archive_ticker_loop())
- # Managed local runtime (local_runtime.enabled): bring llama-server back so a
- # restart doesn't strand a llamacpp main model. Off-thread and best-effort;
- # failure falls back to cloud providers like a cold start. Server only —
- # models load on first inference (an empty router holds no VRAM).
- def _boot_local_runtime():
- try:
- from hermes_cli.config import load_config
- from hermes_cli.local_runtime.bootstrap import ensure_local_runtime
- ensure_local_runtime(load_config())
- except Exception as exc:
- logging.getLogger(__name__).warning("local runtime boot failed: %s", exc)
- threading.Thread(target=_boot_local_runtime, daemon=True, name="local-runtime-boot").start()
- # Records who carries inference for setup.status / free_tier.status and broadcasts `setup.ready`
- # (no identity is created at boot). Off-thread; the desktop's first setup.status waits on the
- # record (bounded) instead.
- from hermes_cli.free_tier_bootstrap import start_background_bootstrap
- start_background_bootstrap()
- try:
- yield
- finally:
- try:
- tui_gateway.server.clear_tui_message_injector()
- except Exception:
- logger.debug("TUI message injector clear skipped", exc_info=True)
- hosted_room_start_cancel.set()
- _hosted_groups.stop_hosted_room_service(timeout=5.0)
- hosted_room_start_thread.join(timeout=1.0)
- if cron_stop is not None:
- cron_stop.set()
- pty_reaper_task.cancel()
- selftest_task.cancel()
- auto_archive_task.cancel()
- eager_reconcile_thread.join()
- await PTY_REGISTRY.close_all()
- # PtySession.close() removes markers owned by live registry sessions.
- # This second pass cleans any channel markers left in app state,
- # including stale paths from sessions reaped earlier.
- _unlink_pty_markers(app)
- # Stop the managed llama-server with its parent (an orphan pins VRAM).
- try:
- from hermes_cli.local_runtime.bootstrap import shutdown_local_runtime
- shutdown_local_runtime()
- except Exception:
- # Shutdown boundary: a supervisor that fails to stop must not skip the
- # managed-gateway teardown below; log it so the orphan is diagnosable.
- logger.warning("managed local runtime shutdown failed", exc_info=True)
- if desktop_owned:
- _terminate_desktop_managed_gateway()
- @asynccontextmanager
- async def http_lifespan(app: FastAPI):
- """The gateway already owns schema, rooms, scheduler and model services."""
- from contextlib import AsyncExitStack
- from hermes_cli.web_server import _dashboard_selftest_loop
- from hermes_cli.web_server_chat import PTY_REGISTRY
- from hermes_cli.web_server_surface import policy
- app.state.event_channels = {}
- app.state.event_lock = asyncio.Lock()
- app.state.pty_active_session_files = {}
- app.state.chat_argv_lock = asyncio.Lock()
- PTY_REGISTRY.reopen() # a prior lifespan's close_all() refuses spawns until reopened
- tasks = [
- asyncio.create_task(run_reaper(PTY_REGISTRY), name="gateway-api-pty-reaper"),
- asyncio.create_task(_dashboard_selftest_loop(), name="gateway-api-selftest"),
- ]
- try:
- async with AsyncExitStack() as surface:
- if policy(app.state).host_terminal:
- from hermes_cli.web_host_terminal_sessions import host_terminal_lifespan
- await surface.enter_async_context(host_terminal_lifespan(app))
- yield
- finally:
- for task in tasks:
- task.cancel()
- await asyncio.gather(*tasks, return_exceptions=True)
- await PTY_REGISTRY.close_all()
- _unlink_pty_markers(app)
- @asynccontextmanager
- async def app_lifespan(app: FastAPI):
- lifespan = http_lifespan if getattr(app.state, "gateway_runner", None) is not None else standalone_lifespan
- async with lifespan(app):
- yield
|