web_server_app.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236
  1. """HTTP lifespan composition; standalone backend services remain opt-in owners."""
  2. from contextlib import asynccontextmanager
  3. import asyncio
  4. import logging
  5. import os
  6. import threading
  7. from fastapi import FastAPI
  8. from hermes_cli.pty_session import run_reaper
  9. logger = logging.getLogger("hermes_cli.web_server")
  10. def _unlink_pty_markers(app: FastAPI) -> None:
  11. """Second pass after ``PTY_REGISTRY.close_all()``: drop channel markers left in app state,
  12. including stale paths from sessions reaped earlier (``PtySession.close()`` only removes
  13. markers owned by live registry sessions)."""
  14. from hermes_cli.web_server import _get_pty_active_session_files
  15. for marker in set(_get_pty_active_session_files(app).values()):
  16. try:
  17. marker.unlink(missing_ok=True)
  18. except OSError:
  19. pass
  20. @asynccontextmanager
  21. async def standalone_lifespan(app: "FastAPI"):
  22. from hermes_cli.web_server import (
  23. _eager_reconcile_own_session_db, _warm_gateway_module,
  24. _start_desktop_cron_ticker, _dashboard_selftest_loop,
  25. _auto_archive_ticker_loop, _terminate_desktop_managed_gateway,
  26. )
  27. from hermes_cli.web_server_chat import PTY_REGISTRY
  28. app.state.event_channels = {} # dict[str, set]
  29. app.state.event_lock = asyncio.Lock()
  30. app.state.pty_active_session_files = {} # dict[str, Path]
  31. # Serializes chat-argv resolution so concurrent /api/pty connections don't
  32. # overlap ``npm install`` / ``npm run build``. Locks live on app.state (not
  33. # module globals) so they bind to the running loop, not the import-time one.
  34. app.state.chat_argv_lock = asyncio.Lock()
  35. # Bring state.db schema current BEFORE the first session-list poll
  36. # (#79531/#80037): a store left behind by `hermes update` otherwise 500s
  37. # every poll until its owner completes schema reconciliation.
  38. # Off-thread so a locked store never delays the socket (Desktop
  39. # ready-probe times out at 10s, GH-73083). NOT a daemon, and joined at
  40. # shutdown: its sqlite connection must be closed by the thread that is
  41. # stepping it. A daemon copy that outlived the lifespan had its
  42. # connection closed from the main thread mid-probe (pytest's leaked-DB
  43. # sweep) and segfaulted the interpreter. The worker is time-bounded by
  44. # SessionDB's lock patience, so the join cannot hang shutdown.
  45. eager_reconcile_thread = threading.Thread(
  46. target=_eager_reconcile_own_session_db,
  47. name="statedb-eager-reconcile",
  48. )
  49. eager_reconcile_thread.start()
  50. # Import hermes_cli.gateway *before* the yield: on Windows + 3.11 the
  51. # import holds the GIL, so run_in_executor still froze the loop 15-22s and
  52. # the Desktop's 10s ready-probe timed out (GH-73083).
  53. _warm_gateway_module()
  54. # Snapshot the checkout revision so lazy-import paths (model picker) can
  55. # refuse with "restart required" after `hermes update` replaced the code
  56. # (#86207); the update flow does not reliably restart the dashboard.
  57. from gateway.code_skew import record_boot_fingerprint
  58. record_boot_fingerprint()
  59. # Hosted Bot rooms belong to the backend process. Recovery may need a
  60. # contended state.db migration, so keep it off the pre-yield path: Group
  61. # Chat must degrade on its own rather than block every Desktop feature.
  62. from tui_gateway import methods_groups as _hosted_groups
  63. import tui_gateway.server
  64. try:
  65. tui_gateway.server.install_tui_message_injector()
  66. except Exception:
  67. logger.warning("TUI message injector did not install", exc_info=True)
  68. hosted_room_start_cancel = threading.Event()
  69. def _start_hosted_rooms() -> None:
  70. try:
  71. _hosted_groups.start_hosted_room_service()
  72. except Exception:
  73. logger.exception("Hosted Group Chat recovery failed during backend startup")
  74. finally:
  75. if hosted_room_start_cancel.is_set():
  76. _hosted_groups.stop_hosted_room_service(timeout=1.0)
  77. hosted_room_start_thread = threading.Thread(
  78. target=_start_hosted_rooms,
  79. daemon=True,
  80. name="hosted-room-startup",
  81. )
  82. hosted_room_start_thread.start()
  83. # Desktop-spawned backends fire cron jobs themselves, since the app has no
  84. # gateway running the scheduler. Server `hermes dashboard` is unaffected —
  85. # it relies on its own gateway.
  86. from hermes_cli.process_identity import is_desktop_owned_backend
  87. cron_stop: threading.Event | None = None
  88. cron_thread: threading.Thread | None = None
  89. desktop_owned = is_desktop_owned_backend()
  90. if desktop_owned:
  91. # Reap an orphaned gateway from an abnormal previous exit (reparented to
  92. # launchd, still holding the platform WebSocket) before forking a fresh
  93. # one that would race the same credential (#77276). Runs
  94. # unconditionally; protection of a healthy standalone gateway lives
  95. # INSIDE the reaper (registration probed with cleanup_stale=False).
  96. # Startup grace: spare a gateway still claiming gateway.pid/lock (#122533).
  97. try:
  98. from hermes_cli.dashboard_procs import _REAP_MIN_AGE_SECONDS
  99. from hermes_cli.gateway import _reap_unsupervised_gateway_orphans
  100. _reap_unsupervised_gateway_orphans(min_age_s=_REAP_MIN_AGE_SECONDS)
  101. except Exception:
  102. logger.exception("Desktop startup: orphan gateway reap failed")
  103. cron_stop = threading.Event()
  104. cron_thread = threading.Thread(
  105. target=_start_desktop_cron_ticker,
  106. args=(cron_stop,),
  107. daemon=True,
  108. name="desktop-cron-ticker",
  109. )
  110. cron_thread.start()
  111. # Reap idle/dead keep-alive PTY sessions (30-min TTL). The registry is process-wide and
  112. # close_all() refuses new spawns, so a re-entered lifespan reopens it first.
  113. PTY_REGISTRY.reopen()
  114. pty_reaper_task = asyncio.create_task(run_reaper(PTY_REGISTRY))
  115. # Periodic authenticated self-test feeding the ``dashboard`` component on /api/status.
  116. selftest_task = asyncio.create_task(_dashboard_selftest_loop())
  117. # Live auto-archive timer, independent of list requests.
  118. auto_archive_task = asyncio.create_task(_auto_archive_ticker_loop())
  119. # Managed local runtime (local_runtime.enabled): bring llama-server back so a
  120. # restart doesn't strand a llamacpp main model. Off-thread and best-effort;
  121. # failure falls back to cloud providers like a cold start. Server only —
  122. # models load on first inference (an empty router holds no VRAM).
  123. def _boot_local_runtime():
  124. try:
  125. from hermes_cli.config import load_config
  126. from hermes_cli.local_runtime.bootstrap import ensure_local_runtime
  127. ensure_local_runtime(load_config())
  128. except Exception as exc:
  129. logging.getLogger(__name__).warning("local runtime boot failed: %s", exc)
  130. threading.Thread(target=_boot_local_runtime, daemon=True, name="local-runtime-boot").start()
  131. # Records who carries inference for setup.status / free_tier.status and broadcasts `setup.ready`
  132. # (no identity is created at boot). Off-thread; the desktop's first setup.status waits on the
  133. # record (bounded) instead.
  134. from hermes_cli.free_tier_bootstrap import start_background_bootstrap
  135. start_background_bootstrap()
  136. try:
  137. yield
  138. finally:
  139. try:
  140. tui_gateway.server.clear_tui_message_injector()
  141. except Exception:
  142. logger.debug("TUI message injector clear skipped", exc_info=True)
  143. hosted_room_start_cancel.set()
  144. _hosted_groups.stop_hosted_room_service(timeout=5.0)
  145. hosted_room_start_thread.join(timeout=1.0)
  146. if cron_stop is not None:
  147. cron_stop.set()
  148. pty_reaper_task.cancel()
  149. selftest_task.cancel()
  150. auto_archive_task.cancel()
  151. eager_reconcile_thread.join()
  152. await PTY_REGISTRY.close_all()
  153. # PtySession.close() removes markers owned by live registry sessions.
  154. # This second pass cleans any channel markers left in app state,
  155. # including stale paths from sessions reaped earlier.
  156. _unlink_pty_markers(app)
  157. # Stop the managed llama-server with its parent (an orphan pins VRAM).
  158. try:
  159. from hermes_cli.local_runtime.bootstrap import shutdown_local_runtime
  160. shutdown_local_runtime()
  161. except Exception:
  162. # Shutdown boundary: a supervisor that fails to stop must not skip the
  163. # managed-gateway teardown below; log it so the orphan is diagnosable.
  164. logger.warning("managed local runtime shutdown failed", exc_info=True)
  165. if desktop_owned:
  166. _terminate_desktop_managed_gateway()
  167. @asynccontextmanager
  168. async def http_lifespan(app: FastAPI):
  169. """The gateway already owns schema, rooms, scheduler and model services."""
  170. from contextlib import AsyncExitStack
  171. from hermes_cli.web_server import _dashboard_selftest_loop
  172. from hermes_cli.web_server_chat import PTY_REGISTRY
  173. from hermes_cli.web_server_surface import policy
  174. app.state.event_channels = {}
  175. app.state.event_lock = asyncio.Lock()
  176. app.state.pty_active_session_files = {}
  177. app.state.chat_argv_lock = asyncio.Lock()
  178. PTY_REGISTRY.reopen() # a prior lifespan's close_all() refuses spawns until reopened
  179. tasks = [
  180. asyncio.create_task(run_reaper(PTY_REGISTRY), name="gateway-api-pty-reaper"),
  181. asyncio.create_task(_dashboard_selftest_loop(), name="gateway-api-selftest"),
  182. ]
  183. try:
  184. async with AsyncExitStack() as surface:
  185. if policy(app.state).host_terminal:
  186. from hermes_cli.web_host_terminal_sessions import host_terminal_lifespan
  187. await surface.enter_async_context(host_terminal_lifespan(app))
  188. yield
  189. finally:
  190. for task in tasks:
  191. task.cancel()
  192. await asyncio.gather(*tasks, return_exceptions=True)
  193. await PTY_REGISTRY.close_all()
  194. _unlink_pty_markers(app)
  195. @asynccontextmanager
  196. async def app_lifespan(app: FastAPI):
  197. lifespan = http_lifespan if getattr(app.state, "gateway_runner", None) is not None else standalone_lifespan
  198. async with lifespan(app):
  199. yield