run_bootstrap.py 52 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918
  1. """Process startup phases; diagnostics remain in the gateway facade."""
  2. from __future__ import annotations
  3. import logging
  4. import threading
  5. import time
  6. from pathlib import Path
  7. from typing import Any, Optional
  8. from gateway.config import GatewayConfig
  9. def _start_deferred_state_db_integrity_check(evidence: dict) -> None:
  10. """Run the store scan an unclean-exit report deferred, once the runner is live.
  11. On the event loop it held every local listener open but unanswered for the whole
  12. ``quick_check`` (minutes on a multi-GB ``state.db``), so local clients hung instead of being
  13. refused. A daemon thread nothing waits on; it only appends the forensic exit-diag record.
  14. """
  15. from agent.memory_provider import spawn_context_thread
  16. from gateway.lifecycle_ledger import STATE_DB_INTEGRITY_DEADLINE_S, run_deferred_integrity_check
  17. # Under the caller's profile scope: the scan resolves this gateway's state.db through it.
  18. spawn_context_thread(
  19. run_deferred_integrity_check, args=(evidence,),
  20. kwargs={"deadline_s": STATE_DB_INTEGRITY_DEADLINE_S}, name="state-db-integrity-check").start()
  21. # Log-record parity with the origin module.
  22. logger = logging.getLogger("gateway.run")
  23. async def _start_gateway_replace_existing_instance(existing_pid: int, replace: bool) -> bool:
  24. """Handle a live gateway PID under this HERMES_HOME: replace it (``--replace``) or refuse.
  25. Returns False when startup must abort (refused, permission denied, target still alive)."""
  26. from gateway.run import (_clear_takeover_marker_quiet, _replace_target_belongs_to_other_profile, _wait_for_pid_exit, get_hermes_home, suppress)
  27. from gateway.status import get_process_start_time, remove_pid_file, terminate_pid
  28. if not replace:
  29. hermes_home = str(get_hermes_home())
  30. logger.error(
  31. "Another gateway instance is already running (PID %d, HERMES_HOME=%s) and did not "
  32. "publish a host record this process could attach to.",
  33. existing_pid, hermes_home)
  34. print(
  35. f"\n❌ A gateway already owns this host (PID {existing_pid}).\n"
  36. f" One gateway per host serves every profile, so there is nothing to start here.\n"
  37. f" Attach is impossible: PID {existing_pid} published no usable host record\n"
  38. f" (an older build, or an unwritable lock directory).\n"
  39. f" Take the host over: hermes gateway run --replace\n"
  40. f" Or stop it first: hermes gateway stop\n")
  41. return False
  42. # Never signal a process not provably ours (a poisoned PID record → cross-profile restart loop).
  43. if _replace_target_belongs_to_other_profile(existing_pid):
  44. from gateway.status import _get_process_hermes_home, write_runtime_status
  45. logger.error(
  46. "Refusing --replace: PID %d cannot be proven to belong "
  47. "to this profile's gateway (HERMES_HOME %s). Remove the "
  48. "stale PID record or stop the owning profile explicitly.",
  49. existing_pid, _get_process_hermes_home())
  50. # The stamp must be ON DISK before this process exits: every CLI exit path funnels
  51. # through _exit_after_graceful_shutdown(), whose os._exit() never flushes the queued
  52. # runtime-status writer. A fire-and-forget stamp can die in the queue, the supervisor
  53. # retries into the same stale record, and the loop this refusal exists to break keeps
  54. # going (#135392). Bounded wait: a stalled disk delays the exit by at most 5 s, it
  55. # never hangs the start.
  56. try:
  57. write_runtime_status(
  58. gateway_state="startup_failed",
  59. exit_reason="gateway_replace_ownership_unproven",
  60. wait_timeout=5.0,
  61. )
  62. except Exception:
  63. # Persistence must never abort the start; the traceback (DEBUG) keeps the
  64. # failure visible for the next bug report.
  65. logger.debug("Could not stamp runtime status after the ownership refusal", exc_info=True)
  66. return False
  67. existing_start_time = get_process_start_time(existing_pid)
  68. logger.info("Replacing existing gateway instance (PID %d) with --replace.", existing_pid)
  69. # Takeover marker: target exits 0 on our SIGTERM (exit 1 → systemd Restart=on-failure flap loop).
  70. try:
  71. from gateway.status import write_takeover_marker
  72. write_takeover_marker(existing_pid)
  73. except Exception as e:
  74. logger.debug("Could not write takeover marker: %s", e)
  75. # Snapshot children BEFORE signalling: reparented orphans are invisible yet hold scoped token locks.
  76. try:
  77. from gateway.status import _snapshot_gateway_children
  78. _old_gateway_children = _snapshot_gateway_children(existing_pid)
  79. except Exception:
  80. _old_gateway_children = []
  81. try:
  82. terminate_pid(existing_pid, force=False)
  83. except ProcessLookupError:
  84. pass # Already gone
  85. except (PermissionError, OSError):
  86. logger.error("Permission denied killing PID %d. Cannot replace.", existing_pid)
  87. _clear_takeover_marker_quiet()
  88. return False
  89. # Up to 10s for SIGTERM, then SIGKILL.
  90. if not await _wait_for_pid_exit(existing_pid, 20, 0.5):
  91. logger.warning("Old gateway (PID %d) did not exit after SIGTERM, sending SIGKILL.", existing_pid)
  92. old_gateway_exited = False
  93. try:
  94. terminate_pid(existing_pid, force=True, expected_start_time=existing_start_time)
  95. except ProcessLookupError:
  96. old_gateway_exited = True
  97. except (PermissionError, OSError):
  98. pass
  99. # Confirm SIGKILL took (D-state/zombie) before clearing PID/locks, or two gateways share a token.
  100. if not old_gateway_exited and not await _wait_for_pid_exit(existing_pid, 20, 0.25):
  101. logger.error(
  102. "Old gateway (PID %d) still appears alive after SIGKILL; "
  103. "aborting replacement to avoid a duplicate gateway.", existing_pid)
  104. _clear_takeover_marker_quiet()
  105. return False
  106. # Reap orphaned children (POSIX; mirrors Windows taskkill /T) so they stop holding scoped token locks.
  107. try:
  108. from gateway.status import reap_gateway_children
  109. reap_gateway_children(_old_gateway_children, parent_pid=existing_pid)
  110. except Exception:
  111. logger.debug("Child reap for replaced gateway PID %d failed", existing_pid, exc_info=True)
  112. remove_pid_file()
  113. # remove_pid_file() is a no-op when the PID doesn't match; force-unlink covers a crashed old process.
  114. with suppress(Exception):
  115. (get_hermes_home() / "gateway.pid").unlink(missing_ok=True)
  116. # The old process may not have consumed the marker (SIGKILL'd before its handler read it).
  117. _clear_takeover_marker_quiet()
  118. # Stopped (Ctrl+Z) processes don't release scoped locks on exit; stale lock files block the new gateway.
  119. try:
  120. from gateway.status_scoped_locks import release_all_scoped_locks
  121. _released = release_all_scoped_locks(owner_pid=existing_pid, owner_start_time=existing_start_time)
  122. if _released:
  123. logger.info("Released %d stale scoped lock(s) from old gateway.", _released)
  124. except Exception:
  125. logger.debug("Stale scoped lock release for replaced gateway PID %d failed", existing_pid, exc_info=True)
  126. return True
  127. def _start_gateway_configure_logging(verbosity: Optional[int]) -> None:
  128. """Sync bundled skills, set up file logging + startup security audit, and the -v/-q stderr handler."""
  129. from gateway.run import (_best_effort, _gateway_stderr_formatter, _hermes_home, logging)
  130. def _sync_skills() -> None:
  131. from tools.skills_sync import sync_skills
  132. sync_skills(quiet=True)
  133. _best_effort(_sync_skills)
  134. # Centralized logging (agent.log INFO+, errors.log WARNING+, gateway.log gateway-only); idempotent.
  135. from hermes_logging import setup_logging, _safe_stderr
  136. setup_logging(hermes_home=_hermes_home, mode="gateway")
  137. def _security_audit() -> None:
  138. # Warn-on-load, never blocks: surfaces root / weak-SSH / unauthenticated-listener exposure.
  139. from hermes_cli.security_audit_startup import log_startup_security_warnings
  140. def _raw_cfg():
  141. from hermes_cli.config import read_raw_config
  142. return read_raw_config()
  143. log_startup_security_warnings(hermes_home=_hermes_home, config=_best_effort(_raw_cfg))
  144. _best_effort(_security_audit, "Startup security audit failed (non-fatal): %s")
  145. # Optional stderr handler from -v/-q: None (quiet) = none; 0 = WARNING; 1 = INFO; 2+ = DEBUG.
  146. if verbosity is not None:
  147. _stderr_level = {0: logging.WARNING, 1: logging.INFO}.get(verbosity, logging.DEBUG)
  148. _stderr_handler = logging.StreamHandler(_safe_stderr())
  149. _stderr_handler.setLevel(_stderr_level)
  150. _stderr_handler.setFormatter(_gateway_stderr_formatter())
  151. root = logging.getLogger()
  152. root.addHandler(_stderr_handler)
  153. if _stderr_level < root.level: # so DEBUG records can reach the handler
  154. root.setLevel(_stderr_level)
  155. def _start_gateway_make_shutdown_signal_handler(runner, _signal_initiated_shutdown: list):
  156. """Build the SIGINT/SIGTERM handler; ``_signal_initiated_shutdown[0]`` records an unplanned signal."""
  157. from gateway.run import (_best_effort, _hermes_home, asyncio, logger, signal)
  158. planned_stop_seen = [False]
  159. def shutdown_signal_handler(received_signal=None):
  160. # Planned --replace takeover (sibling marked this PID): exit 0 so systemd won't revive us.
  161. def _takeover() -> bool:
  162. from gateway.status import consume_takeover_marker_for_self
  163. return consume_takeover_marker_for_self()
  164. # Planned stop: CLI marks first, else its SIGTERM looks like an external kill. SIGINT = Ctrl+C.
  165. def _planned_stop() -> bool:
  166. from gateway.status import consume_planned_stop_marker_for_self
  167. return consume_planned_stop_marker_for_self()
  168. # Fast (<10ms) sync snapshot: stdlib + /proc, no subprocesses (`ps aux` here once blocked ~3s).
  169. def _snapshot():
  170. from gateway.shutdown_forensics import snapshot_shutdown_context
  171. return snapshot_shutdown_context(received_signal)
  172. planned_takeover = bool(_best_effort(_takeover, "Takeover marker check failed: %s"))
  173. planned_stop = received_signal == signal.SIGINT or (
  174. not planned_takeover and bool(_best_effort(_planned_stop, "Planned stop marker check failed: %s")))
  175. # `hermes gateway stop` writes the marker, THEN signals: the planned-stop watcher can consume
  176. # the marker in between, and the CLI's own SIGTERM must not then read as an external kill.
  177. if planned_stop:
  178. planned_stop_seen[0] = True
  179. elif planned_stop_seen[0] and not planned_takeover:
  180. planned_stop = True
  181. _shutdown_ctx = _best_effort(_snapshot, "snapshot_shutdown_context failed: %s")
  182. sig_name = _shutdown_ctx["signal"] if _shutdown_ctx else None
  183. if planned_takeover:
  184. logger.info("Received %s as a planned --replace takeover — exiting cleanly", sig_name or "SIGTERM")
  185. elif planned_stop:
  186. logger.info("Received %s as a planned gateway stop — exiting cleanly", sig_name or "SIGTERM/SIGINT")
  187. else:
  188. # Mirrored onto the runner so _stop_impl suppresses the gateway_state=stopped persist for
  189. # unexpected signals; operator stops take the `planned_stop` branch and leave it False (DO persist).
  190. _signal_initiated_shutdown[0] = runner._signal_initiated_shutdown = True
  191. logger.info("Received %s — initiating shutdown", sig_name or "SIGTERM/SIGINT")
  192. if _shutdown_ctx is not None:
  193. def _log_context() -> None:
  194. # The most useful line for "gateway keeps dying" tickets.
  195. from gateway.shutdown_forensics import format_context_for_log
  196. logger.warning("Shutdown context: %s", format_context_for_log(_shutdown_ctx))
  197. def _diagnostic() -> None:
  198. # Heavyweight (ps auxf, pstree, dmesg), detached so it finishes even if our cgroup is torn
  199. # down; bounded by an internal timeout, never blocks.
  200. from gateway.shutdown_forensics import spawn_async_diagnostic
  201. spawn_async_diagnostic(
  202. _hermes_home / "logs" / "gateway-shutdown-diag.log", _shutdown_ctx["signal"], timeout_seconds=5.0)
  203. _best_effort(_log_context, "format_context_for_log failed: %s")
  204. _best_effort(_diagnostic, "spawn_async_diagnostic failed: %s")
  205. if not planned_takeover:
  206. # Supervisor/operator SIGNAL stop (bootout, kickstart -k, systemd, s6, bare kill) — the
  207. # only kind launchd times with ExitTimeOut. In-band SIGUSR1 restarts never pass through
  208. # here, and a sibling-driven --replace takeover is not launchd-timed either, so both
  209. # keep the configured drain. _stop_impl uses this to cap the drain to the live budget.
  210. runner._stop_requested_by_signal = True
  211. asyncio.create_task(runner.stop())
  212. return shutdown_signal_handler
  213. def _start_gateway_claim_pid_file(force: bool = False) -> bool:
  214. """Claim the runtime lock + PID file (O_EXCL winner is the authoritative gateway). False = lost."""
  215. from gateway.run import (_claim_host_gateway_role, logger, os)
  216. import atexit
  217. from gateway.status import (
  218. acquire_gateway_runtime_lock, get_running_pid, release_gateway_runtime_lock,
  219. remove_pid_file, write_pid_file)
  220. _current_pid = get_running_pid()
  221. if _current_pid is not None and _current_pid != os.getpid():
  222. logger.error("Another gateway instance (PID %d) started during our startup. "
  223. "Exiting to avoid double-running.", _current_pid)
  224. return False
  225. if not acquire_gateway_runtime_lock():
  226. logger.error("Gateway runtime lock is already held by another instance. Exiting.")
  227. return False
  228. try:
  229. write_pid_file()
  230. except FileExistsError:
  231. release_gateway_runtime_lock()
  232. logger.error("PID file race lost to another gateway instance. Exiting.")
  233. return False
  234. atexit.register(remove_pid_file)
  235. atexit.register(release_gateway_runtime_lock)
  236. from hermes_cli.gateway_multiplex_mode import publish_pending_multiplex_decision
  237. publish_pending_multiplex_decision()
  238. _claim_host_gateway_role(force=force)
  239. return True
  240. async def _start_gateway_start_control_socket(runner):
  241. """Start the gateway control socket (identify/status/pause-for-update); None when unavailable."""
  242. from gateway.run import (asyncio, logger, os, threading)
  243. import atexit
  244. import concurrent.futures
  245. _control_server = None
  246. try:
  247. # Started immediately after the PID-file claim: winning that O_EXCL race is the moment this process
  248. # becomes the authoritative gateway for its HERMES_HOME, so from here on "does a socket answer?" is
  249. # a truthful liveness/identity query for updater and fleet consumers. Returns None on failure and
  250. # start_gateway aborts startup: the listener mints every local client's attach ticket, so a
  251. # gateway without it can serve no local session. See #92091.
  252. from gateway.control_socket import GatewayControlServer, build_identify_payload
  253. from gateway.run_profile_reconcile import (
  254. migrate_profile_identity_verb, purge_profile_identity_verb,
  255. unserve_profile_verb, serve_profile_verb,
  256. )
  257. from gateway.run_plugin_rewire import reload_plugins_verb
  258. from gateway.run_active_work_verb import active_work_verb
  259. from gateway.relay.link import relay_link_verb
  260. from gateway.drain_control import RESTART_WAIT_REASON
  261. descriptor = runner.session_runtime_descriptor
  262. def _identify_runtime():
  263. payload = build_identify_payload()
  264. payload.update({key: descriptor[key] for key in (
  265. "instance_id", "runtime_protocol", "authority_epoch", "state", "api_origin",
  266. "served_profiles", "parked_profiles", "capabilities") if key in descriptor})
  267. if getattr(runner, '_draining', False):
  268. payload.update(state='draining')
  269. # The restart's after-turn wait still owns its turns: clients reach it to answer a
  270. # parked approval/clarify (#139041), so the endpoint stays dialable until stop().
  271. if descriptor.get('drain_reason') != RESTART_WAIT_REASON:
  272. payload['capabilities'] = []
  273. if payload.get('state') == 'draining' and descriptor.get('drain_reason'):
  274. payload['drain_reason'] = descriptor['drain_reason']
  275. payload["supervisor"] = {"manual": "none", "desktop": "none"}.get(
  276. payload.get("supervisor"), payload.get("supervisor", "none"))
  277. return payload
  278. # pause-for-update: the updater asks us to drain + exit (freeing venv handles) vs. a tree-kill
  279. # (same path as SIGUSR1). Handler runs on the socket executor thread, so marshal onto the loop.
  280. # pause-for-update (#92091 step 2): the updater asks this gateway to drain in-flight turns and exit
  281. # cleanly — releasing every venv file handle — instead of being tree-killed mid-turn. Same drain
  282. # path as SIGUSR1/service restarts (request_restart(via_service=True)); the updater (or the service
  283. # manager) relaunches after the code swap.
  284. _main_loop = asyncio.get_running_loop()
  285. def _pause_for_update_handler() -> dict:
  286. try:
  287. from hermes_cli.gateway import _get_restart_drain_timeout
  288. _drain = float(_get_restart_drain_timeout())
  289. except Exception:
  290. _drain = 30.0
  291. accepted_box: list[bool] = []
  292. _done = threading.Event()
  293. def _request() -> None:
  294. try:
  295. accepted_box.append(runner.request_restart(detached=False, via_service=True))
  296. finally:
  297. _done.set()
  298. _main_loop.call_soon_threadsafe(_request)
  299. _done.wait(timeout=5.0)
  300. accepted = bool(accepted_box and accepted_box[0])
  301. return {
  302. "pausing": accepted, "already_stopping": not accepted,
  303. "pid": os.getpid(), "drain_timeout": _drain}
  304. def _rescan_profiles_handler() -> dict:
  305. """``hermes profile create/delete`` asks the multiplexer to reconcile ``profiles/`` now
  306. (the watcher also rescans periodically). Runs on the socket executor: marshal onto the loop
  307. and wait briefly so the caller learns whether the profile is served. A gateway standalone
  308. only because the host had one profile starts multiplexing in place on this rescan."""
  309. future = asyncio.run_coroutine_threadsafe(
  310. runner.reconcile_served_profiles(reason="control-socket"), _main_loop)
  311. try:
  312. # Bounded: a token-less create reconciles in milliseconds; a credential-add whose adapter
  313. # connect outlasts this keeps running and the caller sees ``pending`` (not an error).
  314. result = future.result(timeout=5.0)
  315. except concurrent.futures.TimeoutError:
  316. result = {"pending": True, "served_profiles": runner.served_profile_names()}
  317. return {"multiplex": bool(getattr(runner.config, "multiplex_profiles", False)), **result}
  318. _control_server = GatewayControlServer(
  319. verb_handlers={"pause-for-update": _pause_for_update_handler, "identify": _identify_runtime,
  320. "rescan-profiles": _rescan_profiles_handler,
  321. "migrate-profile-identity": migrate_profile_identity_verb(runner),
  322. "unserve-profile": unserve_profile_verb(runner),
  323. "serve-profile": serve_profile_verb(runner),
  324. "purge-profile-identity": purge_profile_identity_verb(runner),
  325. # `hermes gateway stop/restart` refuse while this names live work.
  326. "active-work": active_work_verb(runner, _main_loop),
  327. # A plugin installed/enabled by another process loads now and re-wires the
  328. # live adapters' handlers (#87770); tools/prompt still wait for the next session.
  329. "reload-plugins": reload_plugins_verb(runner, _main_loop),
  330. # The per-gateway relay secret exists only in this process: the dashboard
  331. # mints Team Gateway link codes through it.
  332. "relay-link": relay_link_verb})
  333. _control_server.ticket_store = runner.session_ticket_store
  334. # The Webapp's verbs answer from the socket's first moment ("starting" until the listener
  335. # settles): a client polling a restarting gateway must never read "unknown verb" as a failure.
  336. from gateway.webapp_listener import register_control_verbs
  337. from hermes_constants import get_hermes_home
  338. register_control_verbs(runner, get_hermes_home(), server=_control_server)
  339. if not await _control_server.start():
  340. _control_server = None
  341. else:
  342. atexit.register(_control_server.cleanup_files)
  343. # Hosted-room transports install onto this listener; bind it here so every launcher
  344. # that starts the control socket (not only start_gateway) exposes it.
  345. runner.session_control_server = _control_server
  346. except Exception as _cs_exc:
  347. logger.error("Control socket startup failed; the gateway cannot start without it: %s", _cs_exc)
  348. _control_server = None
  349. return _control_server
  350. def _start_gateway_start_cron_and_housekeeping(runner):
  351. """Start the cron scheduler thread + gateway housekeeping thread; returns
  352. ``(cron_stop, cron_provider, cron_thread, housekeeping_thread)``."""
  353. from gateway.run import (Dict, Platform, _cron_profile_gate, _cron_tick_profile_homes, _start_gateway_housekeeping, asyncio, logger, threading)
  354. # The event loop is passed so cron delivery can use live adapters (E2EE support).
  355. from cron.scheduler_provider import (
  356. InProcessCronScheduler, resolve_cron_scheduler, scheduler_for_profile_mode)
  357. cron_stop = threading.Event()
  358. # ONE gateway process per host multiplexes every profile, so its cron ticker owns EVERY
  359. # profile's store — `gateway.multiplex_profiles` gates adapters, not cron. Gating the tick set
  360. # on that flag left every non-launch profile's jobs in a store no ticker visited: they
  361. # silently never fired.
  362. try:
  363. cron_profile_homes = _cron_tick_profile_homes(runner.config)
  364. except Exception as exc:
  365. logger.warning("Could not resolve profile homes for cron: %s", exc)
  366. cron_profile_homes = []
  367. # External providers own one unscoped remote registry, so they can only serve a single home.
  368. cron_provider = scheduler_for_profile_mode(
  369. resolve_cron_scheduler(), multiplex_profiles=len(cron_profile_homes) > 1)
  370. cron_start_kwargs: Dict[str, Any] = {"adapters": runner.adapters, "loop": asyncio.get_running_loop()}
  371. if isinstance(cron_provider, InProcessCronScheduler) and cron_profile_homes:
  372. # Live enumerator: the ticker re-reads profiles/ every cycle so a profile created while
  373. # the gateway runs gets its jobs fired without a restart (hot-serve).
  374. cron_start_kwargs["profile_homes"] = lambda: _cron_tick_profile_homes(runner.config)
  375. # Stand down, per tick, for a profile whose OWN gateway process ticks it.
  376. cron_start_kwargs["profile_gate"] = _cron_profile_gate
  377. # Per-profile adapters so each profile's cron output goes via its own bot, not the
  378. # default's. Absent (no multiplexed adapters), delivery for a secondary profile falls
  379. # back to the primary's routed adapters or fails closed — the job still FIRES.
  380. cron_start_kwargs["profile_adapters"] = getattr(runner, "_profile_adapters", None)
  381. # A secondary bot that dropped is still that profile's own: never borrow meanwhile.
  382. cron_start_kwargs["profile_bot_platforms"] = getattr(runner, "_profile_bot_platforms", None)
  383. # runner.adapters belongs to the LAUNCH profile (``default``, or the ``--profile``
  384. # name); naming it keeps the ticker from routing a secondary's cron through that bot
  385. # and lets a named multiplexer's own jobs reuse its live adapters.
  386. cron_start_kwargs["default_profile"] = runner._primary_profile_name
  387. logger.info(
  388. "Cron scheduler will tick %d profile(s): %s", len(cron_profile_homes),
  389. [p[0] if isinstance(p, tuple) else p for p in cron_profile_homes])
  390. # Only the in-process ticker polls local due jobs, so only it gets the external-drain dispatch gate.
  391. if isinstance(cron_provider, InProcessCronScheduler):
  392. cron_start_kwargs["can_dispatch"] = lambda: not (
  393. runner._draining or runner._external_drain_active)
  394. # Supervised: a ticker that dies without a stop request is respawned by housekeeping (#111010).
  395. from cron.scheduler_thread import SupervisedTickerThread
  396. cron_thread = SupervisedTickerThread(
  397. cron_provider.start, args=(cron_stop,), kwargs=cron_start_kwargs, stop_event=cron_stop)
  398. from gateway.runtime_ownership import process_ownership
  399. process_ownership.start_writer(cron_thread)
  400. # External providers fire over loopback HTTP to THIS process's api_server; if it never came up (usually
  401. # API_SERVER_KEY missing) every fire fails while manual runs work — misread as a job bug. Say it ONCE.
  402. if not isinstance(cron_provider, InProcessCronScheduler):
  403. try:
  404. _has_api_server = Platform.API_SERVER in (runner.adapters or {})
  405. except Exception:
  406. _has_api_server = True # never let the tell break startup
  407. if not _has_api_server:
  408. logger.warning(
  409. "Cron provider '%s' is active but the api_server adapter is "
  410. "NOT running in this gateway — scheduled fires arrive over "
  411. "loopback HTTP and will all fail (jobs only run when "
  412. "triggered manually). Most common cause: API_SERVER_KEY is "
  413. "missing from this gateway process's environment. Restart "
  414. "the gateway through its supervisor (`hermes gateway "
  415. "restart`) so the profile env loads.",
  416. getattr(cron_provider, "name", "external"))
  417. # Gateway-only housekeeping runs independently of the cron provider; shares cron_stop for shutdown.
  418. housekeeping_thread = threading.Thread(
  419. target=_start_gateway_housekeeping, args=(cron_stop,),
  420. kwargs={"adapters": runner.adapters, "loop": asyncio.get_running_loop(),
  421. "cron_provider": cron_provider, "runner": runner, "cron_thread": cron_thread},
  422. daemon=True, name="gateway-housekeeping")
  423. process_ownership.start_writer(housekeeping_thread)
  424. return cron_stop, cron_provider, cron_thread, housekeeping_thread
  425. async def _start_gateway_shutdown_tail(
  426. runner, _control_server, cron_stop: threading.Event, cron_provider,
  427. cron_thread: Any, housekeeping_thread: threading.Thread,
  428. _planned_stop_watcher_stop: threading.Event, _planned_stop_watcher_thread: threading.Thread,
  429. _signal_initiated_shutdown: list) -> bool:
  430. """Post-``wait_for_shutdown`` teardown; returns the process exit verdict (True = exit 0)."""
  431. from gateway.run import (_CRON_SHUTDOWN_DRAIN_TIMEOUT, _HOUSEKEEPING_SHUTDOWN_DRAIN_TIMEOUT, _await_thread_exit, _best_effort, _resolve_gateway_exit_verdict, _shutdown_mcp_servers_nonblocking, _stop_cron_provider)
  432. # Control socket first: once shutdown begins we are no longer a truthful "serving here" answer and a
  433. # successor must be able to bind. Early-exit paths rely on the atexit cleanup_files hook instead.
  434. if _control_server is not None:
  435. try:
  436. await _control_server.stop()
  437. except Exception:
  438. logger.debug("Control socket stop failed (non-fatal)", exc_info=True)
  439. def _stop_keepalive() -> None:
  440. from hermes_cli.nous_auth_keepalive import stop_nous_auth_keepalive
  441. stop_nous_auth_keepalive()
  442. _best_effort(_stop_keepalive)
  443. # Never join(): an in-flight cron delivery is a coroutine on THIS loop; a sync join would drop it.
  444. # Stop cron scheduler + housekeeping cleanly. These MUST be awaited cooperatively, not join()ed. A cron
  445. # delivery in flight when the gateway restarts is a coroutine scheduled onto THIS event loop
  446. # (safe_schedule_threadsafe); the ticker thread is blocked on its future.result(). A synchronous
  447. # cron_thread.join() would block the loop, so that delivery could never run — it timed out and the
  448. # message was silently dropped (#58818). Awaiting keeps the loop alive so the in-flight delivery
  449. # finishes before we tear down.
  450. cron_stop.set()
  451. _stop_cron_provider(cron_provider)
  452. if not await _await_thread_exit(cron_thread, timeout=_CRON_SHUTDOWN_DRAIN_TIMEOUT):
  453. logger.warning("Cron ticker did not exit within %.0fs of shutdown — an in-flight "
  454. "delivery may have been dropped.", _CRON_SHUTDOWN_DRAIN_TIMEOUT)
  455. await _await_thread_exit(housekeeping_thread, timeout=_HOUSEKEEPING_SHUTDOWN_DRAIN_TIMEOUT)
  456. # Stop the planned-stop watcher (daemon=True so this is belt-and-suspenders).
  457. _planned_stop_watcher_stop.set()
  458. _planned_stop_watcher_thread.join(timeout=2)
  459. # Never suppressed: a raise here is a real teardown failure (it once hid a changed signature,
  460. # leaving every MCP connection and the shared loop up while the gateway reported a clean exit).
  461. try:
  462. await _shutdown_mcp_servers_nonblocking(config=getattr(runner, "config", None))
  463. except Exception:
  464. logger.warning("MCP shutdown failed; connections may be left open", exc_info=True)
  465. # The failure verdict comes AFTER the cooperative teardown: returning early here leaked the
  466. # cron ticker + housekeeping threads (and open MCP connections) for embedded callers (#12175).
  467. return _resolve_gateway_exit_verdict(runner, _signal_initiated_shutdown[0])
  468. def _launch_home_may_multiplex(config=None) -> bool:
  469. """A named profile serving ``default`` as a secondary splits default's local sessions between
  470. the two stores (routing rows follow the launch home, receipts follow the authority), so the
  471. multiplexer is the default profile's job.
  472. An EXPLICIT ``gateway.multiplex_profiles: true`` (config.yaml or ``GATEWAY_MULTIPLEX_PROFILES``)
  473. on a named launch home exits EX_CONFIG: a config verdict, never a retry. The implicit default
  474. (unset key, or the retired ``false`` that ``resolve_multiplex_mode`` settles like unset) is not
  475. the operator's choice, so it is downgraded the way every other implicit-multiplex blocker is:
  476. this process comes up standalone, says why, and *config* is flipped in place so the boot below
  477. reserves and serves only the launch profile."""
  478. from hermes_constants import profile_name_for_home
  479. from gateway.run import get_hermes_home, logger
  480. name = profile_name_for_home(get_hermes_home())
  481. if name in (None, "default"):
  482. return True
  483. from hermes_cli.gateway_multiplex_mode import explicit_multiplex_flag
  484. if config is not None and not explicit_multiplex_flag(get_hermes_home()):
  485. from hermes_cli.gateway_multiplex_mode import MultiplexDecision, log_multiplex_decision
  486. decision = MultiplexDecision(
  487. False, "guard",
  488. f"profile {name!r} launched the gateway, and only the default profile runs the multiplexer "
  489. "(start it with `hermes gateway run` from the default profile to serve every profile)")
  490. config.multiplex_profiles = False
  491. log_multiplex_decision(decision)
  492. return True
  493. from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE
  494. from gateway.run import _write_runtime_status_quiet
  495. logger.error(
  496. "gateway.multiplex_profiles is set on profile %r, but only the default profile can run the "
  497. "multiplexer. Unset the flag in this profile's config.yaml, or start the gateway from the "
  498. "default profile (`hermes gateway run`), which serves this profile too.", name)
  499. _write_runtime_status_quiet(gateway_state="startup_failed", exit_reason="multiplex_requires_default_profile")
  500. raise SystemExit(GATEWAY_FATAL_CONFIG_EXIT_CODE)
  501. def _start_gateway_resolve_terminal_cwd() -> None:
  502. """Messaging-only defaults belong to startup, not incidental imports by the TUI."""
  503. import os
  504. from gateway.cwd_placeholder import CWD_PLACEHOLDERS, resolve_placeholder_terminal_cwd
  505. configured_cwd = os.environ.get("TERMINAL_CWD", "")
  506. if not configured_cwd or configured_cwd in CWD_PLACEHOLDERS:
  507. resolved_cwd = resolve_placeholder_terminal_cwd(
  508. configured_cwd=configured_cwd,
  509. terminal_backend=os.environ.get("TERMINAL_ENV", ""),
  510. messaging_cwd=os.getenv("MESSAGING_CWD"),
  511. docker_mount_cwd_to_workspace=os.getenv(
  512. "TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "false").lower()
  513. in {"true", "1", "yes"},
  514. home_fallback=str(Path.home()))
  515. if resolved_cwd is None:
  516. os.environ.pop("TERMINAL_CWD", None)
  517. else:
  518. os.environ["TERMINAL_CWD"] = resolved_cwd
  519. # A browse that catches a lagging state.db up holds the profile's gateway.lock only for the drift
  520. # re-check and the additive DDL, cut off at hermes_state_reconcile.BROWSE_RECONCILE_BUDGET_S by a
  521. # sqlite progress handler. Its integrity scan (which scales with file size) runs BEFORE the lock, so
  522. # a large store never extends this window. A gateway booting in it waits it out, with margin for lock
  523. # publication and the drift probes, instead of exiting as if another gateway owned the profile.
  524. def _browse_reconcile_reserve_wait_s() -> float:
  525. from hermes_state_reconcile import BROWSE_RECONCILE_BUDGET_S
  526. return BROWSE_RECONCILE_BUDGET_S + 2.0
  527. def _reserve_waiting_out_browse_reconcile(ownership, homes, *, wait_s: float | None = None):
  528. """Boot's all-or-nothing reserve of every served home, waiting out transient holders (a browse
  529. reconcile, an unpublished record, a client's lock probe) for up to the browse window."""
  530. from gateway.runtime_ownership import reserve_waiting_out_transient_holders
  531. if wait_s is None:
  532. wait_s = _browse_reconcile_reserve_wait_s()
  533. return reserve_waiting_out_transient_holders(ownership, homes, wait_s=wait_s)
  534. def _start_gateway_reserve_profiles(profile_homes) -> bool:
  535. """Reserve the launch home plus every served profile home; False when startup must abort."""
  536. from gateway.run import get_hermes_home
  537. from gateway.runtime_ownership import process_ownership, OwnershipConflict
  538. homes = [get_hermes_home(), *(home for _, home in profile_homes)]
  539. try:
  540. _reserve_waiting_out_browse_reconcile(process_ownership, homes)
  541. except OwnershipConflict as exc:
  542. logger.error("Cannot reserve gateway profiles: %s", exc)
  543. # A named profile that the LIVE default multiplexer already serves is a configuration
  544. # verdict (gateway.multiplex_profiles), not a transient fault: exit EX_CONFIG so systemd's
  545. # RestartPreventExitStatus=78 parks the unit instead of restart-looping (#51228, #97120).
  546. # Any other same-user contender keeps the ordinary "already running" exit 1.
  547. from hermes_cli.gateway import named_profile_served_by_running_multiplexer
  548. if named_profile_served_by_running_multiplexer():
  549. from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE
  550. from gateway.run import _write_runtime_status_quiet
  551. logger.error(
  552. "The default gateway is running as a profile multiplexer and already serves this profile. "
  553. "Manage it with `hermes gateway restart` from the default profile instead of starting a "
  554. "second gateway for the profile.")
  555. _write_runtime_status_quiet(gateway_state="startup_failed", exit_reason=str(exc))
  556. raise SystemExit(GATEWAY_FATAL_CONFIG_EXIT_CODE)
  557. return False
  558. except OSError as exc:
  559. logger.error("Cannot reserve gateway profiles: %s", exc)
  560. return False
  561. return True
  562. def _start_gateway_install_signal_handlers(loop, shutdown_signal_handler, restart_signal_handler) -> None:
  563. """SIGINT/SIGTERM drain, SIGUSR1 restart; main thread only."""
  564. import signal
  565. from contextlib import suppress
  566. if threading.current_thread() is threading.main_thread():
  567. # add_signal_handler raises NotImplementedError on Windows; SIGUSR1 is POSIX-only.
  568. handlers = [(sig, shutdown_signal_handler, (sig,)) for sig in (signal.SIGINT, signal.SIGTERM)]
  569. if hasattr(signal, "SIGUSR1"):
  570. handlers.append((signal.SIGUSR1, restart_signal_handler, ())) # windows-footgun: ok — hasattr-guarded
  571. for sig, handler, args in handlers:
  572. with suppress(NotImplementedError):
  573. loop.add_signal_handler(sig, handler, *args) # windows-footgun: ok — suppress(NotImplementedError)
  574. else:
  575. logger.info("Skipping signal handlers (not running in main thread).")
  576. async def _start_gateway_run_runner(runner, _signal_initiated_shutdown: list) -> Optional[bool]:
  577. """MCP discovery, then ``runner.start()``. Returns the exit verdict when the process must end
  578. before running mode (start failed, clean exit, aborted startup); None to continue into cron."""
  579. from gateway.run import (_discover_gateway_mcp_tools, _resolve_gateway_exit_verdict,
  580. _shutdown_gateway_health_export, _shutdown_mcp_servers_nonblocking)
  581. # discover_mcp_tools() blocks up to 120s; on the loop thread it would freeze platform heartbeats.
  582. try:
  583. # MCP tool discovery — run in an executor so the asyncio event loop stays responsive even when a
  584. # configured MCP server is slow or unreachable. discover_mcp_tools() uses a blocking 120s wait
  585. # internally; calling it from the loop thread would freeze platform heartbeats (Discord shard,
  586. # Telegram polling) until it returned. See #16856.
  587. await _discover_gateway_mcp_tools(runner.config)
  588. except Exception:
  589. logger.debug("MCP tool discovery failed (non-fatal)", exc_info=True)
  590. try:
  591. success = await runner.start()
  592. except BaseException:
  593. _shutdown_gateway_health_export(runner)
  594. raise
  595. if not success:
  596. _shutdown_gateway_health_export(runner)
  597. return False
  598. if runner.should_exit_cleanly:
  599. _shutdown_gateway_health_export(runner)
  600. if runner.exit_reason:
  601. logger.error("Gateway exiting cleanly: %s", runner.exit_reason)
  602. # Explicit exit codes (GATEWAY_FATAL_CONFIG_EXIT_CODE) must propagate so s6 finish maps 78 → 125.
  603. if runner.exit_code is not None:
  604. raise SystemExit(runner.exit_code)
  605. return True
  606. if not runner._running:
  607. # Startup aborted by restart/shutdown before running mode; preserve that path without starting cron.
  608. try:
  609. await runner.wait_for_shutdown()
  610. try:
  611. await _shutdown_mcp_servers_nonblocking(config=getattr(runner, "config", None))
  612. except Exception:
  613. logger.warning("MCP shutdown failed; connections may be left open", exc_info=True)
  614. return _resolve_gateway_exit_verdict(runner, _signal_initiated_shutdown[0])
  615. finally:
  616. _shutdown_gateway_health_export(runner)
  617. return None
  618. def _start_gateway_early_environment_probe() -> None:
  619. """Background-start the host toolchain probe the warm-up and first prompt read (idempotent;
  620. honours ``agent.environment_probe: false``). Never raises: the warm-up still runs it lazily."""
  621. try:
  622. from hermes_cli.config import load_config_readonly
  623. agent_cfg = load_config_readonly().get("agent")
  624. if isinstance(agent_cfg, dict) and not agent_cfg.get("environment_probe", True):
  625. return
  626. from tools.env_probe import warm_environment_probe_async
  627. warm_environment_probe_async()
  628. except Exception:
  629. logger.debug("early environment probe did not start", exc_info=True)
  630. async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = False,
  631. verbosity: Optional[int] = 0, force: bool = False, idle_exit: bool = False) -> bool:
  632. """Start the gateway and run until interrupted; False if it failed to start (non-zero exit so
  633. systemd can auto-restart). ``replace`` kills any existing instance first (avoids restart-loop
  634. deadlocks); ``force`` starts without consulting the host owner at all; ``idle_exit`` arms the
  635. unmanaged idle exit (``gateway.run_idle_exit``) for a client-started daemon."""
  636. from gateway.run import (
  637. GatewayRunner,
  638. _best_effort,
  639. _enable_multiplex_log_routing,
  640. _ensure_windows_gateway_venv_imports,
  641. _gateway_loop_exception_handler,
  642. _host_attach_or_none,
  643. _log_standalone_profiles_at_boot,
  644. _multiplex_profile_homes,
  645. _refresh_host_gateway_record,
  646. _run_planned_stop_watcher,
  647. _start_gateway_make_restart_signal_handler,
  648. asyncio,
  649. load_gateway_config_for_runner,
  650. os,
  651. threading,
  652. )
  653. # Set here (not at import) so incidental gateway.run imports from CLI code don't poison it.
  654. os.environ["HERMES_EXEC_ASK"] = "1"
  655. from hermes_cli.update_proxy_env import restore_user_proxy_env
  656. restore_user_proxy_env() # a gateway an update restarted keeps the user's proxy env, not the update's
  657. _start_gateway_resolve_terminal_cwd()
  658. from hermes_cli.resource_limits import apply_nofile_soft_limit
  659. apply_nofile_soft_limit()
  660. # Snapshot the revision while sys.modules matches disk so a later `git pull` is detected safely.
  661. from gateway.code_skew import record_boot_fingerprint
  662. record_boot_fingerprint()
  663. # Config verdicts come before the duplicate-instance guard: `--replace` must not stop a healthy
  664. # gateway for a launch that cannot start.
  665. resolved_config = config if config is not None else load_gateway_config_for_runner()
  666. # The first turn's prompt needs the host toolchain probe (five subprocess calls); start it now
  667. # so it runs beside the boot imports instead of serially inside the pre-READY warm-up.
  668. _start_gateway_early_environment_probe()
  669. profile_homes = (_multiplex_profile_homes(resolved_config)
  670. if getattr(resolved_config, 'multiplex_profiles', False) else [])
  671. if profile_homes and not _launch_home_may_multiplex(resolved_config):
  672. return False
  673. if profile_homes and not getattr(resolved_config, 'multiplex_profiles', False):
  674. profile_homes = [] # implicit default downgraded: this named launch serves only itself
  675. # Multiplex-only: the ONE host gateway decides first. Attach to it, make it serve this profile,
  676. # replace it (--replace) or refuse — before anything below binds a port or claims a PID file.
  677. _host_decision = await _host_attach_or_none(replace, force)
  678. if _host_decision is not None:
  679. return _host_decision
  680. # Duplicate-instance guard scoped to HERMES_HOME (the host record is absent or unusable here).
  681. from gateway.status import get_running_pid
  682. existing_pid = get_running_pid()
  683. if (existing_pid is not None and existing_pid != os.getpid()
  684. and not await _start_gateway_replace_existing_instance(existing_pid, replace)):
  685. return False
  686. from gateway.status import remove_pid_file, release_gateway_runtime_lock
  687. if not _start_gateway_reserve_profiles(profile_homes):
  688. return False
  689. # Freeze discovery: later profile additions must restart and reserve before opening stores.
  690. if profile_homes:
  691. resolved_config._runtime_profile_homes = tuple(profile_homes)
  692. # PID file BEFORE adapters: of two concurrent `run --replace`, only the O_EXCL winner opens sockets.
  693. # Only --force skips the host-lock refusal. Every generated unit carries --replace, so reading it as
  694. # --force there disabled the one arbiter of the two-units-at-once race; a replace that took the
  695. # owner over already freed the lock with that process.
  696. if not _start_gateway_claim_pid_file(force=force):
  697. release_gateway_runtime_lock()
  698. return False
  699. _control_server = None
  700. runner = None
  701. _planned_stop_watcher_stop = None
  702. try:
  703. _start_gateway_configure_logging(verbosity)
  704. from gateway.run_startup import recover_left_core_at_gateway_start
  705. await asyncio.to_thread(recover_left_core_at_gateway_start) # before the runner loads platform config
  706. from gateway.run_runtime import initialize_gateway_runtime, pin_launch_store, unpin_launch_store
  707. _launch_store_pin = await asyncio.to_thread(pin_launch_store)
  708. try:
  709. runner = GatewayRunner(resolved_config)
  710. finally:
  711. unpin_launch_store(_launch_store_pin)
  712. await initialize_gateway_runtime(runner)
  713. # Multiplex: swap the launch-home file handlers for per-profile routers so each profile's records
  714. # land in its own logs/. Must run after the runner resolved (possibly None) config and setup_logging.
  715. # See #82936.
  716. _enable_multiplex_log_routing(runner.config)
  717. # ``--replace`` is explicit startup authority, not a durable reconnect policy: GatewayRunner scopes
  718. # it to cold adapter connects and clears it before the background reconnect watcher starts.
  719. runner._platform_lock_takeover_on_start = bool(replace)
  720. # Unexpected signals exit non-zero so service managers revive us; planned stops write a marker first.
  721. _signal_initiated_shutdown = [False]
  722. shutdown_signal_handler = _start_gateway_make_shutdown_signal_handler(
  723. runner, _signal_initiated_shutdown)
  724. restart_signal_handler = _start_gateway_make_restart_signal_handler(runner)
  725. loop = asyncio.get_running_loop()
  726. # Swallow transient network errors from background tasks; one unhandled httpx error would kill us.
  727. # Issues #31066 / #31110: an unhandled ``telegram.error.TimedOut`` (or peer NetworkError / httpx
  728. # connection error) in any awaited coroutine would propagate to the loop and kill the gateway process,
  729. # taking down every profile attached to the same runner. systemd then restarts the service after ~5s but
  730. # the active conversation turn is lost. The fix is intentionally narrow: only well-known transient
  731. # network errors are swallowed (and logged with full traceback so the originating call site is still
  732. # discoverable). Anything else is forwarded to the default handler so real bugs still surface.
  733. loop.set_exception_handler(_gateway_loop_exception_handler)
  734. _start_gateway_install_signal_handlers(loop, shutdown_signal_handler, restart_signal_handler)
  735. # Windows has no add_signal_handler, so `hermes gateway stop`'s SIGTERM would never drain; poll the
  736. # planned-stop marker (written BEFORE the kill) instead. Runs everywhere so masked-SIGTERM drains.
  737. # Windows fallback: asyncio.add_signal_handler raises NotImplementedError on Windows, so `hermes gateway
  738. # stop`'s SIGTERM (which Python maps to TerminateProcess on Windows) never invokes
  739. # shutdown_signal_handler. That means the drain loop never runs, mark_resume_pending never fires, and
  740. # sessions are silently lost across restarts (issue #33778). The fix is a marker-polling thread: `hermes
  741. # gateway stop` writes the planned-stop marker BEFORE killing, and this thread notices it and drives the
  742. # same shutdown path the signal handler would have. Runs on every platform (cheap, defensive) so
  743. # non-signal-bearing environments (Windows native, sandboxed CI runners that mask SIGTERM) still get a
  744. # clean drain.
  745. _planned_stop_watcher_stop = threading.Event()
  746. _planned_stop_watcher_thread = threading.Thread(
  747. target=_run_planned_stop_watcher,
  748. args=(_planned_stop_watcher_stop, runner, loop, shutdown_signal_handler), daemon=True,
  749. name="planned-stop-watcher")
  750. _planned_stop_watcher_thread.start()
  751. # Right after the PID claim (which makes us authoritative). Fatal: local clients attach only
  752. # with tickets minted on this listener, so startup aborts without it.
  753. _control_server = await _start_gateway_start_control_socket(runner)
  754. if _control_server is None:
  755. raise RuntimeError("gateway session bootstrap control listener unavailable")
  756. # Now the attach channel answers: republish the host record with the settled served set.
  757. _refresh_host_gateway_record(runner)
  758. _log_standalone_profiles_at_boot(runner)
  759. from gateway.run_runtime import start_gateway_runtime_api
  760. await start_gateway_runtime_api(runner)
  761. def _lifecycle_record_startup() -> Optional[dict]:
  762. # Report if the previous life died uncleanly (SIGKILL / OOM / VM death), then claim the
  763. # sentinel for this life. After the PID-file claim so a --replace loser can't clobber it.
  764. # The store scan that report used to run here is deferred: see below.
  765. from gateway.lifecycle_ledger import record_startup
  766. return record_startup()
  767. def _start_keepalive() -> None:
  768. from hermes_cli.nous_auth_keepalive import start_nous_auth_keepalive
  769. start_nous_auth_keepalive()
  770. _unclean_exit = _best_effort(_lifecycle_record_startup, "Lifecycle ledger startup record failed: %s")
  771. _best_effort(_start_keepalive, "Nous auth keepalive did not start: %s")
  772. _ensure_windows_gateway_venv_imports()
  773. _early_verdict = await _start_gateway_run_runner(runner, _signal_initiated_shutdown)
  774. if _early_verdict is not None:
  775. return _early_verdict
  776. if _unclean_exit is not None:
  777. _start_deferred_state_db_integrity_check(_unclean_exit)
  778. cron_stop, cron_provider, cron_thread, housekeeping_thread = (
  779. _start_gateway_start_cron_and_housekeeping(runner))
  780. # From here every exit (a READY publication that raises included) runs the shutdown tail,
  781. # which stops the cron ticker and housekeeping before the adapters they deliver through close.
  782. from gateway.run_runtime import publish_gateway_runtime_ready, wait_gateway_runtime
  783. try:
  784. # READY only once adapters, cron and housekeeping run; missing systemd state just disables watchdog.
  785. publish_gateway_runtime_ready(runner)
  786. runner._start_systemd_watchdog()
  787. if idle_exit:
  788. from gateway.run_idle_exit import arm_unmanaged_idle_exit
  789. arm_unmanaged_idle_exit(runner)
  790. else:
  791. # A supervised/explicit gateway serves Desktop and the dashboard SPA, which call the
  792. # dashboard routes right away: mount them now, after READY. A client's own idle-exit
  793. # gateway (chat -q, TUI) only uses the native WebSocket; it mounts on first request.
  794. from gateway.run_api import warm_gateway_dashboard
  795. warm_gateway_dashboard(runner)
  796. await wait_gateway_runtime(runner)
  797. finally:
  798. try:
  799. await runner.stop()
  800. finally:
  801. verdict = await _start_gateway_shutdown_tail(
  802. runner, _control_server, cron_stop, cron_provider, cron_thread, housekeeping_thread,
  803. _planned_stop_watcher_stop, _planned_stop_watcher_thread, _signal_initiated_shutdown)
  804. return verdict
  805. finally:
  806. try:
  807. if runner is not None:
  808. from gateway.run_runtime import drain_gateway_runtime
  809. await drain_gateway_runtime(runner)
  810. # Startup may have opened adapters before an exception or an early exit verdict. The
  811. # same stop path owns their writers and DB handles on every exit, but when no stop ran
  812. # yet this is cleanup, not a shutdown: keep startup_failed and the crash markers.
  813. if getattr(runner, "_stop_task", None) is None and not getattr(runner, "_running", False):
  814. runner._startup_cleanup_only = True
  815. await runner.stop()
  816. finally:
  817. if _planned_stop_watcher_stop is not None:
  818. _planned_stop_watcher_stop.set()
  819. if _control_server is not None:
  820. await _control_server.stop()
  821. remove_pid_file()
  822. release_gateway_runtime_lock()