gateway_windows.py 86 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909
  1. """Windows gateway service backend (Scheduled Task + Startup-folder fallback).
  2. Mirrors the ``launchd_*`` / ``systemd_*`` contract. ``schtasks /Create ... /RL LIMITED`` runs at the
  3. CURRENT USER's next logon without elevation. Manual starts and ``install --start-now`` use the direct
  4. hidden-console launcher instead of ``schtasks /Run`` so start/restart behavior is consistent.
  5. """
  6. from __future__ import annotations
  7. import codecs
  8. import ctypes
  9. import json
  10. import locale
  11. import logging
  12. import os
  13. import re
  14. import shlex
  15. import shutil
  16. import subprocess
  17. import sys
  18. import time
  19. import uuid
  20. from datetime import datetime, timezone, UTC
  21. from pathlib import Path
  22. from xml.etree import ElementTree
  23. from xml.sax.saxutils import escape
  24. from agent.deadline import poll_until
  25. from hermes_cli._subprocess_compat import (
  26. _WINDOWS_GATEWAY_BREAKAWAY_ENV,
  27. windows_detach_flags,
  28. windows_detach_flags_without_breakaway,
  29. windows_hide_flags,
  30. )
  31. logger = logging.getLogger(__name__)
  32. # Short timeouts: schtasks occasionally wedges and we don't want to hang forever.
  33. _SCHTASKS_TIMEOUT_S = 15
  34. # Patterns in schtasks stderr that mean "fall back to the Startup folder".
  35. # schtasks' localized "access is denied" (en/es/cs/zh-Hans/zh-Hant/ja/ko): one vocabulary for both
  36. # the elevated-install offer and the Startup-folder fallback.
  37. _ACCESS_DENIED_WORDS = (
  38. r"access is denied|acceso denegado|přístup byl odepřen|拒绝访问|拒絕存取|アクセスが拒否されました|"
  39. r"액세스가 거부되었습니다"
  40. )
  41. _FALLBACK_PATTERNS = re.compile(
  42. rf"({_ACCESS_DENIED_WORDS}|schtasks timed out|schtasks produced no output)", re.IGNORECASE
  43. )
  44. _ACCESS_DENIED_PATTERN = re.compile(rf"({_ACCESS_DENIED_WORDS})", re.IGNORECASE)
  45. # Set by _spawn_detached() when the breakaway spawn failed and it retried WITHOUT
  46. # CREATE_BREAKAWAY_FROM_JOB — the child stays in the parent's Job Object and may be killed when this
  47. # shell exits. Dict (not bare bool) so the flag is mutable without ``global``.
  48. _LAST_SPAWN_BREAKAWAY_FALLBACK: dict = {"fallback": False}
  49. _TASK_NAME_DEFAULT = "Hermes_Gateway"
  50. _TASK_DESCRIPTION = "Hermes Agent Gateway - Messaging Platform Integration"
  51. _TASK_LOGON_DELAY = "PT30S"
  52. _TASK_RESTART_INTERVAL = "PT1M"
  53. _TASK_RESTART_COUNT = 999
  54. _GATEWAY_ENV = (("PYTHONIOENCODING", "utf-8"), ("HERMES_GATEWAY_DETACHED", "1"), ("HERMES_SUPERVISED_CHILD", "1"))
  55. def _schtasks_encoding() -> str:
  56. """Console encoding for ``schtasks.exe`` output: localized Windows emits the OEM/ANSI code page,
  57. not UTF-8, and decoding with the wrong codec raised UnicodeDecodeError in subprocess' reader
  58. threads. Prefer the locale's preferred encoding, fall back to UTF-8."""
  59. try:
  60. return locale.getpreferredencoding(False) or "utf-8"
  61. except Exception:
  62. return "utf-8"
  63. def _windows_console_encodings() -> list[str]:
  64. """Code pages a console tool such as ``schtasks.exe`` writes to a pipe, most likely first: the
  65. console output code page (65001 once ``configure_windows_stdio`` ran, else the OEM page), the OEM
  66. page, then the ANSI page. Read from kernel32, so Python's UTF-8 mode cannot disguise them."""
  67. kernel32 = getattr(getattr(ctypes, "windll", None), "kernel32", None)
  68. if kernel32 is None:
  69. return [_schtasks_encoding()]
  70. pages: list[str] = []
  71. for getter in ("GetConsoleOutputCP", "GetOEMCP", "GetACP"):
  72. try:
  73. code_page = int(getattr(kernel32, getter)())
  74. except (AttributeError, OSError, ValueError):
  75. continue
  76. if code_page > 0 and f"cp{code_page}" not in pages:
  77. pages.append(f"cp{code_page}")
  78. return pages or [_schtasks_encoding()]
  79. def _decode_schtasks_output(data: bytes) -> str:
  80. """Decode captured ``schtasks.exe`` bytes. schtasks writes the console/OEM code page even when this
  81. process runs in UTF-8 mode (the launcher sets ``PYTHONUTF8=1``), so ``locale.getpreferredencoding``
  82. is the wrong codec on a non-ASCII account path: ``C:\\Users\\方舟`` came back as ``����`` and the
  83. Scheduled-Task drift check could never settle (#116193). Strict UTF-8 first (ASCII and genuine
  84. UTF-8 output pass; legacy multi-byte text fails loudly), then the native code pages, then a lossy
  85. fallback so a reader thread never raises."""
  86. if data[:2] in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE):
  87. try:
  88. return data.decode("utf-16") # a genuine UTF-16 write (BOM) decodes as such, never as a code page
  89. except UnicodeDecodeError:
  90. pass # malformed UTF-16 falls through to the same fallbacks as any other bytes
  91. for encoding in ("utf-8", *_windows_console_encodings()):
  92. try:
  93. return data.decode(encoding)
  94. except (LookupError, UnicodeDecodeError):
  95. continue
  96. return data.decode("utf-8", errors="replace")
  97. def _assert_windows() -> None:
  98. if sys.platform != "win32":
  99. raise RuntimeError("gateway_windows is Windows-only")
  100. def _hermes_home() -> Path:
  101. from hermes_cli.config import get_hermes_home
  102. return Path(get_hermes_home())
  103. def hermes_service_roots() -> tuple[str, ...]:
  104. """Directories a Hermes-owned SCM service binary lives under: the checkout (its ``venv`` included),
  105. the running interpreter's ``Scripts`` dir (``hermes.exe`` shim) and the ``gateway-service`` launcher dir."""
  106. project_root = Path(__file__).resolve().parent.parent
  107. return (str(project_root), str(Path(sys.executable).parent), str(_hermes_home() / "gateway-service"))
  108. def _normalize_windows_path(value: str) -> str:
  109. return value.strip().lstrip('"').replace("\\", "/").rstrip("/").casefold()
  110. def hermes_owns_windows_service(name: str, binpath: str, hermes_roots: tuple[str, ...]) -> bool:
  111. """Positive ownership of an SCM service: Hermes-named (``hermes*``) or its binary path starts under a
  112. Hermes root. Pure so it is testable off-Windows. A Scheduled-Task-launched gateway descends from
  113. ``svchost.exe`` hosting ``Schedule``; without this gate the updater took Task Scheduler for the
  114. gateway's supervisor and ``sc.exe stop Schedule`` aborted every update (#97208)."""
  115. normalized_name = "".join(char for char in name.casefold() if char.isalnum())
  116. if normalized_name.startswith("hermes"):
  117. return True
  118. candidate = _normalize_windows_path(binpath)
  119. return any(candidate.startswith(_normalize_windows_path(root) + "/") for root in hermes_roots if root)
  120. def _preserve_hermes_home_path(path: str | Path) -> str:
  121. r"""Render Hermes-owned paths under the configured HERMES_HOME spelling.
  122. ``%LOCALAPPDATA%\hermes`` may be a symlink/junction to another drive; launcher files must not
  123. bake in the resolved target for paths under HERMES_HOME.
  124. """
  125. candidate = Path(path)
  126. try:
  127. home = _hermes_home()
  128. resolved_home = home.resolve()
  129. resolved_candidate = candidate.resolve()
  130. home_key = os.path.normcase(str(resolved_home))
  131. candidate_key = os.path.normcase(str(resolved_candidate))
  132. if os.path.commonpath([home_key, candidate_key]) == home_key:
  133. return str(home / os.path.relpath(str(resolved_candidate), str(resolved_home)))
  134. except Exception:
  135. pass
  136. return str(candidate)
  137. # ── Quoting helpers. cmd.exe (.cmd body), VBScript literals and schtasks /TR are three DIFFERENT
  138. # parsers — never reuse one helper for another. The task XML path avoids /TR quoting entirely.
  139. def _quote_cmd_script_arg(value: str) -> str:
  140. """Quote one argument INSIDE a .cmd file for cmd.exe: split on spaces/tabs outside double quotes,
  141. embedded quotes doubled. Line breaks are refused — they'd end the logical command line."""
  142. if "\r" in value or "\n" in value:
  143. raise ValueError(f"refusing to quote value containing newline: {value!r}")
  144. if not value:
  145. return '""'
  146. if not re.search(r'[ \t"]', value):
  147. return value
  148. return '"' + value.replace('"', '""') + '"'
  149. def _quote_vbs_string(value: str) -> str:
  150. """VBScript double-quoted literal (embedded quote doubled; newline refused)."""
  151. if "\r" in value or "\n" in value:
  152. raise ValueError(f"refusing to quote VBScript value containing newline: {value!r}")
  153. return '"' + value.replace('"', '""') + '"'
  154. # ── schtasks.exe wrapper
  155. def _exec_schtasks(args: list[str], timeout: float = _SCHTASKS_TIMEOUT_S) -> tuple[int, str, str]:
  156. """Run ``schtasks.exe`` with a hard timeout. Return (code, stdout, stderr); a wedge returns
  157. code=124 with a synthetic stderr so the fallback regex matches."""
  158. _assert_windows()
  159. schtasks = shutil.which("schtasks")
  160. if schtasks is None:
  161. return (1, "", "schtasks.exe not found on PATH")
  162. try:
  163. # Bytes, decoded by _decode_schtasks_output: a non-UTF-8 status line must never surface a
  164. # UnicodeDecodeError from subprocess' reader threads. CREATE_NO_WINDOW: no flashing console under a TUI.
  165. proc = subprocess.run(
  166. [schtasks, *args], capture_output=True, text=False,
  167. timeout=timeout, creationflags=windows_hide_flags(),
  168. check=False,
  169. )
  170. return (
  171. proc.returncode,
  172. _decode_schtasks_output(proc.stdout or b""),
  173. _decode_schtasks_output(proc.stderr or b""),
  174. )
  175. except subprocess.TimeoutExpired:
  176. return (124, "", f"schtasks timed out after {timeout}s")
  177. except OSError as e:
  178. return (1, "", f"schtasks invocation failed: {e}")
  179. def _should_fall_back(code: int, detail: str) -> bool:
  180. return code == 124 or bool(_FALLBACK_PATTERNS.search(detail or ""))
  181. def _is_access_denied(detail: str) -> bool:
  182. return bool(_ACCESS_DENIED_PATTERN.search(detail or ""))
  183. def _is_running_as_admin() -> bool:
  184. """Return True when the current Windows process is elevated (the shared ``TokenElevation``
  185. probe; ``IsUserAnAdmin`` is deprecated and answered the same question a second way)."""
  186. _assert_windows()
  187. from hermes_dir_acl import process_is_elevated
  188. return process_is_elevated()
  189. def _current_profile_cli_args() -> list[str]:
  190. """Return CLI args that preserve the current Hermes profile."""
  191. from hermes_cli.gateway import _profile_arg
  192. profile_arg = _profile_arg()
  193. return shlex.split(profile_arg) if profile_arg else []
  194. def _launch_elevated_gateway_command(command: str, extra_args: list[str] | None = None) -> bool:
  195. """Launch an elevated gateway subcommand via UAC and return True on handoff. The child is console
  196. ``python.exe`` with ``SW_HIDE``: it owns one hidden console its subprocesses (schtasks, taskkill)
  197. inherit — no visible window and no per-descendant conhost flashes (the console-less pythonw.exe
  198. alternative re-created #54220/#56747 for every descendant).
  199. All operator decisions are already collected in the parent shell before this point. See #54220, #56747.
  200. """
  201. _assert_windows()
  202. args = ["-m", "hermes_cli.main", *_current_profile_cli_args(), "gateway", command, *(extra_args or [])]
  203. params = subprocess.list2cmdline(args)
  204. cwd = str(Path(__file__).resolve().parent.parent)
  205. try:
  206. result = ctypes.windll.shell32.ShellExecuteW(None, "runas", sys.executable, params, cwd, 0) # 0 = SW_HIDE
  207. except Exception as exc:
  208. print(f"⚠ Could not launch elevated gateway {command} prompt: {exc}")
  209. return False
  210. if result <= 32:
  211. print(f"⚠ Elevated gateway {command} prompt was not started (ShellExecuteW={result})")
  212. return False
  213. return True
  214. def _launch_elevated_install(force: bool = False, *, start_now: bool | None = None, start_on_login: bool | None = None) -> bool:
  215. """Launch an elevated gateway install via UAC and return True on handoff."""
  216. overrides = {"HERMES_GATEWAY_ELEVATED_HANDOFF": "1"}
  217. extra_args = ["--elevated-handoff"]
  218. if force:
  219. extra_args.append("--force")
  220. for choice, env_key, flag in (
  221. (start_now, "HERMES_GATEWAY_INSTALL_START_NOW", "start-now"),
  222. (start_on_login, "HERMES_GATEWAY_INSTALL_START_ON_LOGIN", "start-on-login"),
  223. ):
  224. if choice is not None:
  225. overrides[env_key] = "1" if choice else "0"
  226. extra_args.append(f"--{flag}" if choice else f"--no-{flag}")
  227. saved = {key: os.environ.get(key) for key in overrides}
  228. try:
  229. os.environ.update(overrides)
  230. return _launch_elevated_gateway_command("install", extra_args)
  231. finally:
  232. for key, old in saved.items():
  233. if old is None:
  234. os.environ.pop(key, None)
  235. else:
  236. os.environ[key] = old
  237. # ── Paths: where we stash our task script and where Startup lives
  238. def get_task_name() -> str:
  239. """Scheduled Task name, scoped per profile."""
  240. _assert_windows()
  241. from hermes_cli.gateway import _profile_suffix # local: avoids circular init during boot
  242. suffix = _profile_suffix()
  243. return f"{_TASK_NAME_DEFAULT}_{suffix}" if suffix else _TASK_NAME_DEFAULT
  244. def _sanitize_filename(value: str) -> str:
  245. """Remove characters illegal in Windows filenames."""
  246. return re.sub(r'[<>:"/\\|?*\x00-\x1f]', "_", value)
  247. def get_task_script_path() -> Path:
  248. """The generated ``gateway.cmd`` wrapper under ``<HERMES_HOME>/gateway-service/`` (per-profile
  249. installs stay self-contained); the VBS launcher lives beside it."""
  250. _assert_windows()
  251. script_dir = _hermes_home() / "gateway-service"
  252. script_dir.mkdir(parents=True, exist_ok=True)
  253. return script_dir / f"{_sanitize_filename(get_task_name())}.cmd"
  254. def _startup_dir() -> Path:
  255. appdata = os.environ.get("APPDATA", "").strip()
  256. if appdata:
  257. return Path(appdata) / "Microsoft" / "Windows" / "Start Menu" / "Programs" / "Startup"
  258. userprofile = os.environ.get("USERPROFILE", "").strip() or os.environ.get("HOME", "").strip()
  259. if not userprofile:
  260. raise RuntimeError("neither APPDATA nor USERPROFILE is set — cannot resolve Startup folder")
  261. return Path(userprofile).joinpath("AppData", "Roaming", "Microsoft", "Windows", "Start Menu", "Programs", "Startup")
  262. def get_startup_entry_path() -> Path:
  263. _assert_windows()
  264. return _startup_dir() / f"{_sanitize_filename(get_task_name())}.vbs"
  265. def _legacy_startup_entry_path() -> Path:
  266. _assert_windows()
  267. return _startup_dir() / f"{_sanitize_filename(get_task_name())}.cmd"
  268. def _startup_staging_path() -> Path:
  269. """The Startup-folder staging file; also the debris a pre-fix failed swap left behind (#114093)."""
  270. return get_startup_entry_path().with_suffix(".tmp")
  271. def _stable_gateway_working_dir(project_root: Path) -> str:
  272. """Stable cwd for detached/startup runs: anchor at HERMES_HOME when it exists (mirrors the POSIX
  273. service invariant) so a moved checkout/worktree can't fail the ``cd`` step; else the checkout."""
  274. from hermes_cli.config import get_hermes_home
  275. try:
  276. home = get_hermes_home()
  277. if home and Path(home).is_dir():
  278. return str(Path(home))
  279. except Exception:
  280. pass
  281. return str(project_root)
  282. # ── Script rendering
  283. def _gateway_run_argv(python_exe: str, profile_arg: str) -> list[str]:
  284. """``python -m hermes_cli.main [--profile X] gateway run`` — shared by every launcher renderer."""
  285. argv = [python_exe, "-m", "hermes_cli.main"]
  286. if profile_arg:
  287. argv.extend(profile_arg.split())
  288. argv.extend(["gateway", "run"])
  289. return argv
  290. def _launcher_settings(home: Path | None = None) -> tuple[str, str, str, str]:
  291. """Return (python_path, working_dir, hermes_home, profile_arg) for generated launchers.
  292. ``home`` targets another profile's HERMES_HOME (per-profile cold-start, #110959)."""
  293. from hermes_cli.gateway import PROJECT_ROOT, _profile_arg, get_python_path # avoid circular init
  294. hermes_home = str(home if home is not None else _hermes_home())
  295. return (
  296. _preserve_hermes_home_path(get_python_path()),
  297. _stable_gateway_working_dir(PROJECT_ROOT),
  298. hermes_home,
  299. _profile_arg(hermes_home),
  300. )
  301. def _launcher_pythonpath_entries(extra_pythonpath: list[str]) -> list[str]:
  302. return [
  303. _preserve_hermes_home_path(Path(__file__).resolve().parent.parent),
  304. *[_preserve_hermes_home_path(entry) for entry in extra_pythonpath],
  305. ]
  306. def _build_gateway_cmd_script(python_path: str, working_dir: str, hermes_home: str, profile_arg: str) -> str:
  307. """Build the ``gateway.cmd`` wrapper (CRLF-terminated). No PATH overrides (rewriting PATH breaks
  308. Homebrew/nvm-style installs), no ``start`` (extra wrapper process muddles lifecycle/status), no
  309. ``--replace`` (repeated /Run calls must be idempotent, not takeover loops)."""
  310. python_exe_path, venv_dir, extra_pythonpath = _resolve_detached_python(python_path)
  311. pythonpath = ";".join([*_launcher_pythonpath_entries(extra_pythonpath), "%PYTHONPATH%"])
  312. lines = [
  313. "@echo off",
  314. f"rem {_TASK_DESCRIPTION}",
  315. f"cd /d {_quote_cmd_script_arg(working_dir)}",
  316. f'set "HERMES_HOME={hermes_home}"',
  317. *[f'set "{k}={v}"' for k, v in _GATEWAY_ENV],
  318. # VIRTUAL_ENV lets the gateway's own python detection find the venv.
  319. f'set "VIRTUAL_ENV={_preserve_hermes_home_path(venv_dir)}"',
  320. f'set "PYTHONPATH={pythonpath}"',
  321. " ".join(_quote_cmd_script_arg(a) for a in _gateway_run_argv(python_exe_path, profile_arg)),
  322. "exit /b 0",
  323. ]
  324. return "\r\n".join(lines) + "\r\n"
  325. def _build_gateway_vbs_script(python_path: str, working_dir: str, hermes_home: str, profile_arg: str) -> str:
  326. """Build the hidden-console ``gateway.vbs`` launcher (CRLF-terminated).
  327. Run via ``wscript.exe``, not ``cmd.exe``: at logon Windows broadcasts CTRL_CLOSE_EVENT to console
  328. groups, killing a cmd-hosted gateway with STATUS_CONTROL_C_EXIT, which Task Scheduler treats as a
  329. user cancel (``RestartOnFailure`` never fires). wscript has no console; python.exe runs with window
  330. style 0 so descendants inherit one hidden console instead of flashing their own (#54220/#56747).
  331. Why: issue #45599 root cause #1.
  332. ``wscript.exe`` is a GUI-subsystem executable with no console, so this launcher receives no console
  333. control events. It ``Run``s the console ``python.exe`` with window style 0 (hidden): the gateway owns a
  334. single hidden console — never shown, never CTRL_CLOSE'd at logon, and inherited by every
  335. console-subsystem descendant (git, gh, node, …) so none of them allocate a visible flashing conhost
  336. (#54220/#56747; the previous console-less pythonw.exe gateway forced exactly that per-descendant flash).
  337. No cmd.exe anywhere in the chain. Mirrors ``_build_gateway_cmd_script`` (same env + argv via
  338. ``_resolve_detached_python``).
  339. """
  340. python_exe_path, venv_dir, extra_pythonpath = _resolve_detached_python(python_path)
  341. # list2cmdline gives CreateProcess-correct quoting for WScript.Shell.Run.
  342. command_line = subprocess.list2cmdline(_gateway_run_argv(python_exe_path, profile_arg))
  343. static_pythonpath = os.pathsep.join(_launcher_pythonpath_entries(extra_pythonpath))
  344. q = _quote_vbs_string
  345. lines = [
  346. f"' {_TASK_DESCRIPTION}",
  347. "Option Explicit",
  348. "Dim sh, env, existing_pp",
  349. 'Set sh = CreateObject("WScript.Shell")',
  350. 'Set env = sh.Environment("PROCESS")',
  351. f"env.Item({q('HERMES_HOME')}) = {q(hermes_home)}",
  352. *[f"env.Item({q(k)}) = {q(v)}" for k, v in _GATEWAY_ENV],
  353. f"env.Item({q('VIRTUAL_ENV')}) = {q(_preserve_hermes_home_path(venv_dir))}",
  354. # Mirror the cmd wrapper's ``PYTHONPATH=<static>;%PYTHONPATH%`` at runtime.
  355. f"existing_pp = env.Item({q('PYTHONPATH')})",
  356. "If Len(existing_pp) > 0 Then",
  357. f" env.Item({q('PYTHONPATH')}) = {q(static_pythonpath + os.pathsep)} & existing_pp",
  358. "Else",
  359. f" env.Item({q('PYTHONPATH')}) = {q(static_pythonpath)}",
  360. "End If",
  361. f"sh.CurrentDirectory = {q(working_dir)}",
  362. # Window style 0 = hidden; bWaitOnReturn False = detached/async.
  363. f"sh.Run {q(command_line)}, 0, False",
  364. ]
  365. return "\r\n".join(lines) + "\r\n"
  366. def _build_startup_launcher(script_path: Path) -> str:
  367. """The tiny Startup-folder .vbs that chains hidden. Quits silently if the target is gone so a
  368. stale entry doesn't error on every login."""
  369. target = str(script_path.with_suffix(".vbs"))
  370. command = subprocess.list2cmdline(["wscript.exe", target])
  371. lines = [
  372. f"' {_TASK_DESCRIPTION}",
  373. "Option Explicit",
  374. "Dim fso, sh, target",
  375. f"target = {_quote_vbs_string(target)}",
  376. 'Set fso = CreateObject("Scripting.FileSystemObject")',
  377. "If Not fso.FileExists(target) Then WScript.Quit 0",
  378. 'Set sh = CreateObject("WScript.Shell")',
  379. f"sh.Run {_quote_vbs_string(command)}, 0, False",
  380. ]
  381. return "\r\n".join(lines) + "\r\n"
  382. def _write_task_script() -> Path:
  383. """Generate the gateway.cmd wrapper (kept as a compatibility artifact) and the console-less .vbs
  384. launcher used by the Scheduled Task and Startup fallback. Return the .cmd path."""
  385. _assert_windows()
  386. settings = _launcher_settings()
  387. script_path = get_task_script_path()
  388. _atomic_write(script_path, _build_gateway_cmd_script(*settings), script_path.with_suffix(".tmp"))
  389. # Also render the console-less .vbs launcher used by Scheduled Task and the Startup-folder fallback via
  390. # wscript.exe (issue #45599 fix A). The .cmd wrapper stays as a generated helper/compatibility artifact.
  391. vbs_path = script_path.with_suffix(".vbs")
  392. _atomic_write(vbs_path, _build_gateway_vbs_script(*settings), vbs_path.with_name(vbs_path.name + ".tmp"),
  393. encoding="utf-16")
  394. return script_path
  395. def _atomic_write(path: Path, content: str, tmp: Path, *, encoding: str = "utf-8") -> None:
  396. """Write ``content`` verbatim (no newline translation) via ``tmp`` then rename over ``path``.
  397. The staging file is removed even when the rename fails: the Startup-folder caller stages
  398. inside the Startup folder itself, and Windows opens every file there at login — a leftover
  399. ``Hermes_Gateway.tmp`` pops up in Notepad after every sign-in (#114093).
  400. ``.vbs`` launchers pass ``utf-16`` (BOM included): Windows Script Host decodes a BOM-less script
  401. with the ANSI code page, so a non-ASCII install path reached python as mojibake, ``PYTHONPATH``
  402. no longer imported ``hermes_cli`` and every task or Startup start died before logging a line.
  403. """
  404. try:
  405. tmp.write_text(content, encoding=encoding, newline="")
  406. tmp.replace(path)
  407. finally:
  408. try:
  409. tmp.unlink(missing_ok=True)
  410. except OSError:
  411. pass
  412. # ── Install / uninstall
  413. def _resolve_task_user() -> str | None:
  414. """Return ``DOMAIN\\USER`` if available, else bare USERNAME, else None."""
  415. username = os.environ.get("USERNAME") or os.environ.get("USER") or os.environ.get("LOGNAME")
  416. if not username:
  417. return None
  418. if "\\" in username:
  419. return username
  420. domain = os.environ.get("USERDOMAIN")
  421. return f"{domain}\\{username}" if domain else username
  422. def _build_scheduled_task_xml(task_name: str, launcher_path: Path, user: str | None) -> str:
  423. """Task Scheduler XML with safe long-running defaults. ``launcher_path`` is the console-less
  424. ``.vbs`` run via ``wscript.exe`` (see ``_build_gateway_vbs_script`` for why not cmd.exe).
  425. See #45599.
  426. """
  427. user_principal = f"\n <UserId>{escape(user)}</UserId>" if user else ""
  428. return f"""<?xml version="1.0" encoding="UTF-16"?>
  429. <Task version="1.4" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task">
  430. <RegistrationInfo>
  431. <Description>{escape(_TASK_DESCRIPTION)}</Description>
  432. </RegistrationInfo>
  433. <Triggers>
  434. <LogonTrigger>
  435. <Enabled>true</Enabled>
  436. <Delay>{_TASK_LOGON_DELAY}</Delay>
  437. </LogonTrigger>
  438. </Triggers>
  439. <Principals>
  440. <Principal id="Author">{user_principal}
  441. <LogonType>InteractiveToken</LogonType>
  442. <RunLevel>LeastPrivilege</RunLevel>
  443. </Principal>
  444. </Principals>
  445. <Settings>
  446. <MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy>
  447. <DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries>
  448. <StopIfGoingOnBatteries>false</StopIfGoingOnBatteries>
  449. <AllowHardTerminate>true</AllowHardTerminate>
  450. <StartWhenAvailable>true</StartWhenAvailable>
  451. <RunOnlyIfNetworkAvailable>false</RunOnlyIfNetworkAvailable>
  452. <IdleSettings>
  453. <StopOnIdleEnd>false</StopOnIdleEnd>
  454. <RestartOnIdle>false</RestartOnIdle>
  455. </IdleSettings>
  456. <AllowStartOnDemand>true</AllowStartOnDemand>
  457. <Enabled>true</Enabled>
  458. <Hidden>false</Hidden>
  459. <RunOnlyIfIdle>false</RunOnlyIfIdle>
  460. <WakeToRun>false</WakeToRun>
  461. <ExecutionTimeLimit>PT0S</ExecutionTimeLimit>
  462. <Priority>7</Priority>
  463. <RestartOnFailure>
  464. <Interval>{_TASK_RESTART_INTERVAL}</Interval>
  465. <Count>{_TASK_RESTART_COUNT}</Count>
  466. </RestartOnFailure>
  467. </Settings>
  468. <Actions Context="Author">
  469. <Exec>
  470. <Command>wscript.exe</Command>
  471. <Arguments>//B //Nologo "{escape(str(launcher_path))}"</Arguments>
  472. </Exec>
  473. </Actions>
  474. </Task>
  475. """
  476. def _install_scheduled_task(task_name: str, script_path: Path) -> tuple[bool, str]:
  477. """Create or replace the Scheduled Task. Returns (success, detail). Always delete+create, never
  478. ``/Change``: it preserves stale repeat/restart settings that relaunch the gateway every minute."""
  479. delete_code, delete_out, delete_err = _exec_schtasks(["/Delete", "/F", "/TN", task_name])
  480. delete_detail = (delete_err or delete_out or "").strip()
  481. if "cannot find" in delete_detail.lower():
  482. delete_detail = ""
  483. if delete_code != 0 and delete_detail and _is_access_denied(delete_detail):
  484. return (False, f"schtasks /Delete failed (code {delete_code}): {delete_detail}")
  485. # Other /Delete failures are non-fatal: /Create /F may still replace it; keep the detail.
  486. user = _resolve_task_user()
  487. launcher_path = script_path.with_suffix(".vbs") # the task launches the console-less .vbs
  488. xml_path = launcher_path.with_suffix(".task.xml")
  489. xml_path.write_text(_build_scheduled_task_xml(task_name, launcher_path, user), encoding="utf-16", newline="")
  490. # Immediate manual starts use _spawn_detached(). See #45599.
  491. base = ["/Create", "/F", "/TN", task_name, "/XML", str(xml_path)]
  492. variants = [[*base, "/RU", user, "/NP", "/IT"], base] if user else [base]
  493. last_code, last_err = 1, ""
  494. try:
  495. for argv in variants:
  496. code, out, err = _exec_schtasks(argv)
  497. if code == 0:
  498. return (True, f"Created Scheduled Task {task_name!r}")
  499. last_code, last_err = code, (err or out or "")
  500. finally:
  501. try:
  502. xml_path.unlink(missing_ok=True)
  503. except OSError:
  504. pass
  505. if delete_detail:
  506. last_err = f"{last_err.strip()} (delete detail: {delete_detail})"
  507. return (False, f"schtasks /Create failed (code {last_code}): {last_err.strip()}")
  508. def _install_startup_entry(script_path: Path) -> Path:
  509. """Write the Startup-folder fallback launcher. Returns its path."""
  510. entry = get_startup_entry_path()
  511. entry.parent.mkdir(parents=True, exist_ok=True)
  512. _atomic_write(entry, _build_startup_launcher(script_path), _startup_staging_path(), encoding="utf-16")
  513. legacy_entry = _legacy_startup_entry_path()
  514. try:
  515. if legacy_entry.exists():
  516. legacy_entry.unlink()
  517. except OSError:
  518. pass
  519. return entry
  520. def _remove_startup_entries() -> tuple[list[str], list[str]]:
  521. """Unlink the Startup-folder entries (``.vbs`` fallback + legacy ``.cmd``); ``(done, warnings)``.
  522. A failure (file locked, access denied) is reported rather than swallowed so callers warn
  523. instead of claiming a single autostart mechanism.
  524. """
  525. done: list[str] = []
  526. warnings: list[str] = []
  527. for path in (get_startup_entry_path(), _legacy_startup_entry_path()):
  528. try:
  529. path.unlink()
  530. done.append(f"Removed redundant Windows login item: {path}")
  531. except FileNotFoundError:
  532. pass
  533. except OSError:
  534. warnings.append(f"Could not remove redundant Windows login item: {path} (locked or access denied; it still fires at logon)")
  535. return done, warnings
  536. def redundant_autostart_entries() -> list[Path]:
  537. """Startup-folder entries that fire the gateway a second time at logon: every entry beside a
  538. registered Scheduled Task, or a legacy ``.cmd`` beside the ``.vbs`` fallback."""
  539. entries = [p for p in (get_startup_entry_path(), _legacy_startup_entry_path()) if p.exists()]
  540. if is_task_registered():
  541. return entries
  542. return entries[1:]
  543. def reconcile_autostart_launchers() -> tuple[list[str], list[str]]:
  544. """Converge gateway logon persistence to ONE mechanism; returns ``(done, warnings)`` messages.
  545. The Scheduled Task and the Startup-folder entry are alternatives, but a successful task install
  546. never removed an earlier fallback and pre-#45610 installs left a ``cmd.exe`` launcher behind, so
  547. logon could fire the launcher twice (#80569). Task registered: remove the Startup entries. No
  548. task but a legacy ``.cmd``: rewrite it as the console-less ``.vbs`` fallback. No task and only
  549. the ``.vbs`` entry: rewrite it in place so an older UTF-8 chainer becomes UTF-16 (W13); in
  550. practice only update reaches this, because doctor reconciles only when an entry is redundant.
  551. File operations only (no schtasks mutation, no elevation), so install, update and doctor can
  552. all run it.
  553. """
  554. if is_task_registered():
  555. return _remove_startup_entries()
  556. legacy = _legacy_startup_entry_path()
  557. if legacy.exists():
  558. entry = _install_startup_entry(_write_task_script())
  559. if legacy.exists(): # _install_startup_entry swallows the unlink failure; both would fire at logon
  560. return [], [f"Could not remove legacy Windows login item: {legacy} (locked or access denied; it still fires at logon beside {entry})"]
  561. return [f"Migrated legacy Windows login item to: {entry}"], []
  562. if get_startup_entry_path().exists():
  563. # An entry from before the UTF-16 launchers is read in the ANSI code page at logon: rewrite it.
  564. _install_startup_entry(get_task_script_path())
  565. return [], []
  566. def _resolve_detached_python(python_exe: str) -> tuple[str, Path, list[str]]:
  567. """Return (hidden_console_python, venv_dir, extra_pythonpath) for detached runs. ``extra_pythonpath``
  568. is always empty now; the tuple shape is kept so every call site stays unchanged.
  569. Returns the venv's **console** ``python.exe`` — deliberately NOT ``pythonw.exe``. Every detached launch
  570. path pairs this interpreter with a hidden-console mechanism (``CREATE_NO_WINDOW`` creationflags, or
  571. ``WScript.Shell.Run`` window style 0), so the daemon owns a single hidden console that all of its
  572. console-subsystem descendants (git, gh, cmd, node, wmic, powershell, …) inherit instead of each
  573. allocating a visible flashing one. A GUI-subsystem ``pythonw.exe`` daemon has NO console, which is what
  574. made every descendant spawn flash (#54220/#56747) and forced the endless per-call-site CREATE_NO_WINDOW
  575. sweep. Root cause isolated + A/B verified on Windows 11 by the desktop backend fix (commit aa2ae36c3f).
  576. - uv venv launcher: ``venv\\Scripts\\python.exe`` under ``CREATE_NO_WINDOW`` re-execs the base
  577. interpreter *windowless* — the child inherits the shim's hidden console, so no conhost flashes (the
  578. #52239 concern). The historical "CREATE_NO_WINDOW cannot suppress the second window" observations were
  579. made while ``DETACHED_PROCESS`` was in the flag bundle, where MSDN specifies CREATE_NO_WINDOW is IGNORED
  580. — the hide bit was dead, not ineffective. The base-interpreter + PYTHONPATH-overlay detour is therefore
  581. unnecessary; the venv shim resolves imports itself. - Console python restores stdout/stderr, so daemon
  582. logs flow normally.
  583. Legacy normalization: launchers and argv snapshots from pre-aa2ae36c3f installs lead with
  584. ``pythonw.exe``. When the sibling console ``python.exe`` exists, swap to it so respawns and regenerated
  585. launchers get the hidden-console design instead of resurrecting the console-less daemon (the
  586. #54220/#56747 flash class, plus the ``sys.stderr is None`` startup-crash class from #71671).
  587. """
  588. p = Path(python_exe)
  589. if p.name.lower() in ("pythonw.exe", "pythonw"):
  590. sibling = p.with_name("python.exe" if p.suffix else "python")
  591. try:
  592. if sibling.exists():
  593. p = sibling
  594. python_exe = str(sibling)
  595. except OSError:
  596. # Can't stat the sibling — keep the original interpreter: a console-less gateway is
  597. # worse than a hidden-console one, but a failed respawn is worse still.
  598. pass
  599. return (python_exe, p.parent.parent, [])
  600. def _prepend_pythonpath(env_overlay: dict[str, str], entries: list[str]) -> None:
  601. clean_entries = [entry for entry in entries if entry]
  602. if not clean_entries:
  603. return
  604. existing = os.environ.get("PYTHONPATH", "")
  605. if existing:
  606. clean_entries.append(existing)
  607. env_overlay["PYTHONPATH"] = os.pathsep.join(clean_entries)
  608. def _build_gateway_argv(home: Path | None = None) -> tuple[list[str], str, dict[str, str]]:
  609. """Build (argv, working_dir, env_overlay) for the gateway subprocess — the same logical command
  610. as gateway.cmd, assembled as a native argv so no cmd.exe layer sits in between."""
  611. _assert_windows()
  612. from hermes_cli.gateway import PROJECT_ROOT
  613. python_path, working_dir, hermes_home, profile_arg = _launcher_settings(home)
  614. python_exe, venv_dir, extra_pythonpath = _resolve_detached_python(python_path)
  615. env_overlay = {"HERMES_HOME": hermes_home, **dict(_GATEWAY_ENV), "VIRTUAL_ENV": _preserve_hermes_home_path(venv_dir)}
  616. _prepend_pythonpath(env_overlay, [_preserve_hermes_home_path(p) for p in (PROJECT_ROOT, *extra_pythonpath)])
  617. return _gateway_run_argv(python_exe, profile_arg), working_dir, env_overlay
  618. def windowless_gateway_restart_spec(run_argv: list[str], home: str | None = None) -> tuple[list[str], str, dict[str, str]]:
  619. """(argv, cwd, env overlay) for a hidden-console gateway respawn; arguments after the interpreter
  620. are preserved verbatim. Non-Windows or a non-python argv[0] → argv unchanged, empty overlay.
  621. The post-update restart paths build their respawn command from ``get_python_path()`` (the venv's console
  622. ``python.exe``). That is the right interpreter: the watcher launches it with ``CREATE_NO_WINDOW`` detach
  623. flags, so the respawned gateway owns a single hidden console that all of its descendants inherit —
  624. nothing flashes (#54220/#56747; the old pythonw.exe rewrite here produced a console-less gateway whose
  625. every console-subsystem child allocated a visible conhost). This helper now only normalizes the
  626. interpreter via ``_resolve_detached_python`` and supplies the stable cwd + env overlay (HERMES_HOME,
  627. VIRTUAL_ENV, PYTHONPATH) so the respawn doesn't depend on the watcher's transient working directory.
  628. ``HERMES_HOME`` is ``home`` when given: a replayed selectorless argv runs on whatever home its
  629. environment names, so the paused runtime's home, not the updater's, must come back.
  630. """
  631. if not run_argv or sys.platform != "win32":
  632. return run_argv, "", {}
  633. from hermes_cli.gateway import PROJECT_ROOT
  634. try:
  635. hidden_console_python, venv_dir, extra_pythonpath = _resolve_detached_python(run_argv[0])
  636. except Exception:
  637. return run_argv, "", {}
  638. try:
  639. hermes_home = str(Path(home or _hermes_home()).resolve())
  640. except Exception:
  641. hermes_home = ""
  642. env_overlay: dict[str, str] = {"PYTHONIOENCODING": "utf-8", "HERMES_GATEWAY_DETACHED": "1", "VIRTUAL_ENV": str(venv_dir)}
  643. if hermes_home:
  644. env_overlay["HERMES_HOME"] = hermes_home
  645. _prepend_pythonpath(env_overlay, [str(PROJECT_ROOT), *extra_pythonpath])
  646. return [hidden_console_python, *run_argv[1:]], _stable_gateway_working_dir(PROJECT_ROOT), env_overlay
  647. def _spawn_detached(script_path: Path | None = None, home: Path | None = None) -> int:
  648. """Launch the gateway as a fully detached background process (``script_path`` is ignored; kept
  649. for API symmetry; ``home`` spawns another profile's gateway — ``--profile`` is derived from it). Spawns python.exe directly — a cmd.exe shim inherits the parent console and
  650. gets reaped when the shell exits. Flags: CREATE_NEW_PROCESS_GROUP (no Ctrl+C from our group),
  651. CREATE_NO_WINDOW (hidden console descendants inherit, so nothing flashes — #54220/#56747; the old
  652. DETACHED_PROCESS made every descendant spawn flash), CREATE_BREAKAWAY_FROM_JOB
  653. (escape a parent Job Object — some Windows Terminal versions wrap children in one).
  654. With ``CREATE_NO_WINDOW`` the gateway gets its OWN hidden console instead of inheriting ours, so it
  655. survives our shell closing, and every console-subsystem descendant it spawns inherits that hidden
  656. console instead of flashing a visible one (#54220/#56747 — this is why we don't use console-less
  657. pythonw.exe here). Combined with CREATE_NEW_PROCESS_GROUP + DEVNULL stdin + a fresh env, the resulting
  658. process is independent of whichever shell started it.
  659. """
  660. _assert_windows()
  661. argv, working_dir, env_overlay = _build_gateway_argv(home)
  662. from tools.environments.local import served_profile_child_env
  663. # home=None is this process's own gateway, not a forced jump to the default root.
  664. # served_profile_child_env overlays that home's secrets instead of os.environ.copy().
  665. target = home if home is not None else _hermes_home()
  666. env = {**served_profile_child_env(target_home=target, inherit_credentials=True), **env_overlay}
  667. # Stray print()/native stderr goes to a sidecar log; real gateway logs still land in gateway.log
  668. # via the logging FileHandler.
  669. log_dir = _hermes_home() / "logs"
  670. log_dir.mkdir(parents=True, exist_ok=True)
  671. stray_log = log_dir / "gateway-stdio.log"
  672. def _popen(breakaway: str, flags: int):
  673. with open(stray_log, "ab", buffering=0) as log_fh:
  674. return subprocess.Popen(
  675. argv, cwd=working_dir, env={**env, _WINDOWS_GATEWAY_BREAKAWAY_ENV: breakaway}, creationflags=flags,
  676. close_fds=True, stdin=subprocess.DEVNULL, stdout=log_fh, stderr=log_fh,
  677. )
  678. try:
  679. proc = _popen("1", windows_detach_flags())
  680. _LAST_SPAWN_BREAKAWAY_FALLBACK["fallback"] = False
  681. except OSError as exc:
  682. # CREATE_BREAKAWAY_FROM_JOB fails with "access denied" when the parent's job object forbids
  683. # breakaway (some Windows Terminal configs). Retry without it — the hidden-console
  684. # CREATE_NO_WINDOW spawn is usually enough on its own.
  685. error_code = getattr(exc, "winerror", None)
  686. if error_code is None:
  687. error_code = exc.errno
  688. logger.warning("Gateway breakaway spawn failed (error=%s); retrying without CREATE_BREAKAWAY_FROM_JOB", error_code)
  689. proc = _popen("0", windows_detach_flags_without_breakaway())
  690. _LAST_SPAWN_BREAKAWAY_FALLBACK["fallback"] = True
  691. return proc.pid
  692. def _stdin_is_interactive(*, isatty: bool, console_mode_ok: bool | None) -> bool:
  693. """A human can answer a prompt only on a real console. The Windows CRT reports isatty()==True for
  694. every character device — the NUL device included (`hermes gateway start < NUL`, stdin=DEVNULL) — so
  695. isatty must be confirmed by GetConsoleMode accepting the handle (#113977). ``console_mode_ok`` is
  696. None where that fact does not exist (not Windows) and isatty alone decides."""
  697. return isatty and console_mode_ok is not False
  698. def _stdout_isatty() -> bool:
  699. """The question is printed to stdout. When stdout is captured, nobody sees it. Desktop update
  700. hand-offs before #122234 captured each step's stdout while leaving it the console's stdin, so a
  701. prompt there waited forever for an answer to a question nobody saw."""
  702. return sys.stdout is not None and sys.stdout.isatty()
  703. def _stdin_console_mode_ok() -> bool | None:
  704. if sys.platform != "win32":
  705. return None
  706. kernel32 = ctypes.windll.kernel32
  707. handle = kernel32.GetStdHandle(-10) # STD_INPUT_HANDLE
  708. return bool(kernel32.GetConsoleMode(handle, ctypes.byref(ctypes.c_ulong())))
  709. def _install_choice_from_env(name: str) -> bool | None:
  710. raw = os.environ.get(name)
  711. if raw is None:
  712. return None
  713. value = raw.strip().lower()
  714. if value in {"1", "true", "yes", "y", "on"}:
  715. return True
  716. if value in {"0", "false", "no", "n", "off"}:
  717. return False
  718. return None
  719. def _prompt_install_choices(start_now: bool | None = None, start_on_login: bool | None = None) -> tuple[bool, bool]:
  720. """Return (start_now, start_on_login), asking before any UAC escalation."""
  721. if start_now is None:
  722. start_now = _install_choice_from_env("HERMES_GATEWAY_INSTALL_START_NOW")
  723. if start_on_login is None:
  724. start_on_login = _install_choice_from_env("HERMES_GATEWAY_INSTALL_START_ON_LOGIN")
  725. if start_now is not None and start_on_login is not None:
  726. return start_now, start_on_login
  727. from hermes_cli.setup import prompt_yes_no
  728. if start_now is None:
  729. start_now = prompt_yes_no("Start the gateway now after install?", True)
  730. if start_on_login is None:
  731. start_on_login = prompt_yes_no("Start the gateway automatically on Windows login with a Scheduled Task?", True)
  732. return start_now, start_on_login
  733. def _report_already_running(running_pids: list[int]) -> None:
  734. print(f"✓ Gateway already running (PID: {', '.join(map(str, running_pids))})")
  735. def _start_or_report_running(running_pids: list[int] | None = None) -> None:
  736. """Spawn the gateway unless one is already running for this profile."""
  737. if running_pids is None:
  738. running_pids = _gateway_pids()
  739. if running_pids:
  740. _report_already_running(running_pids)
  741. else:
  742. _spawn_detached()
  743. _report_gateway_start("direct spawn")
  744. def _install_startup_fallback(script_path: Path, start_now: bool, detail: str) -> None:
  745. """Install the Startup-folder fallback and optionally start once."""
  746. print(f"↻ Scheduled Task install blocked ({detail.splitlines()[0]}) — using Startup folder fallback")
  747. if is_task_registered():
  748. # An earlier task survives (UAC declined, access denied on re-create) and still fires at
  749. # logon; adding the fallback beside it would start the gateway twice (#80569).
  750. print("⚠ Scheduled Task is still registered — skipped the Startup fallback to avoid a duplicate autostart.")
  751. print(" If that task is disabled or broken, run 'hermes gateway uninstall', then install again.")
  752. else:
  753. entry = _install_startup_entry(script_path)
  754. print(f"✓ Installed Windows login item: {entry}")
  755. print(f" Task script: {script_path}")
  756. # Re-running install must be safe: the fallback only installs login persistence; starting is
  757. # controlled by the pre-UAC start_now answer so every user decision precedes elevation.
  758. running_pids = _gateway_pids()
  759. if running_pids or start_now:
  760. _start_or_report_running(running_pids)
  761. else:
  762. from hermes_cli.gateway import _profile_arg
  763. profile_arg = _profile_arg()
  764. start_cmd = f"hermes {profile_arg} gateway start" if profile_arg else "hermes gateway start"
  765. print("ℹ Startup fallback installed; gateway not started now.")
  766. print(f" Start manually with: {start_cmd}")
  767. _print_next_steps()
  768. def _offer_elevated_install(headline: str, force: bool, start_now: bool, start_on_login: bool) -> bool:
  769. """Offer the UAC prompt for a Scheduled Task install. True when handed off to an elevated child."""
  770. from hermes_cli.setup import prompt_yes_no
  771. print(headline)
  772. print(" UAC is Windows' admin approval prompt; it is needed to create/update the Scheduled Task.")
  773. if prompt_yes_no(" Open the UAC prompt now?", False):
  774. if _launch_elevated_install(force=force, start_now=start_now, start_on_login=start_on_login):
  775. print("✓ Launched elevated Hermes gateway install prompt.")
  776. if start_now:
  777. print(" Approve the Windows UAC prompt; the elevated install will start the gateway afterwards.")
  778. else:
  779. print(" Approve the Windows UAC prompt, then run: hermes gateway status")
  780. return True
  781. print("⚠ Falling back to Startup folder because elevation was unavailable or cancelled.")
  782. else:
  783. print(" Skipped elevation. Falling back to Startup folder.")
  784. return False
  785. def install(
  786. force: bool = False, *, start_now: bool | None = None, start_on_login: bool | None = None,
  787. elevated_handoff: bool = False,
  788. ) -> None:
  789. """Install the gateway as a Windows Scheduled Task (with Startup fallback). Idempotent — we
  790. always reconcile; ``force`` exists for API parity with launchd/systemd."""
  791. _assert_windows()
  792. start_now, start_on_login = _prompt_install_choices(start_now, start_on_login)
  793. if not start_on_login:
  794. print("ℹ Skipped Windows login auto-start install.")
  795. if start_now:
  796. _start_or_report_running()
  797. else:
  798. print("ℹ Gateway not started and no auto-start service installed.")
  799. print(" Run in the foreground later with: hermes gateway run")
  800. return
  801. task_name = get_task_name()
  802. script_path = _write_task_script()
  803. # A pre-fix install that failed its Startup-folder swap left `Hermes_Gateway.tmp` there, and the
  804. # Scheduled Task path below never touches that folder — sweep it so a re-run clears the debris.
  805. try:
  806. _startup_staging_path().unlink(missing_ok=True)
  807. except OSError:
  808. pass
  809. if force:
  810. # Pre-suffix strays (task ``Hermes_Gateway``, Startup ``Hermes_Gateway.vbs``) are unreachable by
  811. # the current names, so a plain reconcile never heals them (#116157).
  812. from hermes_cli.gateway_windows_legacy import remove_legacy_launchers
  813. remove_legacy_launchers()
  814. # On locked-down accounts schtasks can sit for the full timeout before returning Access Denied.
  815. # All intent questions were asked above, so ask for UAC before touching schtasks.
  816. if not _is_running_as_admin() and not elevated_handoff:
  817. if _offer_elevated_install(
  818. "↻ Scheduled Task install may need administrator approval on this Windows account.",
  819. force, start_now, start_on_login,
  820. ):
  821. return
  822. _install_startup_fallback(script_path, start_now, "administrator approval was not used")
  823. return
  824. ok, detail = _install_scheduled_task(task_name, script_path)
  825. if ok:
  826. print(f"✓ {detail}")
  827. print(f" Task script: {script_path}")
  828. print("ℹ Gateway auto-start installed for Windows login.")
  829. # A Startup-folder entry from an earlier fallback install would fire alongside the task (#80569).
  830. done, warnings = _remove_startup_entries()
  831. for message in done:
  832. print(f"✓ {message}")
  833. for message in warnings:
  834. print(f"⚠ {message}")
  835. if start_now:
  836. _start_or_report_running()
  837. else:
  838. print("ℹ Gateway not started now.")
  839. print(" Start manually with: hermes gateway start")
  840. _print_next_steps()
  841. return
  842. # Prefer a real Scheduled Task over the Startup fallback when elevation is the only blocker.
  843. if _is_access_denied(detail) and not _is_running_as_admin() and _offer_elevated_install(
  844. f"↻ Scheduled Task install needs administrator approval ({detail.splitlines()[0]})",
  845. force, start_now, start_on_login,
  846. ):
  847. return
  848. if _should_fall_back(1, detail):
  849. _install_startup_fallback(script_path, start_now, detail)
  850. return
  851. raise RuntimeError(f"Windows gateway install failed: {detail}")
  852. def _live_gateway_pids(
  853. all_profiles: bool = False, home: Path | None = None, pid_filter=None
  854. ) -> list[int]:
  855. """Live gateway PIDs for the readiness poll. ``home`` scopes the probe to ONE profile's identity
  856. files (a still-running sibling must not vouch for a per-profile spawn, #110959); otherwise the
  857. process-table discovery for the active profile or the whole fleet."""
  858. if home is not None:
  859. from gateway.status import get_running_pid
  860. pid = get_running_pid(home / "gateway.pid", cleanup_stale=False)
  861. pids = [pid] if pid else []
  862. else:
  863. from hermes_cli.gateway import find_gateway_pids
  864. pids = list(find_gateway_pids(all_profiles=all_profiles))
  865. return list(pid_filter(pids)) if pid_filter is not None else pids
  866. def _confirm_gateway_stable(
  867. initial_pids: list[int], confirm_s: float, interval_s: float, all_profiles: bool = False,
  868. home: Path | None = None, pid_filter=None,
  869. ) -> list[int]:
  870. """Re-check a freshly detected gateway for ``confirm_s`` seconds: one process-table hit proves
  871. the child was *created*, not that it survived startup (or a parent Job Object teardown).
  872. A single process-table hit only proves the child was *created*, not that it survived startup — a gateway
  873. that crashes moments after spawn (or is reaped by the parent shell's Job Object, #91675/#84185) passes a
  874. first-hit poll and then dies. Require the gateway to stay visible for the whole confirmation window
  875. before we vouch for it. Returns the last observed PID list, or ``[]`` if the gateway vanished
  876. mid-window.
  877. """
  878. if confirm_s <= 0:
  879. return initial_pids
  880. pids = initial_pids
  881. confirm_deadline = time.monotonic() + confirm_s
  882. while time.monotonic() < confirm_deadline:
  883. time.sleep(interval_s)
  884. pids = _live_gateway_pids(all_profiles, home, pid_filter)
  885. if not pids:
  886. return []
  887. return pids
  888. def _wait_for_gateway_ready(
  889. timeout_s: float = 6.0, interval_s: float = 0.4, confirm_s: float = 2.0, all_profiles: bool = False,
  890. home: Path | None = None, pid_filter=None,
  891. ) -> list[int]:
  892. """Poll for a live gateway for up to ``timeout_s``; a first hit is provisional until the gateway
  893. stays visible for ``confirm_s`` more seconds (a child that dies right after spawn earns no ✓)."""
  894. deadline = time.monotonic() + timeout_s
  895. while time.monotonic() < deadline:
  896. pids = _live_gateway_pids(all_profiles, home, pid_filter)
  897. if pids:
  898. confirmed = _confirm_gateway_stable(
  899. pids, confirm_s, interval_s, all_profiles=all_profiles, home=home, pid_filter=pid_filter
  900. )
  901. if confirmed:
  902. return confirmed
  903. continue # died during confirmation — keep polling until deadline
  904. time.sleep(interval_s)
  905. return []
  906. # ---------------------------------------------------------------------------
  907. # Start attestation — honest reporting for deaths AFTER the liveness poll
  908. #
  909. # The poll cannot observe a death after this CLI process exits (the parent shell's Job Object tears
  910. # the gateway down on CLI exit). So every ✓ persists a marker recording which PIDs we vouched for;
  911. # the NEXT gateway CLI invocation checks it: PIDs gone with no clean exit in the lifecycle ledger
  912. # means the earlier ✓ was a lie, and we say so — once — with the schtasks recovery hint.
  913. # ---------------------------------------------------------------------------
  914. _START_ATTESTATION_RELATIVE = ("state", "gateway.start-attestation.json")
  915. def _start_attestation_path(home: Path | None = None) -> Path:
  916. """Marker path; ``home`` addresses another profile's marker (per-profile cold-start, #110959)."""
  917. return (home if home is not None else _hermes_home()).joinpath(*_START_ATTESTATION_RELATIVE)
  918. def _write_start_attestation(pids: list[int], via: str, home: Path | None = None) -> None:
  919. """Persist the PIDs a ✓ vouched for. Best-effort, never raises.
  920. ``generation`` identifies this marker instance: the update resume token records the generation
  921. whose death authorized a cold-start, so execution consumes exactly that marker and never a
  922. newer one written by a concurrent ``hermes gateway start`` (#110020 review)."""
  923. try:
  924. path = _start_attestation_path(home)
  925. path.parent.mkdir(parents=True, exist_ok=True)
  926. from hermes_cli.process_identity import _process_create_time
  927. payload = {
  928. "pids": [int(p) for p in pids], "via": via, "ts": datetime.now(UTC).isoformat(),
  929. "generation": uuid.uuid4().hex,
  930. }
  931. # Bind each PID to its incarnation (#110020 review): the ledger sentinel is matched by PID
  932. # only otherwise, so a stale marker would be re-read against whatever lifecycle wrote last.
  933. create_times = {str(int(p)): _process_create_time(int(p)) for p in pids}
  934. payload["create_times"] = {k: v for k, v in create_times.items() if v is not None}
  935. tmp = path.with_suffix(".json.tmp")
  936. tmp.write_text(json.dumps(payload), encoding="utf-8")
  937. tmp.replace(path)
  938. except Exception:
  939. logger.debug("Failed to write gateway start attestation", exc_info=True)
  940. def _clear_start_attestation(home: Path | None = None) -> None:
  941. try:
  942. _start_attestation_path(home).unlink(missing_ok=True)
  943. except OSError:
  944. pass
  945. # A start attestation older than this is no authority (#110020 review (d)): the marker is a one-shot
  946. # meant to bridge the seconds between a ✓ and the next ``hermes gateway status``/``update``; a
  947. # historical marker must never later override Desktop ownership into a duplicate gateway (#76129).
  948. START_ATTESTATION_MAX_AGE_S = 24 * 3600
  949. # Same slack process_identity uses for psutil create_time comparisons (PID reuse disambiguation).
  950. _CREATE_TIME_TOLERANCE_S = 2.0
  951. # A backwards clock step (NTP) between write and read must not kill a fresh marker.
  952. _ATTESTATION_CLOCK_SLACK_S = 60.0
  953. def _attestation_within_horizon(data: object) -> bool:
  954. """False for a marker whose ``ts`` is missing, unparsable or older than the horizon (fail closed)."""
  955. try:
  956. ts = datetime.fromisoformat(str(data["ts"])) if isinstance(data, dict) else None
  957. if ts is None:
  958. return False
  959. if ts.tzinfo is None:
  960. ts = ts.replace(tzinfo=UTC)
  961. age = time.time() - ts.timestamp()
  962. return -_ATTESTATION_CLOCK_SLACK_S <= age <= START_ATTESTATION_MAX_AGE_S
  963. except Exception:
  964. return False
  965. def _attestation_generation(data: object) -> str | None:
  966. """The marker instance identity, or ``None`` for a marker that carries none."""
  967. return str(data["generation"]) if isinstance(data, dict) and data.get("generation") else None
  968. def _consume_start_attestation(generation: str, home: Path | None = None) -> None:
  969. """Clear the marker only while it is still the ``generation`` that was acted on; a newer
  970. marker belongs to a gateway start this caller knows nothing about and keeps its own report."""
  971. # Best-effort read-then-unlink: a marker written in between loses one post-start report, never authority.
  972. if _attestation_generation(_read_start_attestation(home)) == generation:
  973. _clear_start_attestation(home)
  974. def _read_start_attestation(home: Path | None = None) -> object | None:
  975. """Parsed attestation payload (any JSON type), or ``None`` when absent/unreadable. Never raises."""
  976. try:
  977. return json.loads(_start_attestation_path(home).read_text(encoding="utf-8-sig"))
  978. except (OSError, ValueError):
  979. return None
  980. def _attested_pids_from(data: object) -> list[int]:
  981. """PID list from an attestation payload; empty for anything malformed."""
  982. if not isinstance(data, dict):
  983. return []
  984. pids = data.get("pids")
  985. # Fail closed: a null/malformed marker must never authorize a cold start (or raise on iteration).
  986. # Exact positive ints only — ``isinstance(True, int)`` holds, and 0 / negatives are not PIDs; one
  987. # bad item taints the whole list because the writer never emits such values.
  988. if not isinstance(pids, list) or not all(type(p) is int and p > 0 for p in pids):
  989. return []
  990. return list(pids)
  991. def _attested_create_time(data: object, pid: int) -> float | None:
  992. """The process create time the marker bound ``pid`` to, or ``None`` (older marker / psutil silent)."""
  993. times = data.get("create_times") if isinstance(data, dict) else None
  994. value = times.get(str(pid)) if isinstance(times, dict) else None
  995. return float(value) if type(value) in (int, float) else None
  996. def _attested_pid_exited_cleanly(pid: int, create_time: float | None = None, home: Path | None = None) -> bool:
  997. """True when the lifecycle ledger shows a clean exit for ``pid`` — or, for a marker that bound
  998. ``pid`` to a ``create_time``, whenever the sentinel cannot be shown to describe THAT incarnation
  999. (#110020 review): a sentinel for another PID or another start time means an unrelated lifecycle
  1000. has run since and the marker is stale; "unknown" must never read as "dead". A missing sentinel
  1001. still reads as dead (the attested process never booted far enough to claim it)."""
  1002. try:
  1003. from gateway.lifecycle_ledger import get_lifecycle_sentinel_path
  1004. sentinel = get_lifecycle_sentinel_path(home if home is not None else _hermes_home())
  1005. data = json.loads(sentinel.read_text(encoding="utf-8-sig"))
  1006. except OSError:
  1007. return False
  1008. except Exception:
  1009. return create_time is not None
  1010. if not isinstance(data, dict):
  1011. return create_time is not None
  1012. if create_time is not None:
  1013. if data.get("pid") != pid:
  1014. return True
  1015. sentinel_birth = data.get("create_time")
  1016. # A sentinel from a gateway older than the identity stamp cannot be told apart: PID-only rule.
  1017. if type(sentinel_birth) in (int, float) and abs(float(sentinel_birth) - create_time) > _CREATE_TIME_TOLERANCE_S:
  1018. return True
  1019. return data.get("phase") == "exited" and data.get("pid") == pid
  1020. def _attested_dead(
  1021. attested: list[int], current_pids: list[int], data: object = None, home: Path | None = None
  1022. ) -> bool:
  1023. """The liveness rule shared by the consuming and read-only probes: attested PIDs are dead when
  1024. no gateway runs now and the lifecycle ledger shows no clean exit for any of them."""
  1025. return not current_pids and not any(
  1026. _attested_pid_exited_cleanly(pid, _attested_create_time(data, pid), home) for pid in attested
  1027. )
  1028. def attested_death_generation(current_pids: list[int], home: Path | None = None) -> str | None:
  1029. """The generation of a start attestation that vouches for gateway PID(s) gone without a clean exit,
  1030. or ``None``.
  1031. Read-only twin of :func:`check_start_attestation` for callers that must not consume the
  1032. one-shot marker — ``hermes update`` consults it to decide whether a Desktop-owned install
  1033. still owes a gateway cold-start (#109538) and records the generation in its resume token so the
  1034. execution step consumes exactly the marker it was authorized by. Callers pass the liveness they
  1035. already established (``[]`` after their own discovery came back empty) so the process table is
  1036. not scanned twice. ``None`` for anything undecidable (no marker, no generation, a clean ledger
  1037. exit): "unknown" must never read as "dead". ``home`` probes another profile's marker and ledger
  1038. (the updater evaluates every profile that is not running, #110959)."""
  1039. data = _read_start_attestation(home)
  1040. attested = _attested_pids_from(data)
  1041. if not attested or not _attestation_within_horizon(data) or not _attested_dead(attested, current_pids, data, home):
  1042. return None
  1043. return _attestation_generation(data)
  1044. def check_start_attestation(current_pids: list[int] | None = None) -> str | None:
  1045. """Surface (once) a gateway that died after a ✓ was printed for it. Never raises. Gateway running
  1046. or a clean-exit ledger record: clear silently; otherwise return a warning and consume the marker."""
  1047. data = _read_start_attestation()
  1048. if data is None:
  1049. return None
  1050. attested = _attested_pids_from(data)
  1051. if not attested:
  1052. _clear_start_attestation()
  1053. return None
  1054. if current_pids is None:
  1055. try:
  1056. from hermes_cli.gateway import find_gateway_pids
  1057. current_pids = list(find_gateway_pids())
  1058. except Exception:
  1059. return None
  1060. _clear_start_attestation()
  1061. if not _attested_dead(attested, current_pids, data):
  1062. return None
  1063. return _format_attestation_warning(attested, data)
  1064. def _format_attestation_warning(attested: list[int], data: dict) -> str:
  1065. via = data.get("via") or "direct spawn"
  1066. ts = data.get("ts") or "unknown time"
  1067. lines = [
  1068. f"⚠ The previous gateway start ({via}, {ts}) reported success, but the "
  1069. f"process (PID {', '.join(map(str, attested))}) died without a clean "
  1070. "shutdown record.",
  1071. " This usually means the shell that ran `hermes gateway start` was inside "
  1072. "a Windows Job Object that killed the gateway on exit (#91675).",
  1073. ]
  1074. hint = _task_run_hint(" Recovery: schtasks /Run /TN {} (Task Scheduler starts the gateway outside any Job Object)")
  1075. if hint:
  1076. lines.append(hint)
  1077. return "\n".join(lines)
  1078. def _task_run_hint(fmt: str) -> str | None:
  1079. """``fmt`` with the task name filled in, when a Scheduled Task is registered. Never raises."""
  1080. try:
  1081. if is_task_registered():
  1082. return fmt.format(get_task_name())
  1083. except Exception:
  1084. pass
  1085. return None
  1086. def _print_task_run_hint(fmt: str) -> None:
  1087. hint = _task_run_hint(fmt)
  1088. if hint:
  1089. print(hint)
  1090. def _print_start_attestation_warning() -> None:
  1091. """Print the stale-attestation warning if one is pending. Never raises."""
  1092. try:
  1093. warning = check_start_attestation()
  1094. except Exception:
  1095. return
  1096. if warning:
  1097. print(warning)
  1098. def _report_gateway_start(via: str) -> None:
  1099. pids = _wait_for_gateway_ready()
  1100. if pids:
  1101. print(f"✓ Gateway started via {via} (PID: {', '.join(map(str, pids))})")
  1102. if _LAST_SPAWN_BREAKAWAY_FALLBACK.get("fallback"):
  1103. print("⚠ The gateway could not break away from this shell's Job Object; it may be killed when this shell exits.")
  1104. _print_task_run_hint(" If it dies, start it with: schtasks /Run /TN {}")
  1105. _write_start_attestation(pids, via)
  1106. else:
  1107. print(f"✗ Gateway start via {via} FAILED — no stable gateway process detected within the verification window.")
  1108. print(" (The process may have been created and then killed — e.g. by a parent Job Object, #91675.)")
  1109. print(f" Check the log for startup errors:\n type {_hermes_home()}\\logs\\gateway.log\n type {_hermes_home()}\\logs\\gateway-stdio.log")
  1110. _print_task_run_hint(" Recovery: schtasks /Run /TN {} (starts the gateway outside any Job Object)")
  1111. def _print_next_steps() -> None:
  1112. print("\nNext steps:\n hermes gateway status # Check status")
  1113. print(f" type {_hermes_home()}\\logs\\gateway.log # View logs")
  1114. def uninstall() -> None:
  1115. """Remove both the Scheduled Task and the Startup-folder fallback, if present."""
  1116. _assert_windows()
  1117. task_name = get_task_name()
  1118. script_path = get_task_script_path()
  1119. scheduled_task_removed = False
  1120. if is_task_registered():
  1121. code, _out, err = _exec_schtasks(["/Delete", "/F", "/TN", task_name])
  1122. detail = err.strip()
  1123. if code == 0:
  1124. scheduled_task_removed = True
  1125. print(f"✓ Removed Scheduled Task {task_name!r}")
  1126. elif _is_access_denied(detail) and not _is_running_as_admin():
  1127. from hermes_cli.setup import prompt_yes_no
  1128. print(f"↻ Scheduled Task uninstall needs administrator approval ({detail or 'access denied'})")
  1129. print(" UAC is Windows' admin approval prompt; it is needed to remove the Scheduled Task.")
  1130. if prompt_yes_no(" Open the UAC prompt now?", False):
  1131. if _launch_elevated_gateway_command("uninstall"):
  1132. print("✓ Launched elevated Hermes gateway uninstall prompt.")
  1133. print(" Approve the Windows UAC prompt, then run: hermes gateway status")
  1134. return
  1135. print("⚠ Elevated uninstall prompt was unavailable or cancelled.")
  1136. else:
  1137. print(" Skipped elevation. Scheduled Task was not removed.")
  1138. else:
  1139. print(f"⚠ schtasks /Delete returned code {code}: {detail}")
  1140. for path, label in (
  1141. (get_startup_entry_path(), "Windows login item"), (_legacy_startup_entry_path(), "legacy Windows login item"),
  1142. (_startup_staging_path(), "Windows login item staging file"),
  1143. (script_path, "Task script"), (script_path.with_suffix(".vbs"), "Task launcher"),
  1144. ):
  1145. try:
  1146. path.unlink()
  1147. print(f"✓ Removed {label}: {path}")
  1148. except FileNotFoundError:
  1149. pass
  1150. from hermes_cli.gateway_windows_legacy import remove_legacy_launchers
  1151. remove_legacy_launchers()
  1152. if is_task_registered() and not scheduled_task_removed:
  1153. print(f"⚠ Scheduled Task still registered: {task_name}")
  1154. # ── Status / start / stop / restart
  1155. def is_task_registered() -> bool:
  1156. code, _out, _err = _exec_schtasks(["/Query", "/TN", get_task_name()])
  1157. return code == 0
  1158. def is_startup_entry_installed() -> bool:
  1159. return get_startup_entry_path().exists() or _legacy_startup_entry_path().exists()
  1160. def _query_scheduled_task_xml(task_name: str) -> str | None:
  1161. """Return a registered task's XML, or ``None`` when it cannot be inspected (fail open: a
  1162. localized ``schtasks`` failure is not evidence about an otherwise working task)."""
  1163. code, out, err = _exec_schtasks(["/Query", "/TN", task_name, "/XML"])
  1164. if code != 0 or not out.strip():
  1165. logger.debug("Could not query Scheduled Task XML for %r: %s", task_name, (err or out).strip())
  1166. return None
  1167. return out
  1168. # The install probe runs before the desktop backend says ready: a wedged schtasks must not hold that.
  1169. _TASK_PROBE_TIMEOUT_S = 5
  1170. def registered_task_launchers() -> list[str]:
  1171. """The scripts this OS user's ``Hermes_Gateway*`` Scheduled Tasks start (``install_state``). A task
  1172. install removes the Startup-folder entries, so for a custom home the task is all that still names
  1173. it. Empty off Windows, and when Task Scheduler does not answer, as for an unreadable Startup entry."""
  1174. if sys.platform != "win32":
  1175. return []
  1176. from hermes_cli.gateway_service_owner import service_definition_launchers
  1177. code, listing, _err = _exec_schtasks(["/Query", "/FO", "CSV", "/NH"], timeout=_TASK_PROBE_TIMEOUT_S)
  1178. names = re.findall(rf'^"\\({_TASK_NAME_DEFAULT}[^"\\]*)"', listing, flags=re.MULTILINE) if code == 0 else []
  1179. launchers: list[str] = []
  1180. for name in dict.fromkeys(names):
  1181. code, xml, _err = _exec_schtasks(["/Query", "/TN", name, "/XML"], timeout=_TASK_PROBE_TIMEOUT_S)
  1182. leaves = (_task_xml_leaf_values(xml) if code == 0 else None) or {}
  1183. command = leaves.get("Task/Actions/Exec/Command", "").strip('"')
  1184. launchers += service_definition_launchers(f'"{command}" {leaves.get("Task/Actions/Exec/Arguments", "")}')
  1185. return launchers
  1186. def _task_xml_leaf_values(xml: str) -> dict[str, str] | None:
  1187. """Namespace-agnostic ``Task/Settings/...`` leaf-path → text map, or ``None`` for invalid XML.
  1188. The root ``version`` attribute is exposed as ``Task@version``."""
  1189. try:
  1190. root = ElementTree.fromstring(xml)
  1191. except ElementTree.ParseError:
  1192. return None
  1193. values: dict[str, str] = {"Task@version": root.attrib.get("version", "")}
  1194. def visit(element: ElementTree.Element, path: tuple[str, ...]) -> None:
  1195. current_path = (*path, element.tag.rsplit("}", 1)[-1])
  1196. children = list(element)
  1197. if not children:
  1198. values["/".join(current_path)] = " ".join((element.text or "").split())
  1199. for child in children:
  1200. visit(child, current_path)
  1201. visit(root, ())
  1202. return values
  1203. # Allowlist of template leaves whose absence/mismatch on the live task means it predates the current
  1204. # template (#113670). Never a full-leaf compare: schtasks exports <UserId> as a SID while the template
  1205. # writes DOMAIN\user, so equality would flag every healthy registration.
  1206. _TASK_DRIFT_LEAVES = {
  1207. "Task/Settings/RestartOnFailure/Interval": "RestartOnFailure",
  1208. "Task/Triggers/LogonTrigger/Delay": "LogonTrigger Delay",
  1209. "Task/Actions/Exec/Arguments": "launcher arguments",
  1210. }
  1211. def compare_scheduled_task_drift(registered_xml: str, template_xml: str) -> list[str]:
  1212. """Human-readable drift fragments between a registered task export and the current template,
  1213. over ``_TASK_DRIFT_LEAVES`` plus the Task ``version``. Empty when aligned or when either side
  1214. does not parse (fail open)."""
  1215. live = _task_xml_leaf_values(registered_xml)
  1216. want = _task_xml_leaf_values(template_xml)
  1217. if live is None or want is None:
  1218. return []
  1219. missing = [label for path, label in _TASK_DRIFT_LEAVES.items() if path in want and path not in live]
  1220. differs = [label for path, label in _TASK_DRIFT_LEAVES.items() if path in want and path in live and live[path] != want[path]]
  1221. drift = []
  1222. if missing:
  1223. drift.append(f"missing: {', '.join(missing)}")
  1224. drift.extend(f"{label} differs" for label in differs)
  1225. if live["Task@version"] != want["Task@version"]:
  1226. drift.append(f"version {live['Task@version']} vs {want['Task@version']}")
  1227. return drift
  1228. def scheduled_task_drift(task_name: str) -> list[str]:
  1229. """Drift fragments between the registered task and ``_build_scheduled_task_xml``; empty when
  1230. aligned or when the task cannot be queried."""
  1231. registered = _query_scheduled_task_xml(task_name)
  1232. if registered is None:
  1233. return []
  1234. template = _build_scheduled_task_xml(task_name, get_task_script_path().with_suffix(".vbs"), _resolve_task_user())
  1235. return compare_scheduled_task_drift(registered, template)
  1236. def _print_scheduled_task_drift(task_name: str) -> None:
  1237. """Warn when the registered task predates the current template (status is read-only; the
  1238. repair runs from ``start()`` / ``hermes update`` via ``reconcile_scheduled_task``)."""
  1239. drift = scheduled_task_drift(task_name)
  1240. if drift:
  1241. print(f"⚠ Scheduled Task registration predates the current template ({'; '.join(drift)})")
  1242. print(" Repair: hermes gateway start (or: hermes gateway install)")
  1243. def reconcile_scheduled_task(task_name: str) -> bool:
  1244. """Re-register the task from the current template when it drifts (#113670) — the Windows sibling
  1245. of ``gateway.py::refresh_systemd_unit_if_needed``. Template hardening (``RestartOnFailure``, logon
  1246. ``Delay``) otherwise only ever reaches fresh installs. False when aligned/unqueryable or when
  1247. ``schtasks`` refused (typically Access Denied — the elevating ``hermes gateway install`` is the fallback)."""
  1248. drift = scheduled_task_drift(task_name)
  1249. if not drift:
  1250. return False
  1251. print(f"↻ Repairing outdated Scheduled Task registration ({'; '.join(drift)})")
  1252. ok, detail = _install_scheduled_task(task_name, _write_task_script())
  1253. print(f"{'✓' if ok else '⚠'} {detail}")
  1254. if not ok:
  1255. print(" Repair manually: hermes gateway install")
  1256. return ok
  1257. def is_installed() -> bool:
  1258. """True when either the schtasks entry or the Startup fallback is present."""
  1259. return is_task_registered() or is_startup_entry_installed()
  1260. def query_task_status() -> dict[str, str]:
  1261. """Parse ``schtasks /Query /V /FO LIST`` and pull the interesting keys."""
  1262. code, out, _err = _exec_schtasks(["/Query", "/TN", get_task_name(), "/V", "/FO", "LIST"])
  1263. if code != 0:
  1264. return {}
  1265. info: dict[str, str] = {}
  1266. for raw in out.splitlines():
  1267. line = raw.strip()
  1268. if not line or ":" not in line:
  1269. continue
  1270. key, _, value = line.partition(":")
  1271. key = key.strip().lower()
  1272. value = value.strip()
  1273. # Some Windows locales emit "Last Result" instead of "Last Run Result".
  1274. if key == "last result":
  1275. info.setdefault("last run result", value)
  1276. elif key in {"status", "last run time", "last run result"}:
  1277. info[key] = value
  1278. return info
  1279. def _gateway_pids() -> list[int]:
  1280. """Reuse the cross-platform PID scanner in gateway.py."""
  1281. from hermes_cli.gateway import find_gateway_pids
  1282. return list(find_gateway_pids())
  1283. def _probe(index: int, ok: bool, message: str) -> None:
  1284. print(f" [{index}] {'PASS' if ok else 'FAIL':4s} {message}")
  1285. def _probe_missing(index: int, path: Path, label: str) -> bool:
  1286. if path.exists():
  1287. return False
  1288. _probe(index, False, f"{label} missing: {path}")
  1289. return True
  1290. def _probe_pid_file(pid_path: Path) -> int | None:
  1291. if _probe_missing(1, pid_path, "PID file"):
  1292. return None
  1293. try:
  1294. data = json.loads(pid_path.read_text(encoding="utf-8-sig"))
  1295. pid_value = int(data.get("pid")) if data.get("pid") is not None else None
  1296. _probe(1, True, f"PID file present: {pid_path} (pid={pid_value})")
  1297. return pid_value
  1298. except Exception as exc:
  1299. _probe(1, False, f"PID file present but unreadable: {exc}")
  1300. return None
  1301. def _probe_lock_file(lock_path: Path) -> None:
  1302. if _probe_missing(2, lock_path, "Lock file"):
  1303. return
  1304. try:
  1305. from gateway.status import is_gateway_runtime_lock_active
  1306. _probe(2, is_gateway_runtime_lock_active(lock_path), f"Lock file held by a live process: {lock_path}")
  1307. except Exception as exc:
  1308. _probe(2, False, f"Could not probe lock: {exc}")
  1309. def _probe_running_pid() -> int | None:
  1310. try:
  1311. from gateway.status import get_running_pid
  1312. running_pid = get_running_pid(cleanup_stale=False)
  1313. _probe(3, running_pid is not None, f"get_running_pid() => {running_pid}")
  1314. return running_pid
  1315. except Exception as exc:
  1316. _probe(3, False, f"get_running_pid() raised: {exc!r}")
  1317. return None
  1318. def _probe_pid_exists(candidate_pid: int | None) -> None:
  1319. if candidate_pid is None:
  1320. _probe(4, False, "No candidate PID to verify")
  1321. return
  1322. try:
  1323. from gateway.status import _pid_exists
  1324. alive = bool(_pid_exists(candidate_pid))
  1325. _probe(4, alive, f"_pid_exists({candidate_pid}) => {alive}")
  1326. except Exception as exc:
  1327. _probe(4, False, f"_pid_exists raised: {exc!r}")
  1328. def _probe_state_file(state_path: Path) -> None:
  1329. if _probe_missing(5, state_path, "gateway_state.json"):
  1330. return
  1331. try:
  1332. state_data = json.loads(state_path.read_text(encoding="utf-8-sig"))
  1333. gateway_state = state_data.get("gateway_state")
  1334. updated_at = state_data.get("updated_at")
  1335. age_str = ""
  1336. if updated_at:
  1337. try:
  1338. updated_dt = datetime.fromisoformat(updated_at)
  1339. age_seconds = int((datetime.now(UTC) - updated_dt).total_seconds())
  1340. age_str = f" (updated {age_seconds}s ago)"
  1341. except Exception:
  1342. pass
  1343. _probe(5, gateway_state in ("running", "degraded"), f"gateway_state.json state={gateway_state!r}{age_str}")
  1344. except Exception as exc:
  1345. _probe(5, False, f"gateway_state.json present but unreadable: {exc}")
  1346. def _probe_exit_diag(diag_path: Path) -> None:
  1347. if _probe_missing(6, diag_path, "exit-diag log"):
  1348. return
  1349. try:
  1350. with open(diag_path, "rb") as fh:
  1351. fh.seek(0, 2) # last ~4KB; one event is well under 500 bytes
  1352. size = fh.tell()
  1353. fh.seek(max(0, size - 4096))
  1354. tail = fh.read().decode("utf-8", errors="replace").splitlines()
  1355. last_event = next((ln for ln in reversed(tail) if ln.strip()), "")
  1356. if not last_event:
  1357. _probe(6, False, f"exit-diag log empty: {diag_path}")
  1358. return
  1359. try:
  1360. event = json.loads(last_event)
  1361. tag = event.get("tag", "?")
  1362. _probe(6, tag in ("gateway.start",), f"Last lifecycle event: tag={tag} pid={event.get('pid', '?')} ts={event.get('ts', '?')}")
  1363. except Exception:
  1364. _probe(6, False, f"Last lifecycle line not JSON: {last_event[:120]}")
  1365. except Exception as exc:
  1366. _probe(6, False, f"exit-diag log unreadable: {exc}")
  1367. def _print_deep_probes() -> None:
  1368. """Print PASS/FAIL per individual liveness signal, so when the collapsed ✓/✗ summary disagrees
  1369. with reality the user can see exactly which signal is wrong."""
  1370. home = _hermes_home()
  1371. print("\nDeep probes:")
  1372. pid_value = _probe_pid_file(home / "gateway.pid")
  1373. _probe_lock_file(home / "gateway.lock")
  1374. running_pid = _probe_running_pid()
  1375. _probe_pid_exists(running_pid if running_pid is not None else pid_value)
  1376. _probe_state_file(home / "gateway_state.json")
  1377. _probe_exit_diag(home / "logs" / "gateway-exit-diag.log")
  1378. def status(deep: bool = False) -> None:
  1379. """Print a status report for the Windows gateway service."""
  1380. _assert_windows()
  1381. _print_start_attestation_warning() # once: a gateway that died after a previous ✓
  1382. task_name = get_task_name()
  1383. task_installed = is_task_registered()
  1384. startup_installed = is_startup_entry_installed()
  1385. pids = _gateway_pids()
  1386. if task_installed:
  1387. print(f"✓ Scheduled Task registered: {task_name}")
  1388. info = query_task_status()
  1389. for key in ("status", "last run time", "last run result"):
  1390. if key in info:
  1391. print(f" {key.title()}: {info[key]}")
  1392. _print_scheduled_task_drift(task_name)
  1393. elif startup_installed:
  1394. entry = get_startup_entry_path()
  1395. print(f"✓ Windows login item installed: {entry if entry.exists() else _legacy_startup_entry_path()}")
  1396. else:
  1397. print("✗ Gateway service not installed")
  1398. from hermes_cli.gateway_windows_legacy import warn_legacy_launchers
  1399. warn_legacy_launchers()
  1400. print(f"✓ Gateway process running (PID: {', '.join(map(str, pids))})" if pids else "✗ No gateway process detected")
  1401. if deep:
  1402. print()
  1403. print(f" Task name: {task_name}")
  1404. print(f" Task script: {get_task_script_path()}")
  1405. print(f" Startup entry: {get_startup_entry_path()}")
  1406. _print_deep_probes()
  1407. if not task_installed and not startup_installed and not pids:
  1408. print("\nTo install:\n hermes gateway install")
  1409. def start() -> None:
  1410. """Start the gateway using the canonical detached Windows launch path."""
  1411. _assert_windows()
  1412. _print_start_attestation_warning() # once: the LAST start's ✓ turned out to be false
  1413. running_pids = _gateway_pids()
  1414. if running_pids:
  1415. _report_already_running(running_pids)
  1416. return
  1417. if not is_task_registered() and not is_startup_entry_installed():
  1418. # Login persistence is a lasting system change: a bare ``start`` installs it only on an explicit
  1419. # answer — the HERMES_GATEWAY_INSTALL_START_ON_LOGIN override or a real TTY prompt — never on a
  1420. # non-TTY default (#113977). Declining still starts the gateway; the command is ``start``.
  1421. start_on_login = _install_choice_from_env("HERMES_GATEWAY_INSTALL_START_ON_LOGIN")
  1422. if start_on_login is None:
  1423. from hermes_cli.setup import is_interactive_stdin, is_noninteractive, prompt_yes_no
  1424. print("✗ Gateway service is not installed")
  1425. if is_noninteractive() or not _stdout_isatty() or not _stdin_is_interactive(
  1426. isatty=is_interactive_stdin(), console_mode_ok=_stdin_console_mode_ok()
  1427. ):
  1428. start_on_login = False
  1429. else:
  1430. start_on_login = prompt_yes_no(" Install it now so the gateway starts on login?", True)
  1431. if start_on_login:
  1432. # install() starts the gateway itself (start_now) and reports the outcome — including a UAC
  1433. # hand-off to an elevated child — so there is nothing left to spawn or to warn about here.
  1434. install(force=False, start_now=True, start_on_login=True)
  1435. return
  1436. print("ℹ Login auto-start not installed; add it later with: hermes gateway install")
  1437. elif is_task_registered():
  1438. reconcile_scheduled_task(get_task_name()) # like systemd's regenerate-on-stale before a start
  1439. # Manual starts use the same console-less direct spawn as restart() and install --start-now;
  1440. # Scheduled Task / Startup entries are only login persistence.
  1441. _spawn_detached()
  1442. _report_gateway_start("direct spawn")
  1443. def _drain_gateway_pid(pid: int, drain_timeout: float) -> bool:
  1444. """Write the planned-stop marker and wait for the PID to exit. Windows can't deliver POSIX signals
  1445. to an asyncio loop, so the marker is the ONLY way to ask the gateway to drain and persist."""
  1446. if pid <= 0:
  1447. return False
  1448. try:
  1449. from gateway.status import write_planned_stop_marker, _pid_exists
  1450. except ImportError:
  1451. return False
  1452. try:
  1453. write_planned_stop_marker(pid)
  1454. except Exception:
  1455. pass # best-effort; caller escalates to a hard kill
  1456. return poll_until(lambda: not _pid_exists(pid), max(drain_timeout, 1.0), 0.5)
  1457. def _windows_stop_drain_timeout() -> float:
  1458. """Bounded stop grace period: a real graceful-drain window, but the CLI must never wedge."""
  1459. try:
  1460. from hermes_cli.gateway import _get_restart_drain_timeout
  1461. configured = float(_get_restart_drain_timeout() or 30.0)
  1462. except Exception:
  1463. configured = 30.0
  1464. return max(1.0, min(configured, 30.0))
  1465. def _gateway_pid_identities(pids: list[int]) -> dict[int, int | None]:
  1466. """``{pid: start_time}`` fingerprints, captured BEFORE any drain/wait so a later force-kill can
  1467. detect that the PID was recycled meanwhile."""
  1468. try:
  1469. from gateway.status import get_process_start_time
  1470. except ImportError:
  1471. return {pid: None for pid in pids}
  1472. return {pid: get_process_start_time(pid) for pid in pids}
  1473. def _force_terminate_known_gateway_pids(identities: dict[int, int | None]) -> int:
  1474. """Force-kill known gateway PIDs without a broad process sweep. ``identities`` maps each PID to
  1475. the start time observed when it was identified as a gateway (``_gateway_pid_identities``);
  1476. ``terminate_pid`` refuses the kill when the live process no longer matches. Re-reading the start
  1477. time here would compare the process with itself and taskkill whatever now owns the PID."""
  1478. try:
  1479. from gateway.status import _pid_exists, terminate_pid
  1480. except ImportError:
  1481. return 0
  1482. own_pid = os.getpid()
  1483. killed = 0
  1484. for pid, expected_start_time in identities.items():
  1485. if pid <= 0 or pid == own_pid:
  1486. continue
  1487. try:
  1488. if not _pid_exists(pid):
  1489. continue
  1490. terminate_pid(pid, force=True, expected_start_time=expected_start_time)
  1491. killed += 1
  1492. except ProcessLookupError:
  1493. continue
  1494. except PermissionError:
  1495. print(f"⚠ Permission denied to kill PID {pid}")
  1496. except OSError as exc:
  1497. print(f"Failed to kill PID {pid}: {exc}")
  1498. return killed
  1499. def _collect_gateway_stop_pids(primary_pid: int | None = None) -> list[int]:
  1500. """Collect gateway PIDs for the active profile, preserving primary first."""
  1501. pids: list[int] = []
  1502. if primary_pid is not None and primary_pid > 0:
  1503. pids.append(primary_pid)
  1504. try:
  1505. for pid in _gateway_pids():
  1506. if pid > 0 and pid not in pids:
  1507. pids.append(pid)
  1508. except Exception:
  1509. pass
  1510. return pids
  1511. def stop() -> None:
  1512. """Stop the gateway: planned-stop marker first so it can drain in-flight agents and persist
  1513. ``resume_pending`` (Windows asyncio can't receive SIGTERM — the marker is our only IPC), then
  1514. ``schtasks /End``, then a bounded hard-kill of known PIDs."""
  1515. _assert_windows()
  1516. from gateway.status import get_running_pid
  1517. # A user-initiated stop is a planned death: don't later report it as a silent crash.
  1518. _clear_start_attestation()
  1519. pid = get_running_pid()
  1520. stop_pids = _collect_gateway_stop_pids(pid)
  1521. # Fingerprint before the drain: the kill below must refuse a PID recycled during the wait.
  1522. identities = _gateway_pid_identities(stop_pids)
  1523. drained = pid is not None and _drain_gateway_pid(pid, _windows_stop_drain_timeout())
  1524. stopped_any = drained
  1525. if is_task_registered():
  1526. code, _out, err = _exec_schtasks(["/End", "/TN", get_task_name()])
  1527. # schtasks returns nonzero when the task isn't currently running — not an error.
  1528. if code == 0:
  1529. stopped_any = True
  1530. elif "not running" not in (err or "").lower():
  1531. print(f"⚠ schtasks /End returned code {code}: {err.strip()}")
  1532. # No generic process sweep: starts are profile-scoped and stop must stay bounded even if wedged.
  1533. late_pids = [pid for pid in _collect_gateway_stop_pids() if pid not in identities]
  1534. identities.update(_gateway_pid_identities(late_pids))
  1535. killed = _force_terminate_known_gateway_pids(identities)
  1536. if killed:
  1537. stopped_any = True
  1538. print(f"✓ Killed {killed} gateway process(es)")
  1539. if stopped_any:
  1540. print("✓ Gateway stopped (drained cleanly)" if drained else "✓ Gateway stopped")
  1541. else:
  1542. print("✗ No gateway was running")
  1543. def _wait_for_gateway_absent(timeout_s: float = 30.0, interval_s: float = 0.5) -> bool:
  1544. """Block until no gateway is detectable (authoritative ``get_running_pid()`` plus the strict
  1545. ``_gateway_pids()`` scan) or the timeout elapses, so a relaunch never races a draining process."""
  1546. from gateway.status import get_running_pid
  1547. def _absent() -> bool:
  1548. return get_running_pid() is None and not _gateway_pids()
  1549. return poll_until(_absent, max(timeout_s, interval_s), interval_s)
  1550. def _mark_planned_restart() -> None:
  1551. """Stamp the planned-restart marker once the old gateway is gone, before ``start()``.
  1552. Windows has no SIGUSR1, so ``restart()`` is stop + start: the gateway never runs the in-band
  1553. restart path that stamps this marker itself, and the next boot would read a plain stop and skip
  1554. the "Gateway online" home-channel notice.
  1555. Not before ``stop()``: the old process stays alive until its planned-stop watcher notices, and
  1556. for that window a flapping platform can reconnect, replay the notice and unlink the marker the
  1557. new process then never finds.
  1558. """
  1559. from gateway.restart_notice import write_planned_restart_marker
  1560. write_planned_restart_marker()
  1561. def restart() -> None:
  1562. """Stop then start. Waits for the old gateway to be authoritatively gone first; otherwise
  1563. ``start()``'s "already running" guard sees the draining process and no-ops, and nothing
  1564. replaces it when it exits (a silent outage). Fails loudly on either side."""
  1565. _assert_windows()
  1566. stop()
  1567. if not _wait_for_gateway_absent(timeout_s=30.0):
  1568. print("⚠ Gateway still present after stop; forcing termination before restart...")
  1569. _force_terminate_known_gateway_pids(_gateway_pid_identities(_collect_gateway_stop_pids()))
  1570. if not _wait_for_gateway_absent(timeout_s=10.0):
  1571. raise RuntimeError(
  1572. "Gateway process still detected after force kill; refusing to "
  1573. "start a duplicate. Investigate stray PIDs before retrying."
  1574. )
  1575. from hermes_cli.gateway import _wait_for_api_server_port_free # avoid circular init
  1576. _wait_for_api_server_port_free()
  1577. _mark_planned_restart()
  1578. start()
  1579. if not _wait_for_gateway_ready(timeout_s=15.0):
  1580. raise RuntimeError(
  1581. "Gateway restart did not produce a running gateway process. "
  1582. "Check logs/gateway.log and run `hermes gateway status`."
  1583. )