gateway_launchd.py 50 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999
  1. """Gateway launchd (macOS LaunchAgent) backend: plist generation/refresh, bootstrap, start/stop/restart/status.
  2. Extracted from ``hermes_cli/gateway.py``. Bodies read facade helpers through ``_gw()`` (late
  3. binding on ``hermes_cli.gateway``) so the seams tests and callers patch on the facade keep
  4. intercepting the moved code.
  5. """
  6. from __future__ import annotations
  7. from pathlib import Path
  8. import contextlib
  9. import json
  10. import os
  11. import shlex
  12. import subprocess
  13. import sys
  14. import time
  15. from xml.sax.saxutils import escape
  16. from datetime import UTC
  17. from hermes_cli import gateway_service_owner
  18. from agent.deadline import poll_until
  19. def _gw():
  20. from hermes_cli import gateway # late: the facade imports this module
  21. return gateway
  22. def get_launchd_label() -> str:
  23. """Return the launchd service label, scoped per profile."""
  24. suffix = _gw()._profile_suffix()
  25. return f"ai.hermes.gateway-{suffix}" if suffix else "ai.hermes.gateway"
  26. def _probe_launchd_domain_for_label(label: str) -> str:
  27. """Launchd domain managing ``label`` (uncached): ``gui/<uid>`` (Aqua), then ``user/<uid>``
  28. (Background/SSH), else the ``launchctl managername`` heuristic. Sibling profiles may live in
  29. different domains, so never reuse the cached ``_launchd_domain()`` for another label."""
  30. uid = os.getuid() # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
  31. gui_domain, user_domain = f"gui/{uid}", f"user/{uid}"
  32. launchctl_errors = (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError)
  33. for domain in (gui_domain, user_domain):
  34. try:
  35. subprocess.run(["launchctl", "print", f"{domain}/{label}"], check=True, timeout=5, capture_output=True)
  36. return domain
  37. except launchctl_errors:
  38. pass
  39. # Not loaded anywhere: Aqua → gui/<uid>; anything else (Background, loginwindow) → user/<uid>,
  40. # the pre-probing default and the recommended domain on macOS 26+.
  41. try:
  42. result = subprocess.run(["launchctl", "managername"], timeout=5, **_gw()._CAPTURE_TEXT, check=False)
  43. if "Aqua" in (result.stdout or ""):
  44. return gui_domain
  45. except launchctl_errors:
  46. pass
  47. return user_domain
  48. def _launchd_domain() -> str:
  49. """Domain managing the current profile's gateway; cached per process so start/stop/restart agree.
  50. See #40831, #23387.
  51. """
  52. # The cache lives on the facade: tests and callers reset ``hermes_cli.gateway._resolved_launchd_domain``.
  53. gw = _gw()
  54. if gw._resolved_launchd_domain is None:
  55. gw._resolved_launchd_domain = _probe_launchd_domain_for_label(gw.get_launchd_label())
  56. return gw._resolved_launchd_domain
  57. # 125 ("Domain does not support specified action") and 3/113 ("Could not find service") all mean
  58. # the job isn't loaded in the target domain: re-bootstrap the plist and retry.
  59. _LAUNCHD_JOB_UNLOADED_EXIT_CODES = frozenset({3, 113, 125})
  60. # 5 (EIO) / persistent 125 mean either a stale still-registered label (recoverable: bootout +
  61. # bootstrap, which `_launchctl_bootstrap()` tries first) or a domain that genuinely can't manage
  62. # services (macOS 26+). Only when the retry ALSO fails do callers degrade to a detached process.
  63. # launchctl returns 5 ("Input/output error") or a persistent 125 in two very different situations, so exit 5
  64. # is NOT on its own proof the domain is broken: 1. See #42914. 2. Here launchd cannot supervise the gateway
  65. # at all and we degrade to a detached background process (the `nohup hermes gateway run` workaround). See
  66. # #23387.
  67. _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES = frozenset({5, 125})
  68. def _launchd_error_indicates_unloaded(exc: subprocess.CalledProcessError) -> bool:
  69. """True when launchctl failed because the job isn't loaded (retry bootstrap)."""
  70. return exc.returncode in _LAUNCHD_JOB_UNLOADED_EXIT_CODES
  71. def _launchctl_domain_unsupported(returncode: int) -> bool:
  72. """True when launchctl can't manage the domain even after a fresh bootstrap (macOS 26+) — degrade to detached."""
  73. return returncode in _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES
  74. # EIO from `launchctl bootstrap` = label *already* registered (stale load); recoverable, not an unmanageable domain.
  75. _LAUNCHCTL_BOOTSTRAP_EIO = 5
  76. def _launchctl_bootstrap(domain: str, plist_path, label: str, *, timeout: int = 30) -> None:
  77. """Bootstrap a launchd job, recovering from a stale still-registered label (EIO 5). Without the
  78. bootout + retry that case is misread as an unmanageable domain and degrades to detached, silently
  79. losing auto-start and crash-restart."""
  80. bootstrap = ["launchctl", "bootstrap", domain, str(plist_path)]
  81. try:
  82. subprocess.run(bootstrap, check=True, timeout=timeout)
  83. except subprocess.CalledProcessError as exc:
  84. if exc.returncode != _LAUNCHCTL_BOOTSTRAP_EIO:
  85. raise
  86. # Stale registration — bootout the leftover label and bootstrap once more.
  87. # Captured: the bootout is best-effort (a drained job may already be
  88. # unloaded), so its expected 3/113/125 stderr must not leak to the terminal.
  89. subprocess.run(
  90. ["launchctl", "bootout", f"{domain}/{label}"],
  91. check=False, timeout=timeout, **_gw()._CAPTURE_TEXT)
  92. subprocess.run(bootstrap, check=True, timeout=timeout)
  93. def _launchd_reload_log_path() -> Path:
  94. """Path the launchd reload watchdog tails for persistent-orphan detection."""
  95. return _gw().get_hermes_home() / "logs" / "launchd-reload.log"
  96. def _append_launchd_reload_log(message: str) -> None:
  97. """Append a timestamped line to the launchd reload log (best-effort)."""
  98. path = _launchd_reload_log_path()
  99. try:
  100. path.parent.mkdir(parents=True, exist_ok=True)
  101. from datetime import datetime as _dt
  102. stamp = _dt.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %z")
  103. with path.open("a", encoding="utf-8") as fh:
  104. fh.write(f"[{stamp}] {message}\n")
  105. except OSError:
  106. pass
  107. def _launchd_reload_budget() -> float:
  108. """Bootstrap retry window for a plist reload: the failure happens while the old gateway is still
  109. draining (default 180s), so size it to the drain timeout with a 30s floor."""
  110. return max(30.0, _gw()._get_restart_drain_timeout())
  111. def _launchctl_supervised_pid(label: str) -> int | None:
  112. """PID launchd currently runs for ``label``, or None when it runs none. ``launchctl list`` exits 0 for
  113. a mere registered definition (``state = not running`` on macOS 26+), so a PID — not the exit code — is
  114. the answer. Domain-agnostic on purpose: ``launchctl print`` domain probes fail on macOS-26 per-user
  115. domains, which is why the invoking profile verifies through this and not ``_launchd_print_service_pid``."""
  116. try:
  117. result = subprocess.run(["launchctl", "list", label], check=False, timeout=10, **_gw()._CAPTURE_TEXT)
  118. except (subprocess.TimeoutExpired, OSError):
  119. return None
  120. if result.returncode != 0:
  121. return None
  122. return _gw()._parse_launchd_pid_from_list_output(result.stdout)
  123. def _launchctl_label_supervising_process(label: str) -> bool:
  124. """True when launchd knows ``label`` AND runs a process for it."""
  125. return _gw()._launchctl_supervised_pid(label) is not None
  126. def _retry_launchctl_bootstrap_until_registered(
  127. domain: str, plist_path, label: str, *, deadline: float
  128. ) -> bool:
  129. """Retry ``_launchctl_bootstrap`` until the label supervises a process or ``deadline`` passes. Under
  130. load bootstrap can fail even after bootout, during a drain (default 180s) — ~10s is too short."""
  131. attempt = 0
  132. while True:
  133. attempt += 1
  134. try:
  135. _gw()._launchctl_bootstrap(domain, plist_path, label, timeout=30)
  136. if _gw()._launchctl_label_supervising_process(label):
  137. return True
  138. outcome = f"exited 0 but {domain}/{label} has no supervised process (launchctl list)"
  139. except subprocess.CalledProcessError as exc:
  140. outcome = f"failed (rc={exc.returncode}) for {domain}/{label}"
  141. except subprocess.TimeoutExpired:
  142. outcome = f"timed out for {domain}/{label}"
  143. _gw()._append_launchd_reload_log(f"bootstrap attempt {attempt} {outcome} — retrying")
  144. if time.monotonic() >= deadline:
  145. return False
  146. time.sleep(2)
  147. # launchd-unsupported marker: written when the domain can't be managed (exit 5/125, macOS 26+) so
  148. # `launchd_status()` can explain missing supervision; cleared on successful bootstrap/kickstart.
  149. def _launchd_unsupported_marker_path() -> Path:
  150. return _gw().get_hermes_home() / ".gateway-launchd-unsupported"
  151. def _write_launchd_unsupported_marker() -> None:
  152. """Persist that launchd cannot supervise the gateway on this host."""
  153. from datetime import datetime, timezone
  154. payload = {
  155. "written_at": datetime.now(UTC).isoformat(),
  156. "reason": "launchd domain unsupported (exit 5/125)",
  157. }
  158. with contextlib.suppress(OSError):
  159. _launchd_unsupported_marker_path().write_text(json.dumps(payload), encoding="utf-8")
  160. def _clear_launchd_unsupported_marker() -> None:
  161. """Clear the unsupported marker when launchd bootstrap succeeds."""
  162. with contextlib.suppress(OSError):
  163. _launchd_unsupported_marker_path().unlink(missing_ok=True)
  164. def _launchd_unsupported_marker_exists() -> bool:
  165. return _launchd_unsupported_marker_path().exists()
  166. def _gateway_run_command() -> list[str]:
  167. from hermes_cli._launchers import runtime_command
  168. from hermes_constants import get_default_hermes_root
  169. # Every caller spawns an UNSUPERVISED child, which follows the sticky active_profile unless a
  170. # selector pins it (main._apply_profile_override): the root's replacement booted that profile.
  171. selector = shlex.split(_gw()._profile_arg())
  172. if not selector and _gw().get_hermes_home().resolve() == get_default_hermes_root().resolve():
  173. selector = ["--profile", "default"]
  174. return runtime_command(_gw().PROJECT_ROOT, [*selector, "gateway", "run", "--replace"],
  175. python=_gw().get_python_path())
  176. def launchd_program_arguments(command: list[str], stdout_log: Path, stderr_log: Path) -> list[str]:
  177. """launchd ``ProgramArguments`` that run ``command`` with a Local Network identity macOS accepts (#71206).
  178. macOS Local Network Privacy attributes a socket to the process launchd spawned for the job. A bare
  179. venv Python has no application ID and is not platform-entitled, so every LAN connect from the
  180. launchd gateway dies with ``EHOSTUNREACH`` while the same code works from Terminal (whose grant it
  181. inherits). An ad-hoc-signed helper .app does not help: nehelper never prompts for it and denies
  182. (#57812 dead-end table, re-verified live on macOS 26.3). ``/usr/bin/osascript`` spawning the child
  183. makes it osascript-responsible — an Apple platform binary — so the child is exempt; ``/bin/sh -c
  184. exec …`` and ``/usr/bin/time`` wrappers are NOT (the launchd job identity is the non-entitled first
  185. executable).
  186. Standard Additions' ``do shell script`` polls WindowServer for a user-cancel event while it waits.
  187. That is appropriate for a short interactive script but, for the gateway's process lifetime, burns CPU
  188. and keeps a WindowServer event connection busy (external-display wake stalls ~10s on macOS 27, #123595).
  189. JXA calling libc ``system()`` waits in the kernel instead while retaining osascript as the responsible
  190. process. The shell's ``exec`` keeps the gateway in the launchd job's process group, so ``launchctl
  191. bootout`` / ``kickstart -k`` still deliver SIGTERM to it. stdout/stderr are appended inside the shell
  192. command because ``system()`` otherwise inherits osascript's plist log handles. The encoded wait status
  193. is translated back to a process exit code so KeepAlive's ``SuccessfulExit`` semantics are preserved.
  194. """
  195. shell = f"exec {shlex.join(command)} >> {shlex.quote(str(stdout_log))} 2>> {shlex.quote(str(stderr_log))}"
  196. javascript = (
  197. 'ObjC.import("stdlib"); '
  198. f"const status=$.system({json.dumps(shell)}); "
  199. "const signal=status & 127; "
  200. "$.exit(status === -1 ? 1 : signal === 0 ? (status >> 8) & 255 : 128 + signal);"
  201. )
  202. return ["/usr/bin/osascript", "-l", "JavaScript", "-e", javascript]
  203. def _timestamped_stderr_gateway_command(error_log: Path, *, external_supervisor: bool = False) -> list[str]:
  204. """Wrap gateway run so raw stderr lines are timestamped before file write. ``external_supervisor``
  205. (launchd ProgramArguments only) adds ``--external-supervisor`` so ``hermes update`` hands back to
  206. launchd, and drops ``--replace``: KeepAlive respawns would re-arm takeover, so two profiles sharing
  207. a token would kill each other forever.
  208. ``external_supervisor=True`` is for launchd ProgramArguments only: the inner ``gateway run`` must carry
  209. ``--external-supervisor`` so ``hermes update`` sees the flag on the live grandchild argv and hands the
  210. process back to launchd instead of starting a detached watcher (#86893 / #87005). The detached nohup
  211. fallback stays unmarked.
  212. Supervised starts also drop ``--replace`` (issue #79048): a launchd service is respawned by KeepAlive,
  213. so takeover authority would be re-armed on every respawn — two profiles legitimately sharing one
  214. platform token would each terminate the sibling, and launchd would revive the victim forever. Bounded
  215. replacement is the lifecycle commands' job (``launchctl kickstart -k``, drain in ``launchd_restart()``,
  216. bootout+bootstrap in install/refresh), which run before supervision resumes. Mirrors
  217. ``generate_systemd_unit``, whose ExecStart also runs ``gateway run`` without ``--replace``.
  218. """
  219. from hermes_cli._launchers import installation_command, runtime_command
  220. inner = _gw()._gateway_run_command()
  221. if external_supervisor:
  222. inner = installation_command(_gw().PROJECT_ROOT, [*shlex.split(_gw()._profile_arg()), "gateway", "run"],
  223. python=_gw().get_python_path())
  224. inner = [part for part in inner if part != "--replace"]
  225. if "--external-supervisor" not in inner:
  226. inner.append("--external-supervisor")
  227. command = installation_command if external_supervisor else runtime_command
  228. return command(_gw().PROJECT_ROOT, ["--error-log", str(error_log), "--", *inner],
  229. module="hermes_cli.stderr_timestamp", python=_gw().get_python_path())
  230. def _spawn_detached_gateway() -> bool:
  231. """Launch the gateway detached (launchd fallback for macOS 26+). CLI-managed nohup equivalent:
  232. stdout → gateway.log, timestamped stderr → gateway.error.log, PID via gateway.pid so stop/status work.
  233. Used when launchctl can no longer bootstrap/kickstart the gateway on macOS 26+ (issue #23387). Mirrors
  234. the `nohup hermes gateway run --replace` workaround but keeps it CLI-managed: stdout goes to
  235. gateway.log, stderr is timestamped into gateway.error.log, and the PID is tracked via the gateway.pid
  236. file that `run_gateway` writes, so stop/status/restart keep working.
  237. """
  238. from hermes_cli._subprocess_compat import windows_detach_popen_kwargs
  239. from hermes_constants import get_hermes_home
  240. from tools.environments.local import served_profile_child_env
  241. log_dir = _gw().get_hermes_home() / "logs"
  242. log_dir.mkdir(parents=True, exist_ok=True)
  243. child_env = served_profile_child_env(
  244. target_home=get_hermes_home(), inherit_credentials=True,
  245. )
  246. child_env.pop("_HERMES_GATEWAY", None)
  247. try:
  248. with open(log_dir / "gateway.log", "ab") as out:
  249. subprocess.Popen(
  250. _timestamped_stderr_gateway_command(log_dir / "gateway.error.log"),
  251. stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.DEVNULL,
  252. env=child_env,
  253. # detached-spawn: ok — foreground-less gateway launch, a service meant to outlive the CLI
  254. **windows_detach_popen_kwargs(),
  255. )
  256. except OSError:
  257. return False
  258. return True
  259. def _launchd_fallback_to_detached(reason: str, *, exit_on_failure: bool = True) -> bool:
  260. """Start the gateway detached when launchd can't manage it; on failure print the manual workaround
  261. and (by default) exit 1."""
  262. from hermes_constants import display_hermes_home as _dhh
  263. _gw()._write_launchd_unsupported_marker()
  264. print(f"⚠ launchd cannot manage the gateway on this macOS version ({reason}).")
  265. if _gw()._spawn_detached_gateway():
  266. print("✓ Started gateway as a background process instead")
  267. print(" It will NOT auto-start at login or auto-restart on crash.")
  268. print(f" Logs: {_dhh()}/logs/gateway.log")
  269. print(" Stop it with: hermes gateway stop")
  270. return True
  271. _gw().print_error("Failed to start the gateway as a background process.")
  272. print(f" Try manually: nohup hermes gateway run --replace > {_dhh()}/logs/gateway.log 2>&1 &")
  273. if exit_on_failure:
  274. sys.exit(1)
  275. return False
  276. def _launchd_degrade_or_raise(exc: subprocess.CalledProcessError, what: str) -> None:
  277. """Shared launchctl failure policy: domain unmanageable (5/125) → detached fallback; else re-raise.
  278. A 5/125 exit is evidence about the *domain* only when launchd is not already supervising this job.
  279. EIO (5) is ``launchctl bootstrap``'s answer for a label that is already loaded, so the ordinary
  280. "reinstall/restart over the live gateway" case lands here with the service up and supervised.
  281. Degrading there is not a graceful fallback: it writes the permanent launchd-unsupported marker and
  282. starts a detached gateway *beside* the supervised one, and the marker makes
  283. :func:`wait_for_launchd_gateway_supervision` answer True unconditionally — so no later
  284. install/update can tell that nothing ties the gateway to launchd any more. A live supervised PID is
  285. direct evidence this macOS does manage the job, so surface the failure instead of branding the host.
  286. """
  287. if not _launchctl_domain_unsupported(exc.returncode):
  288. raise exc
  289. label = _gw().get_launchd_label()
  290. if _gw()._launchctl_label_supervising_process(label):
  291. print(f"⚠ {what} failed (exit {exc.returncode}), but launchd still supervises {label}")
  292. print(" Not switching to the detached fallback — this host manages the job.")
  293. print(" Apply the definition with: hermes gateway stop && hermes gateway install --force")
  294. raise exc
  295. _launchd_fallback_to_detached(f"{what} exit {exc.returncode}")
  296. def generate_launchd_plist() -> str:
  297. from html import escape
  298. # Stable cwd anchor — never the volatile source checkout (same rot risk as systemd's WorkingDirectory).
  299. working_dir = _gw()._stable_service_working_dir()
  300. hermes_home = str(_gw().get_hermes_home().resolve())
  301. log_dir = _gw().get_hermes_home() / "logs"
  302. log_dir.mkdir(parents=True, exist_ok=True)
  303. label = _gw().get_launchd_label()
  304. # launchd's default PATH misses Homebrew, nvm, cargo…; prepend venv/bin + node dirs (as in the
  305. # systemd unit) so node stays resolvable even if the shell PATH changes, then the shell PATH.
  306. priority_dirs = _gw()._build_service_path_dirs()
  307. _gw()._append_node_dir_for_service(priority_dirs)
  308. sane_path = ":".join(dict.fromkeys(priority_dirs + [p for p in os.environ.get("PATH", "").split(":") if p]))
  309. # ProgramArguments (incl. --profile); the stderr wrapper keeps launchd restart semantics while timestamping
  310. # stderr; the osascript wrapper gives the job a Local Network identity (see launchd_program_arguments).
  311. stdout_log, stderr_log = log_dir / "gateway.log", log_dir / "gateway.error.log"
  312. command = _timestamped_stderr_gateway_command(stderr_log, external_supervisor=True)
  313. prog_args_xml = "\n ".join(
  314. f"<string>{escape(part)}</string>" for part in launchd_program_arguments(command, stdout_log, stderr_log)
  315. )
  316. # Persist the configured RLIMIT_NOFILE floor: launchd defaults to soft 256, and every plist
  317. # rewrite would otherwise strip a manual limit and reintroduce EMFILE crashes.
  318. nofile_block = ""
  319. try:
  320. from hermes_cli.resource_limits import configured_nofile_soft_limit
  321. nofile_target = configured_nofile_soft_limit()
  322. except Exception:
  323. nofile_target = None
  324. if nofile_target:
  325. nofile_block = f"""
  326. <key>SoftResourceLimits</key>
  327. <dict>
  328. <key>NumberOfFiles</key>
  329. <integer>{nofile_target}</integer>
  330. </dict>
  331. """
  332. return f"""<?xml version="1.0" encoding="UTF-8"?>
  333. <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
  334. <plist version="1.0">
  335. <dict>
  336. <key>Label</key>
  337. <string>{label}</string>
  338. <key>ProgramArguments</key>
  339. <array>
  340. {prog_args_xml}
  341. </array>
  342. <key>WorkingDirectory</key>
  343. <string>{working_dir}</string>
  344. <key>EnvironmentVariables</key>
  345. <dict>
  346. <key>PATH</key>
  347. <string>{sane_path}</string>
  348. <key>HERMES_HOME</key>
  349. <string>{hermes_home}</string>
  350. <key>HERMES_SUPERVISED_CHILD</key>
  351. <string>1</string>
  352. </dict>
  353. <key>LimitLoadToSessionType</key>
  354. <array>
  355. <string>Aqua</string>
  356. <string>Background</string>
  357. </array>
  358. <key>RunAtLoad</key>
  359. <true/>
  360. <key>KeepAlive</key>
  361. <dict>
  362. <key>SuccessfulExit</key>
  363. <false/>
  364. </dict>
  365. <!-- SuccessfulExit=false parks a clean stop (exit 0), including gateway
  366. EX_CONFIG 78 after stderr_timestamp maps it to 0 — launchd cannot
  367. honor RestartPreventExitStatus, and KeepAlive=true respawned token
  368. collisions forever (#89477). Exit 75 and crashes still relaunch.
  369. ThrottleInterval raises launchd's default 10s minimum respawn interval
  370. to 30s so a crash-looping gateway can't hammer launchd into a rapid
  371. respawn storm; ExitTimeOut is the graceful-drain headroom before
  372. launchd escalates from SIGTERM to SIGKILL on stop. The per-user
  373. (gui) launchd domain clamps it to 60s regardless of what is written
  374. here, so 60 is the most a LaunchAgent can get; the gateway reads
  375. the live value at boot and fits its signal-driven drain inside it
  376. (gateway.restart.read_launchd_exit_timeout_s). -->
  377. <key>ThrottleInterval</key>
  378. <integer>30</integer>
  379. <key>ExitTimeOut</key>
  380. <integer>60</integer>
  381. {nofile_block}
  382. <key>StandardOutPath</key>
  383. <string>{stdout_log}</string>
  384. <key>StandardErrorPath</key>
  385. <string>{stderr_log}</string>
  386. </dict>
  387. </plist>
  388. """
  389. def launchd_plist_is_current() -> bool:
  390. """Check if the installed launchd plist matches the currently generated one."""
  391. plist_path = _gw().get_launchd_plist_path()
  392. if not plist_path.exists():
  393. return False
  394. installed = plist_path.read_text(encoding="utf-8-sig")
  395. norm = _gw()._normalize_launchd_plist_for_comparison
  396. return norm(installed) == norm(_gw().generate_launchd_plist())
  397. def _spawn_deferred_launchd_reload(
  398. *, domain: str, label: str, target: str, plist_path: Path, gateway_pid: int
  399. ) -> bool:
  400. """Hand the bootout/bootstrap cycle to a transient ``launchctl submit`` job; True if spawned. The
  401. helper waits for the OLD gateway to exit (bootstrap during drain fails EIO), then retries bootstrap
  402. until ``launchctl list`` shows a positive PID or the drain budget elapses."""
  403. reload_log_path = _launchd_reload_log_path()
  404. with contextlib.suppress(OSError):
  405. reload_log_path.parent.mkdir(parents=True, exist_ok=True)
  406. # Durable pre-bootout marker: distinguishes "helper never started" from "helper ran but failed".
  407. _gw()._append_launchd_reload_log(f"Launchd reload helper started for {target}")
  408. _reload_budget = int(_launchd_reload_budget())
  409. q_target, q_label, q_log = shlex.quote(target), shlex.quote(label), shlex.quote(str(reload_log_path))
  410. stamp = "$(date '+%Y-%m-%d %H:%M:%S %z')"
  411. # Require a POSITIVE PID: `launchctl list` also exits 0 for a registered-but-not-running
  412. # definition, and a crashed job reports `"PID" = -1` (mirrors _parse_launchd_pid_from_list_output).
  413. listed = f"launchctl list {q_label} 2>/dev/null | grep -qE '\\\"PID\\\" = [0-9]+;'"
  414. # Unique per reload so concurrent/repeated reloads never collide.
  415. submit_label = f"{label}.reload.{os.getpid()}.{int(time.time())}"
  416. reload_script = (
  417. f"sleep 2; "
  418. f"launchctl bootout {q_target} 2>/dev/null; "
  419. # Wait for the OLD gateway to exit: bootout only SIGTERMs and every bootstrap during the drain fails EIO.
  420. f"_wait_deadline=$(($(date +%s) + {_reload_budget})); "
  421. f"while kill -0 {gateway_pid} 2>/dev/null; do if [ $(date +%s) -ge $_wait_deadline ]; then "
  422. f" echo \"[{stamp}] old gateway pid {gateway_pid} still alive after {_reload_budget}s drain wait — bootstrapping anyway\" >> {q_log}; "
  423. f" break; fi; sleep 1; done; "
  424. # Let launchd finish unregistering the label after the process exits.
  425. f"sleep 1; _deadline=$(($(date +%s) + {_reload_budget})); while :; do "
  426. f" launchctl bootstrap {shlex.quote(domain)} {shlex.quote(str(plist_path))} 2>/dev/null; "
  427. f" if {listed}; then break; fi; "
  428. f" echo \"[{stamp}] bootstrap not yet registered for {q_target} — retrying\" >> {q_log}; "
  429. f" if [ $(date +%s) -ge $_deadline ]; then break; fi; sleep 2; done; "
  430. f"if ! {listed}; then "
  431. f" echo \"[{stamp}] FAILED launchd reload for {q_target} — service NOT registered after {_reload_budget}s of retries\" >> {q_log}; "
  432. f"fi; "
  433. # Submitted jobs stay registered after the script exits; removing our own label ends the one-shot job.
  434. f"launchctl remove {shlex.quote(submit_label)} 2>/dev/null"
  435. )
  436. try:
  437. # `launchctl submit` rather than setsid: setsid does NOT leave the launchd coalition that bootout kills.
  438. # Spawn the reload helper via `launchctl submit` (a transient launchd one-shot job) instead of
  439. # `start_new_session=True`. `start_new_session=True` only calls setsid(2), which creates a new POSIX
  440. # session but does NOT move the child outside the launchd job's process coalition. When `launchctl
  441. # bootout` fires on the gateway label, launchd terminates ALL processes in that coalition —
  442. # including a setsid-detached child (#69098). `launchctl submit` creates a wholly independent
  443. # transient launchd job that launchd manages separately from the gateway, so bootout of the gateway
  444. # job cannot reach the helper.
  445. subprocess.Popen(
  446. [
  447. "launchctl", "submit", "-l", submit_label, "-o", str(reload_log_path), "-e", str(reload_log_path),
  448. "--", "/bin/bash", "-c", reload_script,
  449. ],
  450. stdout=subprocess.DEVNULL,
  451. stderr=subprocess.DEVNULL,
  452. )
  453. except Exception as e:
  454. # Fall through to in-process bootout/bootstrap: risky in the coalition, but better than a never-reloaded plist.
  455. _gw().logger.warning("Deferred launchd reload could not be spawned: %s", e)
  456. _gw()._append_launchd_reload_log(
  457. f"FAILED to spawn launchd reload helper for {target}: {e} — falling back to in-process bootout/bootstrap"
  458. )
  459. return False
  460. return True
  461. def _bootout_drain_and_rebootstrap(domain, plist_path, label, old_pid, budget: float) -> bool:
  462. """Boot the job out, wait out the old gateway's drain, then bootstrap until launchd registers it.
  463. Bootstrap can fail silently under load during a drain, and KeepAlive can't revive an
  464. unregistered job, so the retry runs to a deadline. Waiting for *old_pid* first keeps the
  465. budget from burning on guaranteed EIO ("already loaded"). The bootout is best effort (the job
  466. may already be unloaded) and captured to keep expected noise off the terminal.
  467. """
  468. target = f"{domain}/{label}"
  469. subprocess.run(["launchctl", "bootout", target], check=False, timeout=90, **_gw()._CAPTURE_TEXT)
  470. if old_pid is not None and not _gw()._wait_for_pid_exit(old_pid, budget):
  471. _gw()._append_launchd_reload_log(
  472. f"old gateway pid {old_pid} still alive after "
  473. f"{int(budget)}s drain wait — bootstrapping {target} anyway"
  474. )
  475. return _gw()._retry_launchctl_bootstrap_until_registered(
  476. domain, plist_path, label, deadline=time.monotonic() + budget)
  477. def refresh_launchd_plist_if_needed() -> bool:
  478. """Rewrite the installed plist when the generated one differs, then bootout/bootstrap so launchd
  479. re-reads it immediately."""
  480. plist_path = _gw().get_launchd_plist_path()
  481. if not plist_path.exists() or _gw().launchd_plist_is_current():
  482. return False
  483. # Runs on every gateway start/restart: a plist pinning another home is another install's gateway.
  484. from hermes_cli.gateway_service_owner import definition_belongs_to_home
  485. if not definition_belongs_to_home(plist_path, _gw().get_hermes_home(), "rewrite"):
  486. return False
  487. new_plist = _gw().generate_launchd_plist()
  488. if gateway_service_owner.refuse_temp_home_service_write(new_plist, "launchd plist"):
  489. return False
  490. _gw()._prepare_service_launcher()
  491. plist_path.write_text(new_plist, encoding="utf-8")
  492. label = _gw().get_launchd_label()
  493. domain = _gw()._launchd_domain()
  494. target = f"{domain}/{label}"
  495. # Inside the gateway's launchd process tree (agent self-update) a direct bootout kills THIS CLI
  496. # before bootstrap runs, leaving the job unloaded with no KeepAlive.
  497. try:
  498. from gateway.status import get_running_pid
  499. gateway_pid = get_running_pid()
  500. except Exception:
  501. gateway_pid = None
  502. # POSIX ancestry is NOT a reliable "bootout will kill us" test (coalition membership survives
  503. # reparenting), so always prefer the detached helper; in-process is only the spawn-failure fallback.
  504. if (
  505. gateway_pid is not None
  506. and hasattr(os, "setsid") # POSIX-only; launchd is macOS so always true here
  507. ) and _spawn_deferred_launchd_reload(
  508. domain=domain, label=label, target=target, plist_path=plist_path, gateway_pid=gateway_pid
  509. ):
  510. print(
  511. "↻ Updated gateway launchd service definition; reload deferred to "
  512. "a transient launchd job (survives the bootout of this process)"
  513. )
  514. return True
  515. # Bootout/bootstrap so launchd reads the new definition.
  516. _reload_budget = _launchd_reload_budget()
  517. if not _bootout_drain_and_rebootstrap(domain, plist_path, label, gateway_pid, _reload_budget):
  518. _gw()._append_launchd_reload_log(
  519. f"FAILED launchd reload of {target} — service NOT registered after "
  520. f"retrying for {int(_reload_budget)}s (in-process fallback path)"
  521. )
  522. _gw().logger.error(
  523. "launchd reload of %s failed — service not registered after %ds of retries; see %s",
  524. target, int(_reload_budget), _launchd_reload_log_path(),
  525. )
  526. return False
  527. print("↻ Updated gateway launchd service definition to match the current Hermes install")
  528. return True
  529. def launchd_install(force: bool = False, *, start_now: bool = True, force_unit_path: bool = False):
  530. plist_path = _gw().get_launchd_plist_path()
  531. label = _gw().get_launchd_label()
  532. # A plist pinning another home is another install's gateway, --force included; --force-unit-path repoints.
  533. from hermes_cli.gateway_service_owner import definition_belongs_to_home
  534. if not force_unit_path and not definition_belongs_to_home(plist_path, _gw().get_hermes_home(), "overwrite"):
  535. sys.exit(1)
  536. # Loading the plist starts the gateway (RunAtLoad), so a no-start install writes it without
  537. # loading it. A gateway that launchd already runs is still reloaded; this install did not start it.
  538. load = start_now or _gw()._launchctl_label_supervising_process(label)
  539. # A forced reinstall over a live job must remember the incumbent before bootstrap starts.
  540. # _launchctl_bootstrap() handles EIO by booting the stale label out and retrying once, but
  541. # that immediate retry races the old gateway's drain (#120629). The install path can wait
  542. # because it runs outside the supervised gateway process.
  543. force_old_pid = _gw()._launchctl_supervised_pid(label) if force and load else None
  544. if plist_path.exists() and not (force or force_unit_path):
  545. if _gw().launchd_plist_is_current():
  546. print(f"Service already installed at: {plist_path}")
  547. print("Use --force to reinstall")
  548. return
  549. if load:
  550. print(f"↻ Repairing outdated launchd service at: {plist_path}")
  551. if _gw().refresh_launchd_plist_if_needed():
  552. print("✓ Service definition updated")
  553. else:
  554. # The plist was rewritten but launchd never registered it (or the write was refused):
  555. # a success line here would hide an unloaded service with no KeepAlive.
  556. from hermes_constants import display_hermes_home
  557. print(
  558. "⚠ Service definition could not be reloaded with launchd. "
  559. "Run 'hermes gateway install --force' or check "
  560. f"{display_hermes_home()}/logs/launchd-reload.log for details."
  561. )
  562. return
  563. plist_path.parent.mkdir(parents=True, exist_ok=True)
  564. new_plist = _gw().generate_launchd_plist()
  565. if gateway_service_owner.refuse_temp_home_service_write(new_plist, "launchd plist"):
  566. return
  567. print(f"Installing launchd service to: {plist_path}")
  568. _gw()._prepare_service_launcher()
  569. plist_path.write_text(new_plist, encoding="utf-8")
  570. if not load:
  571. # A job left loaded but idle (a parked clean exit) keeps its old definition, and that is
  572. # what `hermes gateway start` would kickstart instead of loading this plist.
  573. subprocess.run(
  574. ["launchctl", "bootout", f"{_gw()._launchd_domain()}/{label}"],
  575. check=False, timeout=90, **_gw()._CAPTURE_TEXT)
  576. print()
  577. print("✓ Service installed, not started (launchd starts it at your next login)")
  578. print()
  579. print("Next steps:")
  580. print(" hermes gateway start # Start it now")
  581. print(" hermes gateway status # Check status")
  582. return
  583. if force_old_pid is not None:
  584. domain = _gw()._launchd_domain()
  585. target = f"{domain}/{label}"
  586. budget = _gw()._launchd_reload_budget()
  587. if not _bootout_drain_and_rebootstrap(domain, plist_path, label, force_old_pid, budget):
  588. _gw()._append_launchd_reload_log(
  589. f"FAILED forced launchd install of {target} — service NOT registered after "
  590. f"retrying for {int(budget)}s"
  591. )
  592. _gw().print_error(
  593. f"launchctl bootstrap did not register {target} within {int(budget)}s; "
  594. "the gateway service is NOT loaded."
  595. )
  596. sys.exit(1)
  597. else:
  598. try:
  599. _gw()._launchctl_bootstrap(_gw()._launchd_domain(), plist_path, label, timeout=30)
  600. except subprocess.CalledProcessError as e:
  601. _gw()._launchd_degrade_or_raise(e, "launchctl bootstrap")
  602. return
  603. print()
  604. print("✓ Service installed and loaded!")
  605. _gw()._clear_launchd_unsupported_marker()
  606. print()
  607. print("Next steps:")
  608. print(" hermes gateway status # Check status")
  609. from hermes_constants import display_hermes_home as _dhh
  610. print(f" tail -f {_dhh()}/logs/gateway.log # View logs")
  611. def launchd_uninstall():
  612. gateway_service_owner.refuse_foreign_launchd_plist("uninstall")
  613. plist_path = _gw().get_launchd_plist_path()
  614. # Captured: uninstalling an already-unloaded job is fine — don't print Boot-out failed: 3.
  615. subprocess.run(
  616. ["launchctl", "bootout", f"{_launchd_domain()}/{get_launchd_label()}"],
  617. check=False, timeout=90, **_gw()._CAPTURE_TEXT)
  618. if plist_path.exists():
  619. plist_path.unlink()
  620. print(f"✓ Removed {plist_path}")
  621. print("✓ Service uninstalled")
  622. def launchd_start():
  623. gateway_service_owner.refuse_foreign_launchd_plist("start")
  624. plist_path = _gw().get_launchd_plist_path()
  625. label = _gw().get_launchd_label()
  626. # Self-heal if the plist is missing entirely (e.g., manual cleanup, failed upgrade)
  627. if not plist_path.exists():
  628. new_plist = _gw().generate_launchd_plist()
  629. if gateway_service_owner.refuse_temp_home_service_write(new_plist, "launchd plist"):
  630. sys.exit(1)
  631. print("↻ launchd plist missing; regenerating service definition")
  632. plist_path.parent.mkdir(parents=True, exist_ok=True)
  633. _gw()._prepare_service_launcher()
  634. plist_path.write_text(new_plist, encoding="utf-8")
  635. if _launchd_bootstrap_and_kickstart(plist_path, label):
  636. _launchd_ok("✓ Service started")
  637. return
  638. _gw().refresh_launchd_plist_if_needed()
  639. try:
  640. _launchctl_kickstart_current(label)
  641. except subprocess.CalledProcessError as e:
  642. if not _launchd_error_indicates_unloaded(e):
  643. raise
  644. # Job not loaded in this domain — re-bootstrap the plist and retry.
  645. print("↻ launchd job was unloaded; reloading service definition")
  646. if not _launchd_bootstrap_and_kickstart(plist_path, label):
  647. return
  648. _launchd_ok("✓ Service started")
  649. def _launchctl_kickstart_current(label: str) -> None:
  650. subprocess.run(["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], check=True, timeout=30)
  651. def _launchd_bootstrap_and_kickstart(plist_path: Path, label: str) -> bool:
  652. """Bootstrap then kickstart; False after degrading to detached (domain unsupported). Other errors propagate."""
  653. try:
  654. _gw()._launchctl_bootstrap(_gw()._launchd_domain(), plist_path, label, timeout=30)
  655. _launchctl_kickstart_current(label)
  656. except subprocess.CalledProcessError as e:
  657. _gw()._launchd_degrade_or_raise(e, "launchctl")
  658. return False
  659. return True
  660. def _launchd_ok(message: str) -> None:
  661. """Print a launchd success line and clear the unsupported marker (an OS fix recovers automatically)."""
  662. print(message)
  663. _gw()._clear_launchd_unsupported_marker()
  664. def launchd_stop():
  665. gateway_service_owner.refuse_foreign_launchd_plist("stop")
  666. target = f"{_launchd_domain()}/{get_launchd_label()}"
  667. _gw()._mark_planned_stop()
  668. # bootout unloads the definition so KeepAlive doesn't respawn; `hermes gateway start` re-bootstraps.
  669. try:
  670. # Captured: an already-unloaded job (3/113/125) is handled below, so launchctl's own
  671. # "Boot-out failed: 3" must not print around the ✓ line; e.stderr stays on the raised error.
  672. subprocess.run(["launchctl", "bootout", target], check=True, timeout=90, **_gw()._CAPTURE_TEXT)
  673. except subprocess.CalledProcessError as e:
  674. # Job already unloaded (3/113/125), or the domain can't be managed at all (5/125, macOS 26+
  675. # detached-fallback process, issue #23387) — in both cases just fall through to the PID-based kill
  676. # below.
  677. if not (_launchd_error_indicates_unloaded(e) or _launchctl_domain_unsupported(e.returncode)):
  678. raise
  679. _gw()._wait_for_gateway_exit(timeout=10.0, force_after=5.0)
  680. print("✓ Service stopped")
  681. def _launchd_kickstart(label: str, domain: str) -> None:
  682. """``launchctl kickstart -k domain/label``; raises so callers own per-label failure accounting."""
  683. subprocess.run(["launchctl", "kickstart", "-k", f"{domain}/{label}"], check=True, timeout=90, **_gw()._CAPTURE_TEXT)
  684. def _wait_for_launchd_service_pid(
  685. label: str, old_pid: int | None, timeout: float = 10.0, *, domain: str
  686. ) -> bool:
  687. """Poll ``domain/label`` (0.5s) until it runs on a fresh PID or ``timeout`` passes — KeepAlive respawn
  688. isn't instantaneous. launchctl ``TimeoutExpired`` propagates; callers own failure accounting."""
  689. def _fresh_pid() -> bool:
  690. pid = _gw()._launchd_print_service_pid(domain, label)[1]
  691. return pid is not None and pid > 0 and pid != old_pid
  692. return poll_until(_fresh_pid, max(timeout, 0.5), 0.5)
  693. def launchd_restart():
  694. gateway_service_owner.refuse_foreign_launchd_plist("restart")
  695. label = _gw().get_launchd_label()
  696. domain = _gw()._launchd_domain()
  697. target = f"{domain}/{label}"
  698. # A kickstart re-runs whatever definition launchd already holds. After an
  699. # update that definition was generated by the OLD checkout, so restarting
  700. # without refreshing first faithfully revives a stale service (and a wedged
  701. # stale job is what hangs kickstart into its 90s timeout). Refresh rewrites
  702. # the plist and bootout/bootstraps when it changed; no-op when current.
  703. # When the refresh could not re-register the job (launchd wedged), the
  704. # kickstart below would hang on the same wall — go straight to the
  705. # bootout/bootstrap-retry path, which is bounded and reports its own
  706. # failure instead of stalling the update for 90s.
  707. refresh_ok = _gw().refresh_launchd_plist_if_needed()
  708. from gateway.status import get_running_pid
  709. try:
  710. pid = get_running_pid()
  711. if pid is not None and _gw()._request_gateway_self_restart(pid):
  712. _launchd_ok("✓ Service restart requested")
  713. return
  714. if pid is not None and _gw().probe_gateway_loop_liveness(pid) == _gw().GATEWAY_LOOP_WEDGED:
  715. # Event loop provably dead: it can't process a graceful shutdown, so a full drain wait
  716. # only stalls the restart (and `hermes update`). Bounded SIGTERM → SIGKILL, ~10s.
  717. print(f"⚠ Gateway PID {pid} event loop is unresponsive — " "skipping drain and forcing a bounded stop...")
  718. _gw()._escalate_wedged_gateway(pid)
  719. pid = None
  720. if pid is not None:
  721. # Graceful in-band restart via SIGUSR1 (mirrors systemd); the budget covers both the idle wait
  722. # and the drain. A bare SIGTERM would lose the resume_pending handoff. Announce BEFORE waiting:
  723. # surfaces with no other feedback (desktop updater) read silence as "update stuck".
  724. wait_budget = _gw()._get_restart_exit_wait_budget()
  725. print(f"→ Stopping gateway (PID {pid}) — draining in-flight runs (up to {wait_budget:.0f}s)...")
  726. from hermes_cli.update_cmd_drain_report import drain_progress_reporter
  727. if _gw()._graceful_restart_via_sigusr1(pid, wait_budget, on_progress=drain_progress_reporter(budget_s=wait_budget)):
  728. # KeepAlive revives a planned exit, so do NOT kickstart (-k would kill the replacement) —
  729. # but a clean exit doesn't prove supervision, so verify a replacement PID appears first.
  730. if _gw()._wait_for_launchd_service_pid(label, pid, timeout=15.0, domain=domain):
  731. _launchd_ok("✓ Service restart requested")
  732. return
  733. print("⚠ launchd did not revive the gateway after its graceful exit — forcing restart")
  734. else:
  735. print(f"⚠ Gateway drain timed out after {wait_budget:.0f}s — forcing launchd restart")
  736. if not refresh_ok and _gw().get_launchd_plist_path().exists() and not _gw().launchd_plist_is_current():
  737. # The refresh attempted a reload and launchd never re-registered
  738. # the (rewritten) job: kickstart would hang on the same wall. The
  739. # bootout already happened inside the refresh — bootstrap is the
  740. # bounded revival path. (False alone is ambiguous: a missing or
  741. # already-current plist also returns False, and both must keep
  742. # the ordinary kickstart flow.)
  743. print("↻ launchd job was not re-registered by the plist refresh; reloading")
  744. plist_path = str(_gw().get_launchd_plist_path())
  745. subprocess.run(["launchctl", "bootstrap", _gw()._launchd_domain(), plist_path], check=True, timeout=30)
  746. subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30)
  747. _launchd_ok("✓ Service restarted")
  748. return
  749. # Captured: an unloaded job (3/113/125) is the expected case below, which
  750. # prints its own ↻ line — and e.stderr feeds the update_cmd failure diagnostic.
  751. _gw()._wait_for_api_server_port_free()
  752. subprocess.run(["launchctl", "kickstart", "-k", target], check=True, timeout=90, **_gw()._CAPTURE_TEXT)
  753. _launchd_ok("✓ Service restarted")
  754. except subprocess.CalledProcessError as e:
  755. if not _launchd_error_indicates_unloaded(e):
  756. _gw()._launchd_degrade_or_raise(e, "launchctl kickstart")
  757. return
  758. # Job not loaded — bootstrap and start fresh
  759. print("↻ launchd job was unloaded; reloading")
  760. try:
  761. # After a drain the job is usually still registered (bootstrap would hit EIO): boot it out first.
  762. # Captured: best-effort (the job may already be unloaded after the drain),
  763. # so an expected Boot-out failed: 3 must not leak past the ↻ line below.
  764. subprocess.run(["launchctl", "bootout", target], check=False, timeout=90, **_gw()._CAPTURE_TEXT)
  765. plist_path = str(_gw().get_launchd_plist_path())
  766. subprocess.run(["launchctl", "bootstrap", _gw()._launchd_domain(), plist_path], check=True, timeout=30)
  767. subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30)
  768. except subprocess.CalledProcessError as e2:
  769. _gw()._launchd_degrade_or_raise(e2, "launchctl")
  770. return
  771. _launchd_ok("✓ Service restarted")
  772. # KeepAlive relaunches at most ~once per 10s, so a self-restart leaves the label pid-less that long.
  773. LAUNCHD_SUPERVISION_VERIFY_TIMEOUT = 20.0
  774. def wait_for_launchd_gateway_supervision(
  775. *,
  776. timeout: float = LAUNCHD_SUPERVISION_VERIFY_TIMEOUT,
  777. label: str | None = None,
  778. poll_interval: float = 0.5,
  779. old_pid: int | None = None,
  780. ) -> bool:
  781. """Poll launchd until it supervises a live gateway; True at once if the detached fallback is active.
  782. ``launchd_restart`` returns once the restart is *requested* (asynchronous), so it can't see a helper
  783. dying before bootstrap or a ``launchctl bootstrap`` that exits 0 without registering.
  784. The ``_request_gateway_self_restart`` branch hands the work to the running gateway and returns
  785. immediately, and a plist reload is handed to a detached helper. Both are asynchronous, so a caller that
  786. reads "returned without raising" as "the service is up" cannot see a helper that dies before its first
  787. bootstrap (#88848) — nor a ``launchctl bootstrap`` that exits 0 without registering, which the reporter
  788. measured on macOS 26.6.1.
  789. Judge the outcome the way #80491 taught the helper to judge it: by a live supervised pid, never by an
  790. exit code. :func:`_launchctl_supervised_pid` is already that probe, so this only adds the wait.
  791. ``old_pid`` is the pid launchd ran for the label *before* the restart: a restart that leaves the same
  792. process running is not a restart, so passing it holds the invoking profile to the same fresh-pid
  793. contract :func:`_wait_for_launchd_service_pid` enforces for sibling labels. With ``old_pid=None``
  794. (no pre-restart pid was observable) any supervised pid counts, as before.
  795. """
  796. if _gw()._launchd_unsupported_marker_exists():
  797. return True
  798. label = label or _gw().get_launchd_label()
  799. def _supervised() -> bool:
  800. pid = _gw()._launchctl_supervised_pid(label)
  801. return pid is not None and pid != old_pid
  802. return poll_until(_supervised, max(timeout, 0.0), max(poll_interval, 0.01))
  803. def launchd_status(deep: bool = False):
  804. plist_path = _gw().get_launchd_plist_path()
  805. label = _gw().get_launchd_label()
  806. try:
  807. result = subprocess.run(["launchctl", "list", label], timeout=10, **_gw()._CAPTURE_TEXT, check=False)
  808. service_listed = result.returncode == 0
  809. list_output = result.stdout
  810. except subprocess.TimeoutExpired:
  811. service_listed = False
  812. list_output = ""
  813. # `launchctl list` exits 0 for any registered definition (even `state = not running`); only a PID proves a process.
  814. launchd_pid = _gw()._parse_launchd_pid_from_list_output(list_output) if service_listed else None
  815. # Hermes PID may be a detached fallback process; when launchd IS supervising both PIDs match — don't double-count.
  816. from gateway.status import get_running_pid
  817. fallback_pid = get_running_pid(cleanup_stale=False)
  818. if launchd_pid is not None and fallback_pid == launchd_pid:
  819. fallback_pid = None
  820. # Marker from a 5/125 bootstrap/kickstart failure explains *why* launchd can't supervise.
  821. launchd_unsupported = _gw()._launchd_unsupported_marker_exists()
  822. print(f"Launchd plist: {plist_path}")
  823. if _gw().launchd_plist_is_current():
  824. print("✓ Service definition matches the current Hermes install")
  825. else:
  826. print("⚠ Service definition is stale relative to the current Hermes install")
  827. print(" Run: hermes gateway start")
  828. if not service_listed:
  829. print("✗ Gateway service is not loaded")
  830. print(" Service definition exists locally but launchd has not loaded it.")
  831. print(" Run: hermes gateway start")
  832. if fallback_pid:
  833. print(f" Note: a detached gateway process is running (PID {fallback_pid})")
  834. elif launchd_pid is not None:
  835. print(f"✓ Gateway is supervised by launchd (PID {launchd_pid})")
  836. print(" Auto-start at login and auto-restart on crash are available.")
  837. if launchd_unsupported:
  838. print(" (launchd domain was previously unavailable but is now working)")
  839. elif launchd_unsupported:
  840. print("⚠ Gateway service is registered but launchd is not supervising it")
  841. print(" launchd cannot manage the gateway on this macOS version.")
  842. if fallback_pid:
  843. print(f"✓ Detached fallback process is running (PID {fallback_pid})")
  844. print(" Cron jobs will fire. Stop with: hermes gateway stop")
  845. else:
  846. print("✗ No fallback process is running")
  847. print(" Run: hermes gateway start")
  848. print(" ⚠ Auto-start at login and auto-restart on crash are NOT available.")
  849. else:
  850. print("✓ Gateway service is registered with launchd")
  851. print(list_output)
  852. if fallback_pid:
  853. print(f" Detached gateway process is running (PID {fallback_pid})")
  854. if deep:
  855. log_file = _gw().get_hermes_home() / "logs" / "gateway.log"
  856. if log_file.exists():
  857. print()
  858. print("Recent logs:")
  859. subprocess.run(["tail", "-20", str(log_file)], timeout=10, check=False)