| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999 |
- """Gateway launchd (macOS LaunchAgent) backend: plist generation/refresh, bootstrap, start/stop/restart/status.
- Extracted from ``hermes_cli/gateway.py``. Bodies read facade helpers through ``_gw()`` (late
- binding on ``hermes_cli.gateway``) so the seams tests and callers patch on the facade keep
- intercepting the moved code.
- """
- from __future__ import annotations
- from pathlib import Path
- import contextlib
- import json
- import os
- import shlex
- import subprocess
- import sys
- import time
- from xml.sax.saxutils import escape
- from datetime import UTC
- from hermes_cli import gateway_service_owner
- from agent.deadline import poll_until
- def _gw():
- from hermes_cli import gateway # late: the facade imports this module
- return gateway
- def get_launchd_label() -> str:
- """Return the launchd service label, scoped per profile."""
- suffix = _gw()._profile_suffix()
- return f"ai.hermes.gateway-{suffix}" if suffix else "ai.hermes.gateway"
- def _probe_launchd_domain_for_label(label: str) -> str:
- """Launchd domain managing ``label`` (uncached): ``gui/<uid>`` (Aqua), then ``user/<uid>``
- (Background/SSH), else the ``launchctl managername`` heuristic. Sibling profiles may live in
- different domains, so never reuse the cached ``_launchd_domain()`` for another label."""
- uid = os.getuid() # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
- gui_domain, user_domain = f"gui/{uid}", f"user/{uid}"
- launchctl_errors = (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError)
- for domain in (gui_domain, user_domain):
- try:
- subprocess.run(["launchctl", "print", f"{domain}/{label}"], check=True, timeout=5, capture_output=True)
- return domain
- except launchctl_errors:
- pass
- # Not loaded anywhere: Aqua → gui/<uid>; anything else (Background, loginwindow) → user/<uid>,
- # the pre-probing default and the recommended domain on macOS 26+.
- try:
- result = subprocess.run(["launchctl", "managername"], timeout=5, **_gw()._CAPTURE_TEXT, check=False)
- if "Aqua" in (result.stdout or ""):
- return gui_domain
- except launchctl_errors:
- pass
- return user_domain
- def _launchd_domain() -> str:
- """Domain managing the current profile's gateway; cached per process so start/stop/restart agree.
- See #40831, #23387.
- """
- # The cache lives on the facade: tests and callers reset ``hermes_cli.gateway._resolved_launchd_domain``.
- gw = _gw()
- if gw._resolved_launchd_domain is None:
- gw._resolved_launchd_domain = _probe_launchd_domain_for_label(gw.get_launchd_label())
- return gw._resolved_launchd_domain
- # 125 ("Domain does not support specified action") and 3/113 ("Could not find service") all mean
- # the job isn't loaded in the target domain: re-bootstrap the plist and retry.
- _LAUNCHD_JOB_UNLOADED_EXIT_CODES = frozenset({3, 113, 125})
- # 5 (EIO) / persistent 125 mean either a stale still-registered label (recoverable: bootout +
- # bootstrap, which `_launchctl_bootstrap()` tries first) or a domain that genuinely can't manage
- # services (macOS 26+). Only when the retry ALSO fails do callers degrade to a detached process.
- # launchctl returns 5 ("Input/output error") or a persistent 125 in two very different situations, so exit 5
- # is NOT on its own proof the domain is broken: 1. See #42914. 2. Here launchd cannot supervise the gateway
- # at all and we degrade to a detached background process (the `nohup hermes gateway run` workaround). See
- # #23387.
- _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES = frozenset({5, 125})
- def _launchd_error_indicates_unloaded(exc: subprocess.CalledProcessError) -> bool:
- """True when launchctl failed because the job isn't loaded (retry bootstrap)."""
- return exc.returncode in _LAUNCHD_JOB_UNLOADED_EXIT_CODES
- def _launchctl_domain_unsupported(returncode: int) -> bool:
- """True when launchctl can't manage the domain even after a fresh bootstrap (macOS 26+) — degrade to detached."""
- return returncode in _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES
- # EIO from `launchctl bootstrap` = label *already* registered (stale load); recoverable, not an unmanageable domain.
- _LAUNCHCTL_BOOTSTRAP_EIO = 5
- def _launchctl_bootstrap(domain: str, plist_path, label: str, *, timeout: int = 30) -> None:
- """Bootstrap a launchd job, recovering from a stale still-registered label (EIO 5). Without the
- bootout + retry that case is misread as an unmanageable domain and degrades to detached, silently
- losing auto-start and crash-restart."""
- bootstrap = ["launchctl", "bootstrap", domain, str(plist_path)]
- try:
- subprocess.run(bootstrap, check=True, timeout=timeout)
- except subprocess.CalledProcessError as exc:
- if exc.returncode != _LAUNCHCTL_BOOTSTRAP_EIO:
- raise
- # Stale registration — bootout the leftover label and bootstrap once more.
- # Captured: the bootout is best-effort (a drained job may already be
- # unloaded), so its expected 3/113/125 stderr must not leak to the terminal.
- subprocess.run(
- ["launchctl", "bootout", f"{domain}/{label}"],
- check=False, timeout=timeout, **_gw()._CAPTURE_TEXT)
- subprocess.run(bootstrap, check=True, timeout=timeout)
- def _launchd_reload_log_path() -> Path:
- """Path the launchd reload watchdog tails for persistent-orphan detection."""
- return _gw().get_hermes_home() / "logs" / "launchd-reload.log"
- def _append_launchd_reload_log(message: str) -> None:
- """Append a timestamped line to the launchd reload log (best-effort)."""
- path = _launchd_reload_log_path()
- try:
- path.parent.mkdir(parents=True, exist_ok=True)
- from datetime import datetime as _dt
- stamp = _dt.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %z")
- with path.open("a", encoding="utf-8") as fh:
- fh.write(f"[{stamp}] {message}\n")
- except OSError:
- pass
- def _launchd_reload_budget() -> float:
- """Bootstrap retry window for a plist reload: the failure happens while the old gateway is still
- draining (default 180s), so size it to the drain timeout with a 30s floor."""
- return max(30.0, _gw()._get_restart_drain_timeout())
- def _launchctl_supervised_pid(label: str) -> int | None:
- """PID launchd currently runs for ``label``, or None when it runs none. ``launchctl list`` exits 0 for
- a mere registered definition (``state = not running`` on macOS 26+), so a PID — not the exit code — is
- the answer. Domain-agnostic on purpose: ``launchctl print`` domain probes fail on macOS-26 per-user
- domains, which is why the invoking profile verifies through this and not ``_launchd_print_service_pid``."""
- try:
- result = subprocess.run(["launchctl", "list", label], check=False, timeout=10, **_gw()._CAPTURE_TEXT)
- except (subprocess.TimeoutExpired, OSError):
- return None
- if result.returncode != 0:
- return None
- return _gw()._parse_launchd_pid_from_list_output(result.stdout)
- def _launchctl_label_supervising_process(label: str) -> bool:
- """True when launchd knows ``label`` AND runs a process for it."""
- return _gw()._launchctl_supervised_pid(label) is not None
- def _retry_launchctl_bootstrap_until_registered(
- domain: str, plist_path, label: str, *, deadline: float
- ) -> bool:
- """Retry ``_launchctl_bootstrap`` until the label supervises a process or ``deadline`` passes. Under
- load bootstrap can fail even after bootout, during a drain (default 180s) — ~10s is too short."""
- attempt = 0
- while True:
- attempt += 1
- try:
- _gw()._launchctl_bootstrap(domain, plist_path, label, timeout=30)
- if _gw()._launchctl_label_supervising_process(label):
- return True
- outcome = f"exited 0 but {domain}/{label} has no supervised process (launchctl list)"
- except subprocess.CalledProcessError as exc:
- outcome = f"failed (rc={exc.returncode}) for {domain}/{label}"
- except subprocess.TimeoutExpired:
- outcome = f"timed out for {domain}/{label}"
- _gw()._append_launchd_reload_log(f"bootstrap attempt {attempt} {outcome} — retrying")
- if time.monotonic() >= deadline:
- return False
- time.sleep(2)
- # launchd-unsupported marker: written when the domain can't be managed (exit 5/125, macOS 26+) so
- # `launchd_status()` can explain missing supervision; cleared on successful bootstrap/kickstart.
- def _launchd_unsupported_marker_path() -> Path:
- return _gw().get_hermes_home() / ".gateway-launchd-unsupported"
- def _write_launchd_unsupported_marker() -> None:
- """Persist that launchd cannot supervise the gateway on this host."""
- from datetime import datetime, timezone
- payload = {
- "written_at": datetime.now(UTC).isoformat(),
- "reason": "launchd domain unsupported (exit 5/125)",
- }
- with contextlib.suppress(OSError):
- _launchd_unsupported_marker_path().write_text(json.dumps(payload), encoding="utf-8")
- def _clear_launchd_unsupported_marker() -> None:
- """Clear the unsupported marker when launchd bootstrap succeeds."""
- with contextlib.suppress(OSError):
- _launchd_unsupported_marker_path().unlink(missing_ok=True)
- def _launchd_unsupported_marker_exists() -> bool:
- return _launchd_unsupported_marker_path().exists()
- def _gateway_run_command() -> list[str]:
- from hermes_cli._launchers import runtime_command
- from hermes_constants import get_default_hermes_root
- # Every caller spawns an UNSUPERVISED child, which follows the sticky active_profile unless a
- # selector pins it (main._apply_profile_override): the root's replacement booted that profile.
- selector = shlex.split(_gw()._profile_arg())
- if not selector and _gw().get_hermes_home().resolve() == get_default_hermes_root().resolve():
- selector = ["--profile", "default"]
- return runtime_command(_gw().PROJECT_ROOT, [*selector, "gateway", "run", "--replace"],
- python=_gw().get_python_path())
- def launchd_program_arguments(command: list[str], stdout_log: Path, stderr_log: Path) -> list[str]:
- """launchd ``ProgramArguments`` that run ``command`` with a Local Network identity macOS accepts (#71206).
- macOS Local Network Privacy attributes a socket to the process launchd spawned for the job. A bare
- venv Python has no application ID and is not platform-entitled, so every LAN connect from the
- launchd gateway dies with ``EHOSTUNREACH`` while the same code works from Terminal (whose grant it
- inherits). An ad-hoc-signed helper .app does not help: nehelper never prompts for it and denies
- (#57812 dead-end table, re-verified live on macOS 26.3). ``/usr/bin/osascript`` spawning the child
- makes it osascript-responsible — an Apple platform binary — so the child is exempt; ``/bin/sh -c
- exec …`` and ``/usr/bin/time`` wrappers are NOT (the launchd job identity is the non-entitled first
- executable).
- Standard Additions' ``do shell script`` polls WindowServer for a user-cancel event while it waits.
- That is appropriate for a short interactive script but, for the gateway's process lifetime, burns CPU
- and keeps a WindowServer event connection busy (external-display wake stalls ~10s on macOS 27, #123595).
- JXA calling libc ``system()`` waits in the kernel instead while retaining osascript as the responsible
- process. The shell's ``exec`` keeps the gateway in the launchd job's process group, so ``launchctl
- bootout`` / ``kickstart -k`` still deliver SIGTERM to it. stdout/stderr are appended inside the shell
- command because ``system()`` otherwise inherits osascript's plist log handles. The encoded wait status
- is translated back to a process exit code so KeepAlive's ``SuccessfulExit`` semantics are preserved.
- """
- shell = f"exec {shlex.join(command)} >> {shlex.quote(str(stdout_log))} 2>> {shlex.quote(str(stderr_log))}"
- javascript = (
- 'ObjC.import("stdlib"); '
- f"const status=$.system({json.dumps(shell)}); "
- "const signal=status & 127; "
- "$.exit(status === -1 ? 1 : signal === 0 ? (status >> 8) & 255 : 128 + signal);"
- )
- return ["/usr/bin/osascript", "-l", "JavaScript", "-e", javascript]
- def _timestamped_stderr_gateway_command(error_log: Path, *, external_supervisor: bool = False) -> list[str]:
- """Wrap gateway run so raw stderr lines are timestamped before file write. ``external_supervisor``
- (launchd ProgramArguments only) adds ``--external-supervisor`` so ``hermes update`` hands back to
- launchd, and drops ``--replace``: KeepAlive respawns would re-arm takeover, so two profiles sharing
- a token would kill each other forever.
- ``external_supervisor=True`` is for launchd ProgramArguments only: the inner ``gateway run`` must carry
- ``--external-supervisor`` so ``hermes update`` sees the flag on the live grandchild argv and hands the
- process back to launchd instead of starting a detached watcher (#86893 / #87005). The detached nohup
- fallback stays unmarked.
- Supervised starts also drop ``--replace`` (issue #79048): a launchd service is respawned by KeepAlive,
- so takeover authority would be re-armed on every respawn — two profiles legitimately sharing one
- platform token would each terminate the sibling, and launchd would revive the victim forever. Bounded
- replacement is the lifecycle commands' job (``launchctl kickstart -k``, drain in ``launchd_restart()``,
- bootout+bootstrap in install/refresh), which run before supervision resumes. Mirrors
- ``generate_systemd_unit``, whose ExecStart also runs ``gateway run`` without ``--replace``.
- """
- from hermes_cli._launchers import installation_command, runtime_command
- inner = _gw()._gateway_run_command()
- if external_supervisor:
- inner = installation_command(_gw().PROJECT_ROOT, [*shlex.split(_gw()._profile_arg()), "gateway", "run"],
- python=_gw().get_python_path())
- inner = [part for part in inner if part != "--replace"]
- if "--external-supervisor" not in inner:
- inner.append("--external-supervisor")
- command = installation_command if external_supervisor else runtime_command
- return command(_gw().PROJECT_ROOT, ["--error-log", str(error_log), "--", *inner],
- module="hermes_cli.stderr_timestamp", python=_gw().get_python_path())
- def _spawn_detached_gateway() -> bool:
- """Launch the gateway detached (launchd fallback for macOS 26+). CLI-managed nohup equivalent:
- stdout → gateway.log, timestamped stderr → gateway.error.log, PID via gateway.pid so stop/status work.
- Used when launchctl can no longer bootstrap/kickstart the gateway on macOS 26+ (issue #23387). Mirrors
- the `nohup hermes gateway run --replace` workaround but keeps it CLI-managed: stdout goes to
- gateway.log, stderr is timestamped into gateway.error.log, and the PID is tracked via the gateway.pid
- file that `run_gateway` writes, so stop/status/restart keep working.
- """
- from hermes_cli._subprocess_compat import windows_detach_popen_kwargs
- from hermes_constants import get_hermes_home
- from tools.environments.local import served_profile_child_env
- log_dir = _gw().get_hermes_home() / "logs"
- log_dir.mkdir(parents=True, exist_ok=True)
- child_env = served_profile_child_env(
- target_home=get_hermes_home(), inherit_credentials=True,
- )
- child_env.pop("_HERMES_GATEWAY", None)
- try:
- with open(log_dir / "gateway.log", "ab") as out:
- subprocess.Popen(
- _timestamped_stderr_gateway_command(log_dir / "gateway.error.log"),
- stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.DEVNULL,
- env=child_env,
- # detached-spawn: ok — foreground-less gateway launch, a service meant to outlive the CLI
- **windows_detach_popen_kwargs(),
- )
- except OSError:
- return False
- return True
- def _launchd_fallback_to_detached(reason: str, *, exit_on_failure: bool = True) -> bool:
- """Start the gateway detached when launchd can't manage it; on failure print the manual workaround
- and (by default) exit 1."""
- from hermes_constants import display_hermes_home as _dhh
- _gw()._write_launchd_unsupported_marker()
- print(f"⚠ launchd cannot manage the gateway on this macOS version ({reason}).")
- if _gw()._spawn_detached_gateway():
- print("✓ Started gateway as a background process instead")
- print(" It will NOT auto-start at login or auto-restart on crash.")
- print(f" Logs: {_dhh()}/logs/gateway.log")
- print(" Stop it with: hermes gateway stop")
- return True
- _gw().print_error("Failed to start the gateway as a background process.")
- print(f" Try manually: nohup hermes gateway run --replace > {_dhh()}/logs/gateway.log 2>&1 &")
- if exit_on_failure:
- sys.exit(1)
- return False
- def _launchd_degrade_or_raise(exc: subprocess.CalledProcessError, what: str) -> None:
- """Shared launchctl failure policy: domain unmanageable (5/125) → detached fallback; else re-raise.
- A 5/125 exit is evidence about the *domain* only when launchd is not already supervising this job.
- EIO (5) is ``launchctl bootstrap``'s answer for a label that is already loaded, so the ordinary
- "reinstall/restart over the live gateway" case lands here with the service up and supervised.
- Degrading there is not a graceful fallback: it writes the permanent launchd-unsupported marker and
- starts a detached gateway *beside* the supervised one, and the marker makes
- :func:`wait_for_launchd_gateway_supervision` answer True unconditionally — so no later
- install/update can tell that nothing ties the gateway to launchd any more. A live supervised PID is
- direct evidence this macOS does manage the job, so surface the failure instead of branding the host.
- """
- if not _launchctl_domain_unsupported(exc.returncode):
- raise exc
- label = _gw().get_launchd_label()
- if _gw()._launchctl_label_supervising_process(label):
- print(f"⚠ {what} failed (exit {exc.returncode}), but launchd still supervises {label}")
- print(" Not switching to the detached fallback — this host manages the job.")
- print(" Apply the definition with: hermes gateway stop && hermes gateway install --force")
- raise exc
- _launchd_fallback_to_detached(f"{what} exit {exc.returncode}")
- def generate_launchd_plist() -> str:
- from html import escape
- # Stable cwd anchor — never the volatile source checkout (same rot risk as systemd's WorkingDirectory).
- working_dir = _gw()._stable_service_working_dir()
- hermes_home = str(_gw().get_hermes_home().resolve())
- log_dir = _gw().get_hermes_home() / "logs"
- log_dir.mkdir(parents=True, exist_ok=True)
- label = _gw().get_launchd_label()
- # launchd's default PATH misses Homebrew, nvm, cargo…; prepend venv/bin + node dirs (as in the
- # systemd unit) so node stays resolvable even if the shell PATH changes, then the shell PATH.
- priority_dirs = _gw()._build_service_path_dirs()
- _gw()._append_node_dir_for_service(priority_dirs)
- sane_path = ":".join(dict.fromkeys(priority_dirs + [p for p in os.environ.get("PATH", "").split(":") if p]))
- # ProgramArguments (incl. --profile); the stderr wrapper keeps launchd restart semantics while timestamping
- # stderr; the osascript wrapper gives the job a Local Network identity (see launchd_program_arguments).
- stdout_log, stderr_log = log_dir / "gateway.log", log_dir / "gateway.error.log"
- command = _timestamped_stderr_gateway_command(stderr_log, external_supervisor=True)
- prog_args_xml = "\n ".join(
- f"<string>{escape(part)}</string>" for part in launchd_program_arguments(command, stdout_log, stderr_log)
- )
- # Persist the configured RLIMIT_NOFILE floor: launchd defaults to soft 256, and every plist
- # rewrite would otherwise strip a manual limit and reintroduce EMFILE crashes.
- nofile_block = ""
- try:
- from hermes_cli.resource_limits import configured_nofile_soft_limit
- nofile_target = configured_nofile_soft_limit()
- except Exception:
- nofile_target = None
- if nofile_target:
- nofile_block = f"""
- <key>SoftResourceLimits</key>
- <dict>
- <key>NumberOfFiles</key>
- <integer>{nofile_target}</integer>
- </dict>
- """
- return f"""<?xml version="1.0" encoding="UTF-8"?>
- <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
- <plist version="1.0">
- <dict>
- <key>Label</key>
- <string>{label}</string>
- <key>ProgramArguments</key>
- <array>
- {prog_args_xml}
- </array>
-
- <key>WorkingDirectory</key>
- <string>{working_dir}</string>
-
- <key>EnvironmentVariables</key>
- <dict>
- <key>PATH</key>
- <string>{sane_path}</string>
- <key>HERMES_HOME</key>
- <string>{hermes_home}</string>
- <key>HERMES_SUPERVISED_CHILD</key>
- <string>1</string>
- </dict>
- <key>LimitLoadToSessionType</key>
- <array>
- <string>Aqua</string>
- <string>Background</string>
- </array>
-
- <key>RunAtLoad</key>
- <true/>
-
- <key>KeepAlive</key>
- <dict>
- <key>SuccessfulExit</key>
- <false/>
- </dict>
- <!-- SuccessfulExit=false parks a clean stop (exit 0), including gateway
- EX_CONFIG 78 after stderr_timestamp maps it to 0 — launchd cannot
- honor RestartPreventExitStatus, and KeepAlive=true respawned token
- collisions forever (#89477). Exit 75 and crashes still relaunch.
- ThrottleInterval raises launchd's default 10s minimum respawn interval
- to 30s so a crash-looping gateway can't hammer launchd into a rapid
- respawn storm; ExitTimeOut is the graceful-drain headroom before
- launchd escalates from SIGTERM to SIGKILL on stop. The per-user
- (gui) launchd domain clamps it to 60s regardless of what is written
- here, so 60 is the most a LaunchAgent can get; the gateway reads
- the live value at boot and fits its signal-driven drain inside it
- (gateway.restart.read_launchd_exit_timeout_s). -->
- <key>ThrottleInterval</key>
- <integer>30</integer>
- <key>ExitTimeOut</key>
- <integer>60</integer>
- {nofile_block}
- <key>StandardOutPath</key>
- <string>{stdout_log}</string>
-
- <key>StandardErrorPath</key>
- <string>{stderr_log}</string>
- </dict>
- </plist>
- """
- def launchd_plist_is_current() -> bool:
- """Check if the installed launchd plist matches the currently generated one."""
- plist_path = _gw().get_launchd_plist_path()
- if not plist_path.exists():
- return False
- installed = plist_path.read_text(encoding="utf-8-sig")
- norm = _gw()._normalize_launchd_plist_for_comparison
- return norm(installed) == norm(_gw().generate_launchd_plist())
- def _spawn_deferred_launchd_reload(
- *, domain: str, label: str, target: str, plist_path: Path, gateway_pid: int
- ) -> bool:
- """Hand the bootout/bootstrap cycle to a transient ``launchctl submit`` job; True if spawned. The
- helper waits for the OLD gateway to exit (bootstrap during drain fails EIO), then retries bootstrap
- until ``launchctl list`` shows a positive PID or the drain budget elapses."""
- reload_log_path = _launchd_reload_log_path()
- with contextlib.suppress(OSError):
- reload_log_path.parent.mkdir(parents=True, exist_ok=True)
- # Durable pre-bootout marker: distinguishes "helper never started" from "helper ran but failed".
- _gw()._append_launchd_reload_log(f"Launchd reload helper started for {target}")
- _reload_budget = int(_launchd_reload_budget())
- q_target, q_label, q_log = shlex.quote(target), shlex.quote(label), shlex.quote(str(reload_log_path))
- stamp = "$(date '+%Y-%m-%d %H:%M:%S %z')"
- # Require a POSITIVE PID: `launchctl list` also exits 0 for a registered-but-not-running
- # definition, and a crashed job reports `"PID" = -1` (mirrors _parse_launchd_pid_from_list_output).
- listed = f"launchctl list {q_label} 2>/dev/null | grep -qE '\\\"PID\\\" = [0-9]+;'"
- # Unique per reload so concurrent/repeated reloads never collide.
- submit_label = f"{label}.reload.{os.getpid()}.{int(time.time())}"
- reload_script = (
- f"sleep 2; "
- f"launchctl bootout {q_target} 2>/dev/null; "
- # Wait for the OLD gateway to exit: bootout only SIGTERMs and every bootstrap during the drain fails EIO.
- f"_wait_deadline=$(($(date +%s) + {_reload_budget})); "
- f"while kill -0 {gateway_pid} 2>/dev/null; do if [ $(date +%s) -ge $_wait_deadline ]; then "
- f" echo \"[{stamp}] old gateway pid {gateway_pid} still alive after {_reload_budget}s drain wait — bootstrapping anyway\" >> {q_log}; "
- f" break; fi; sleep 1; done; "
- # Let launchd finish unregistering the label after the process exits.
- f"sleep 1; _deadline=$(($(date +%s) + {_reload_budget})); while :; do "
- f" launchctl bootstrap {shlex.quote(domain)} {shlex.quote(str(plist_path))} 2>/dev/null; "
- f" if {listed}; then break; fi; "
- f" echo \"[{stamp}] bootstrap not yet registered for {q_target} — retrying\" >> {q_log}; "
- f" if [ $(date +%s) -ge $_deadline ]; then break; fi; sleep 2; done; "
- f"if ! {listed}; then "
- f" echo \"[{stamp}] FAILED launchd reload for {q_target} — service NOT registered after {_reload_budget}s of retries\" >> {q_log}; "
- f"fi; "
- # Submitted jobs stay registered after the script exits; removing our own label ends the one-shot job.
- f"launchctl remove {shlex.quote(submit_label)} 2>/dev/null"
- )
- try:
- # `launchctl submit` rather than setsid: setsid does NOT leave the launchd coalition that bootout kills.
- # Spawn the reload helper via `launchctl submit` (a transient launchd one-shot job) instead of
- # `start_new_session=True`. `start_new_session=True` only calls setsid(2), which creates a new POSIX
- # session but does NOT move the child outside the launchd job's process coalition. When `launchctl
- # bootout` fires on the gateway label, launchd terminates ALL processes in that coalition —
- # including a setsid-detached child (#69098). `launchctl submit` creates a wholly independent
- # transient launchd job that launchd manages separately from the gateway, so bootout of the gateway
- # job cannot reach the helper.
- subprocess.Popen(
- [
- "launchctl", "submit", "-l", submit_label, "-o", str(reload_log_path), "-e", str(reload_log_path),
- "--", "/bin/bash", "-c", reload_script,
- ],
- stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL,
- )
- except Exception as e:
- # Fall through to in-process bootout/bootstrap: risky in the coalition, but better than a never-reloaded plist.
- _gw().logger.warning("Deferred launchd reload could not be spawned: %s", e)
- _gw()._append_launchd_reload_log(
- f"FAILED to spawn launchd reload helper for {target}: {e} — falling back to in-process bootout/bootstrap"
- )
- return False
- return True
- def _bootout_drain_and_rebootstrap(domain, plist_path, label, old_pid, budget: float) -> bool:
- """Boot the job out, wait out the old gateway's drain, then bootstrap until launchd registers it.
- Bootstrap can fail silently under load during a drain, and KeepAlive can't revive an
- unregistered job, so the retry runs to a deadline. Waiting for *old_pid* first keeps the
- budget from burning on guaranteed EIO ("already loaded"). The bootout is best effort (the job
- may already be unloaded) and captured to keep expected noise off the terminal.
- """
- target = f"{domain}/{label}"
- subprocess.run(["launchctl", "bootout", target], check=False, timeout=90, **_gw()._CAPTURE_TEXT)
- if old_pid is not None and not _gw()._wait_for_pid_exit(old_pid, budget):
- _gw()._append_launchd_reload_log(
- f"old gateway pid {old_pid} still alive after "
- f"{int(budget)}s drain wait — bootstrapping {target} anyway"
- )
- return _gw()._retry_launchctl_bootstrap_until_registered(
- domain, plist_path, label, deadline=time.monotonic() + budget)
- def refresh_launchd_plist_if_needed() -> bool:
- """Rewrite the installed plist when the generated one differs, then bootout/bootstrap so launchd
- re-reads it immediately."""
- plist_path = _gw().get_launchd_plist_path()
- if not plist_path.exists() or _gw().launchd_plist_is_current():
- return False
- # Runs on every gateway start/restart: a plist pinning another home is another install's gateway.
- from hermes_cli.gateway_service_owner import definition_belongs_to_home
- if not definition_belongs_to_home(plist_path, _gw().get_hermes_home(), "rewrite"):
- return False
- new_plist = _gw().generate_launchd_plist()
- if gateway_service_owner.refuse_temp_home_service_write(new_plist, "launchd plist"):
- return False
- _gw()._prepare_service_launcher()
- plist_path.write_text(new_plist, encoding="utf-8")
- label = _gw().get_launchd_label()
- domain = _gw()._launchd_domain()
- target = f"{domain}/{label}"
- # Inside the gateway's launchd process tree (agent self-update) a direct bootout kills THIS CLI
- # before bootstrap runs, leaving the job unloaded with no KeepAlive.
- try:
- from gateway.status import get_running_pid
- gateway_pid = get_running_pid()
- except Exception:
- gateway_pid = None
- # POSIX ancestry is NOT a reliable "bootout will kill us" test (coalition membership survives
- # reparenting), so always prefer the detached helper; in-process is only the spawn-failure fallback.
- if (
- gateway_pid is not None
- and hasattr(os, "setsid") # POSIX-only; launchd is macOS so always true here
- ) and _spawn_deferred_launchd_reload(
- domain=domain, label=label, target=target, plist_path=plist_path, gateway_pid=gateway_pid
- ):
- print(
- "↻ Updated gateway launchd service definition; reload deferred to "
- "a transient launchd job (survives the bootout of this process)"
- )
- return True
- # Bootout/bootstrap so launchd reads the new definition.
- _reload_budget = _launchd_reload_budget()
- if not _bootout_drain_and_rebootstrap(domain, plist_path, label, gateway_pid, _reload_budget):
- _gw()._append_launchd_reload_log(
- f"FAILED launchd reload of {target} — service NOT registered after "
- f"retrying for {int(_reload_budget)}s (in-process fallback path)"
- )
- _gw().logger.error(
- "launchd reload of %s failed — service not registered after %ds of retries; see %s",
- target, int(_reload_budget), _launchd_reload_log_path(),
- )
- return False
- print("↻ Updated gateway launchd service definition to match the current Hermes install")
- return True
- def launchd_install(force: bool = False, *, start_now: bool = True, force_unit_path: bool = False):
- plist_path = _gw().get_launchd_plist_path()
- label = _gw().get_launchd_label()
- # A plist pinning another home is another install's gateway, --force included; --force-unit-path repoints.
- from hermes_cli.gateway_service_owner import definition_belongs_to_home
- if not force_unit_path and not definition_belongs_to_home(plist_path, _gw().get_hermes_home(), "overwrite"):
- sys.exit(1)
- # Loading the plist starts the gateway (RunAtLoad), so a no-start install writes it without
- # loading it. A gateway that launchd already runs is still reloaded; this install did not start it.
- load = start_now or _gw()._launchctl_label_supervising_process(label)
- # A forced reinstall over a live job must remember the incumbent before bootstrap starts.
- # _launchctl_bootstrap() handles EIO by booting the stale label out and retrying once, but
- # that immediate retry races the old gateway's drain (#120629). The install path can wait
- # because it runs outside the supervised gateway process.
- force_old_pid = _gw()._launchctl_supervised_pid(label) if force and load else None
- if plist_path.exists() and not (force or force_unit_path):
- if _gw().launchd_plist_is_current():
- print(f"Service already installed at: {plist_path}")
- print("Use --force to reinstall")
- return
- if load:
- print(f"↻ Repairing outdated launchd service at: {plist_path}")
- if _gw().refresh_launchd_plist_if_needed():
- print("✓ Service definition updated")
- else:
- # The plist was rewritten but launchd never registered it (or the write was refused):
- # a success line here would hide an unloaded service with no KeepAlive.
- from hermes_constants import display_hermes_home
- print(
- "⚠ Service definition could not be reloaded with launchd. "
- "Run 'hermes gateway install --force' or check "
- f"{display_hermes_home()}/logs/launchd-reload.log for details."
- )
- return
- plist_path.parent.mkdir(parents=True, exist_ok=True)
- new_plist = _gw().generate_launchd_plist()
- if gateway_service_owner.refuse_temp_home_service_write(new_plist, "launchd plist"):
- return
- print(f"Installing launchd service to: {plist_path}")
- _gw()._prepare_service_launcher()
- plist_path.write_text(new_plist, encoding="utf-8")
- if not load:
- # A job left loaded but idle (a parked clean exit) keeps its old definition, and that is
- # what `hermes gateway start` would kickstart instead of loading this plist.
- subprocess.run(
- ["launchctl", "bootout", f"{_gw()._launchd_domain()}/{label}"],
- check=False, timeout=90, **_gw()._CAPTURE_TEXT)
- print()
- print("✓ Service installed, not started (launchd starts it at your next login)")
- print()
- print("Next steps:")
- print(" hermes gateway start # Start it now")
- print(" hermes gateway status # Check status")
- return
- if force_old_pid is not None:
- domain = _gw()._launchd_domain()
- target = f"{domain}/{label}"
- budget = _gw()._launchd_reload_budget()
- if not _bootout_drain_and_rebootstrap(domain, plist_path, label, force_old_pid, budget):
- _gw()._append_launchd_reload_log(
- f"FAILED forced launchd install of {target} — service NOT registered after "
- f"retrying for {int(budget)}s"
- )
- _gw().print_error(
- f"launchctl bootstrap did not register {target} within {int(budget)}s; "
- "the gateway service is NOT loaded."
- )
- sys.exit(1)
- else:
- try:
- _gw()._launchctl_bootstrap(_gw()._launchd_domain(), plist_path, label, timeout=30)
- except subprocess.CalledProcessError as e:
- _gw()._launchd_degrade_or_raise(e, "launchctl bootstrap")
- return
- print()
- print("✓ Service installed and loaded!")
- _gw()._clear_launchd_unsupported_marker()
- print()
- print("Next steps:")
- print(" hermes gateway status # Check status")
- from hermes_constants import display_hermes_home as _dhh
- print(f" tail -f {_dhh()}/logs/gateway.log # View logs")
- def launchd_uninstall():
- gateway_service_owner.refuse_foreign_launchd_plist("uninstall")
- plist_path = _gw().get_launchd_plist_path()
- # Captured: uninstalling an already-unloaded job is fine — don't print Boot-out failed: 3.
- subprocess.run(
- ["launchctl", "bootout", f"{_launchd_domain()}/{get_launchd_label()}"],
- check=False, timeout=90, **_gw()._CAPTURE_TEXT)
- if plist_path.exists():
- plist_path.unlink()
- print(f"✓ Removed {plist_path}")
- print("✓ Service uninstalled")
- def launchd_start():
- gateway_service_owner.refuse_foreign_launchd_plist("start")
- plist_path = _gw().get_launchd_plist_path()
- label = _gw().get_launchd_label()
- # Self-heal if the plist is missing entirely (e.g., manual cleanup, failed upgrade)
- if not plist_path.exists():
- new_plist = _gw().generate_launchd_plist()
- if gateway_service_owner.refuse_temp_home_service_write(new_plist, "launchd plist"):
- sys.exit(1)
- print("↻ launchd plist missing; regenerating service definition")
- plist_path.parent.mkdir(parents=True, exist_ok=True)
- _gw()._prepare_service_launcher()
- plist_path.write_text(new_plist, encoding="utf-8")
- if _launchd_bootstrap_and_kickstart(plist_path, label):
- _launchd_ok("✓ Service started")
- return
- _gw().refresh_launchd_plist_if_needed()
- try:
- _launchctl_kickstart_current(label)
- except subprocess.CalledProcessError as e:
- if not _launchd_error_indicates_unloaded(e):
- raise
- # Job not loaded in this domain — re-bootstrap the plist and retry.
- print("↻ launchd job was unloaded; reloading service definition")
- if not _launchd_bootstrap_and_kickstart(plist_path, label):
- return
- _launchd_ok("✓ Service started")
- def _launchctl_kickstart_current(label: str) -> None:
- subprocess.run(["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], check=True, timeout=30)
- def _launchd_bootstrap_and_kickstart(plist_path: Path, label: str) -> bool:
- """Bootstrap then kickstart; False after degrading to detached (domain unsupported). Other errors propagate."""
- try:
- _gw()._launchctl_bootstrap(_gw()._launchd_domain(), plist_path, label, timeout=30)
- _launchctl_kickstart_current(label)
- except subprocess.CalledProcessError as e:
- _gw()._launchd_degrade_or_raise(e, "launchctl")
- return False
- return True
- def _launchd_ok(message: str) -> None:
- """Print a launchd success line and clear the unsupported marker (an OS fix recovers automatically)."""
- print(message)
- _gw()._clear_launchd_unsupported_marker()
- def launchd_stop():
- gateway_service_owner.refuse_foreign_launchd_plist("stop")
- target = f"{_launchd_domain()}/{get_launchd_label()}"
- _gw()._mark_planned_stop()
- # bootout unloads the definition so KeepAlive doesn't respawn; `hermes gateway start` re-bootstraps.
- try:
- # Captured: an already-unloaded job (3/113/125) is handled below, so launchctl's own
- # "Boot-out failed: 3" must not print around the ✓ line; e.stderr stays on the raised error.
- subprocess.run(["launchctl", "bootout", target], check=True, timeout=90, **_gw()._CAPTURE_TEXT)
- except subprocess.CalledProcessError as e:
- # Job already unloaded (3/113/125), or the domain can't be managed at all (5/125, macOS 26+
- # detached-fallback process, issue #23387) — in both cases just fall through to the PID-based kill
- # below.
- if not (_launchd_error_indicates_unloaded(e) or _launchctl_domain_unsupported(e.returncode)):
- raise
- _gw()._wait_for_gateway_exit(timeout=10.0, force_after=5.0)
- print("✓ Service stopped")
- def _launchd_kickstart(label: str, domain: str) -> None:
- """``launchctl kickstart -k domain/label``; raises so callers own per-label failure accounting."""
- subprocess.run(["launchctl", "kickstart", "-k", f"{domain}/{label}"], check=True, timeout=90, **_gw()._CAPTURE_TEXT)
- def _wait_for_launchd_service_pid(
- label: str, old_pid: int | None, timeout: float = 10.0, *, domain: str
- ) -> bool:
- """Poll ``domain/label`` (0.5s) until it runs on a fresh PID or ``timeout`` passes — KeepAlive respawn
- isn't instantaneous. launchctl ``TimeoutExpired`` propagates; callers own failure accounting."""
- def _fresh_pid() -> bool:
- pid = _gw()._launchd_print_service_pid(domain, label)[1]
- return pid is not None and pid > 0 and pid != old_pid
- return poll_until(_fresh_pid, max(timeout, 0.5), 0.5)
- def launchd_restart():
- gateway_service_owner.refuse_foreign_launchd_plist("restart")
- label = _gw().get_launchd_label()
- domain = _gw()._launchd_domain()
- target = f"{domain}/{label}"
- # A kickstart re-runs whatever definition launchd already holds. After an
- # update that definition was generated by the OLD checkout, so restarting
- # without refreshing first faithfully revives a stale service (and a wedged
- # stale job is what hangs kickstart into its 90s timeout). Refresh rewrites
- # the plist and bootout/bootstraps when it changed; no-op when current.
- # When the refresh could not re-register the job (launchd wedged), the
- # kickstart below would hang on the same wall — go straight to the
- # bootout/bootstrap-retry path, which is bounded and reports its own
- # failure instead of stalling the update for 90s.
- refresh_ok = _gw().refresh_launchd_plist_if_needed()
- from gateway.status import get_running_pid
- try:
- pid = get_running_pid()
- if pid is not None and _gw()._request_gateway_self_restart(pid):
- _launchd_ok("✓ Service restart requested")
- return
- if pid is not None and _gw().probe_gateway_loop_liveness(pid) == _gw().GATEWAY_LOOP_WEDGED:
- # Event loop provably dead: it can't process a graceful shutdown, so a full drain wait
- # only stalls the restart (and `hermes update`). Bounded SIGTERM → SIGKILL, ~10s.
- print(f"⚠ Gateway PID {pid} event loop is unresponsive — " "skipping drain and forcing a bounded stop...")
- _gw()._escalate_wedged_gateway(pid)
- pid = None
- if pid is not None:
- # Graceful in-band restart via SIGUSR1 (mirrors systemd); the budget covers both the idle wait
- # and the drain. A bare SIGTERM would lose the resume_pending handoff. Announce BEFORE waiting:
- # surfaces with no other feedback (desktop updater) read silence as "update stuck".
- wait_budget = _gw()._get_restart_exit_wait_budget()
- print(f"→ Stopping gateway (PID {pid}) — draining in-flight runs (up to {wait_budget:.0f}s)...")
- from hermes_cli.update_cmd_drain_report import drain_progress_reporter
- if _gw()._graceful_restart_via_sigusr1(pid, wait_budget, on_progress=drain_progress_reporter(budget_s=wait_budget)):
- # KeepAlive revives a planned exit, so do NOT kickstart (-k would kill the replacement) —
- # but a clean exit doesn't prove supervision, so verify a replacement PID appears first.
- if _gw()._wait_for_launchd_service_pid(label, pid, timeout=15.0, domain=domain):
- _launchd_ok("✓ Service restart requested")
- return
- print("⚠ launchd did not revive the gateway after its graceful exit — forcing restart")
- else:
- print(f"⚠ Gateway drain timed out after {wait_budget:.0f}s — forcing launchd restart")
- if not refresh_ok and _gw().get_launchd_plist_path().exists() and not _gw().launchd_plist_is_current():
- # The refresh attempted a reload and launchd never re-registered
- # the (rewritten) job: kickstart would hang on the same wall. The
- # bootout already happened inside the refresh — bootstrap is the
- # bounded revival path. (False alone is ambiguous: a missing or
- # already-current plist also returns False, and both must keep
- # the ordinary kickstart flow.)
- print("↻ launchd job was not re-registered by the plist refresh; reloading")
- plist_path = str(_gw().get_launchd_plist_path())
- subprocess.run(["launchctl", "bootstrap", _gw()._launchd_domain(), plist_path], check=True, timeout=30)
- subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30)
- _launchd_ok("✓ Service restarted")
- return
- # Captured: an unloaded job (3/113/125) is the expected case below, which
- # prints its own ↻ line — and e.stderr feeds the update_cmd failure diagnostic.
- _gw()._wait_for_api_server_port_free()
- subprocess.run(["launchctl", "kickstart", "-k", target], check=True, timeout=90, **_gw()._CAPTURE_TEXT)
- _launchd_ok("✓ Service restarted")
- except subprocess.CalledProcessError as e:
- if not _launchd_error_indicates_unloaded(e):
- _gw()._launchd_degrade_or_raise(e, "launchctl kickstart")
- return
- # Job not loaded — bootstrap and start fresh
- print("↻ launchd job was unloaded; reloading")
- try:
- # After a drain the job is usually still registered (bootstrap would hit EIO): boot it out first.
- # Captured: best-effort (the job may already be unloaded after the drain),
- # so an expected Boot-out failed: 3 must not leak past the ↻ line below.
- subprocess.run(["launchctl", "bootout", target], check=False, timeout=90, **_gw()._CAPTURE_TEXT)
- plist_path = str(_gw().get_launchd_plist_path())
- subprocess.run(["launchctl", "bootstrap", _gw()._launchd_domain(), plist_path], check=True, timeout=30)
- subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30)
- except subprocess.CalledProcessError as e2:
- _gw()._launchd_degrade_or_raise(e2, "launchctl")
- return
- _launchd_ok("✓ Service restarted")
- # KeepAlive relaunches at most ~once per 10s, so a self-restart leaves the label pid-less that long.
- LAUNCHD_SUPERVISION_VERIFY_TIMEOUT = 20.0
- def wait_for_launchd_gateway_supervision(
- *,
- timeout: float = LAUNCHD_SUPERVISION_VERIFY_TIMEOUT,
- label: str | None = None,
- poll_interval: float = 0.5,
- old_pid: int | None = None,
- ) -> bool:
- """Poll launchd until it supervises a live gateway; True at once if the detached fallback is active.
- ``launchd_restart`` returns once the restart is *requested* (asynchronous), so it can't see a helper
- dying before bootstrap or a ``launchctl bootstrap`` that exits 0 without registering.
- The ``_request_gateway_self_restart`` branch hands the work to the running gateway and returns
- immediately, and a plist reload is handed to a detached helper. Both are asynchronous, so a caller that
- reads "returned without raising" as "the service is up" cannot see a helper that dies before its first
- bootstrap (#88848) — nor a ``launchctl bootstrap`` that exits 0 without registering, which the reporter
- measured on macOS 26.6.1.
- Judge the outcome the way #80491 taught the helper to judge it: by a live supervised pid, never by an
- exit code. :func:`_launchctl_supervised_pid` is already that probe, so this only adds the wait.
- ``old_pid`` is the pid launchd ran for the label *before* the restart: a restart that leaves the same
- process running is not a restart, so passing it holds the invoking profile to the same fresh-pid
- contract :func:`_wait_for_launchd_service_pid` enforces for sibling labels. With ``old_pid=None``
- (no pre-restart pid was observable) any supervised pid counts, as before.
- """
- if _gw()._launchd_unsupported_marker_exists():
- return True
- label = label or _gw().get_launchd_label()
- def _supervised() -> bool:
- pid = _gw()._launchctl_supervised_pid(label)
- return pid is not None and pid != old_pid
- return poll_until(_supervised, max(timeout, 0.0), max(poll_interval, 0.01))
- def launchd_status(deep: bool = False):
- plist_path = _gw().get_launchd_plist_path()
- label = _gw().get_launchd_label()
- try:
- result = subprocess.run(["launchctl", "list", label], timeout=10, **_gw()._CAPTURE_TEXT, check=False)
- service_listed = result.returncode == 0
- list_output = result.stdout
- except subprocess.TimeoutExpired:
- service_listed = False
- list_output = ""
- # `launchctl list` exits 0 for any registered definition (even `state = not running`); only a PID proves a process.
- launchd_pid = _gw()._parse_launchd_pid_from_list_output(list_output) if service_listed else None
- # Hermes PID may be a detached fallback process; when launchd IS supervising both PIDs match — don't double-count.
- from gateway.status import get_running_pid
- fallback_pid = get_running_pid(cleanup_stale=False)
- if launchd_pid is not None and fallback_pid == launchd_pid:
- fallback_pid = None
- # Marker from a 5/125 bootstrap/kickstart failure explains *why* launchd can't supervise.
- launchd_unsupported = _gw()._launchd_unsupported_marker_exists()
- print(f"Launchd plist: {plist_path}")
- if _gw().launchd_plist_is_current():
- print("✓ Service definition matches the current Hermes install")
- else:
- print("⚠ Service definition is stale relative to the current Hermes install")
- print(" Run: hermes gateway start")
- if not service_listed:
- print("✗ Gateway service is not loaded")
- print(" Service definition exists locally but launchd has not loaded it.")
- print(" Run: hermes gateway start")
- if fallback_pid:
- print(f" Note: a detached gateway process is running (PID {fallback_pid})")
- elif launchd_pid is not None:
- print(f"✓ Gateway is supervised by launchd (PID {launchd_pid})")
- print(" Auto-start at login and auto-restart on crash are available.")
- if launchd_unsupported:
- print(" (launchd domain was previously unavailable but is now working)")
- elif launchd_unsupported:
- print("⚠ Gateway service is registered but launchd is not supervising it")
- print(" launchd cannot manage the gateway on this macOS version.")
- if fallback_pid:
- print(f"✓ Detached fallback process is running (PID {fallback_pid})")
- print(" Cron jobs will fire. Stop with: hermes gateway stop")
- else:
- print("✗ No fallback process is running")
- print(" Run: hermes gateway start")
- print(" ⚠ Auto-start at login and auto-restart on crash are NOT available.")
- else:
- print("✓ Gateway service is registered with launchd")
- print(list_output)
- if fallback_pid:
- print(f" Detached gateway process is running (PID {fallback_pid})")
- if deep:
- log_file = _gw().get_hermes_home() / "logs" / "gateway.log"
- if log_file.exists():
- print()
- print("Recent logs:")
- subprocess.run(["tail", "-20", str(log_file)], timeout=10, check=False)
|