service_manager.py 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773
  1. """Abstract service manager interface + systemd/launchd/Windows/s6 backends."""
  2. from __future__ import annotations
  3. import json
  4. import os
  5. import re
  6. import shlex
  7. import shutil
  8. import stat
  9. import subprocess
  10. import tempfile
  11. import time
  12. from pathlib import Path
  13. from typing import Literal, Protocol, runtime_checkable
  14. ServiceManagerKind = Literal["systemd", "launchd", "windows", "s6", "none"]
  15. # Profile names become s6 service directory names (``<scandir>/gateway-<profile>/``), so they
  16. # must not traverse paths, span filesystems, or break s6's own naming rules.
  17. _VALID_PROFILE_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
  18. _MAX_PROFILE_LEN = 251 # s6-svscan default name_max
  19. def validate_profile_name(name: str) -> None:
  20. """Raise ValueError unless ``name`` is a filesystem/s6-safe profile name."""
  21. if not name:
  22. raise ValueError("profile name must not be empty")
  23. if len(name) > _MAX_PROFILE_LEN:
  24. raise ValueError(f"profile name too long ({len(name)} > {_MAX_PROFILE_LEN})")
  25. if not _VALID_PROFILE_RE.match(name):
  26. raise ValueError(f"profile name must match [a-z0-9][a-z0-9_-]*, got {name!r}")
  27. @runtime_checkable
  28. class ServiceManager(Protocol):
  29. """Init-system-specific service operations.
  30. Lifecycle methods exist on every backend. Runtime registration (register/unregister/
  31. list_profile_gateways) is s6-only — check ``supports_runtime_registration()`` first.
  32. """
  33. kind: ServiceManagerKind
  34. def start(self, name: str) -> None: ...
  35. def stop(self, name: str) -> None: ...
  36. def restart(self, name: str) -> None: ...
  37. def is_running(self, name: str) -> bool: ...
  38. def supports_runtime_registration(self) -> bool: ...
  39. def register_profile_gateway(
  40. self, profile: str, *, extra_env: dict[str, str] | None = None, start_now: bool = True
  41. ) -> None: ...
  42. def unregister_profile_gateway(self, profile: str) -> None: ...
  43. def list_profile_gateways(self) -> list[str]: ...
  44. def detect_service_manager() -> ServiceManagerKind:
  45. """Return "s6" (s6-svscan is PID 1), "windows", "launchd", "systemd" (working bus) or "none".
  46. This function does NOT replace ``supports_systemd_services()`` —
  47. host call sites continue to use that. It exists for new backend-
  48. agnostic code (profile create/delete hooks, the s6 dispatch path
  49. in ``hermes gateway start/stop/restart``).
  50. """
  51. # Deferred so importing this module (Protocol type, validate_profile_name) doesn't drag in
  52. # the whole gateway dependency graph.
  53. from hermes_cli.gateway import is_macos, is_windows, supports_systemd_services
  54. # Gate on _s6_running() alone, NOT is_container(): the latter only detects Docker/Podman/lxc
  55. # and is False on Fly's Firecracker microVMs even though s6-overlay is PID 1 there — that
  56. # made the s6 dispatch inert on Fly, so `hermes gateway start` spawned a foreground gateway
  57. # competing with the supervised one.
  58. if _s6_running():
  59. return "s6"
  60. if is_windows():
  61. return "windows"
  62. if is_macos():
  63. return "launchd"
  64. if supports_systemd_services():
  65. return "systemd"
  66. return "none"
  67. def _s6_running() -> bool:
  68. """True when s6-svscan is PID 1 in this container.
  69. Must work for the unprivileged hermes user too: ``/proc/1/exe`` is unreadable for other UIDs
  70. (``resolve()`` silently yields the literal ``exe``), which made runtime registration inert in
  71. production. Probe the world-readable ``/proc/1/comm`` AND ``/run/s6/basedir`` — either alone
  72. can false-positive.
  73. The obvious probe — ``Path('/proc/1/exe').resolve()`` — only works as root: for any other UID, the
  74. symlink at ``/proc/1/exe`` is unreadable and ``resolve()`` silently returns the path unchanged, so the
  75. resolved name is the literal ``"exe"`` and detection always fails. Since every Hermes runtime call
  76. inside the container drops to hermes via ``s6-setuidgid``, that silent failure made the entire
  77. service-manager runtime-registration path inert in production (PR #30136 review).
  78. """
  79. try:
  80. comm = Path("/proc/1/comm").read_text(encoding="utf-8").strip()
  81. except OSError:
  82. return False
  83. return comm == "s6-svscan" and Path("/run/s6/basedir").is_dir()
  84. # ---------------------------------------------------------------------------
  85. # Host backends: thin facades over ``hermes_cli.gateway`` (systemd/launchd) and
  86. # ``hermes_cli.gateway_windows``. The protocol's ``name`` parameter is unused here — host backends
  87. # operate on the currently active profile (``hermes -p <profile>``); the shape exists for s6 where
  88. # each profile maps to a distinct service directory.
  89. # ---------------------------------------------------------------------------
  90. class _HostServiceManager:
  91. """``start``/``stop``/``restart`` resolve to ``<_fn_prefix><op>`` on ``hermes_cli.<_backend>``
  92. at call time (lazy import; tests monkeypatch the submodule or its functions). Runtime
  93. registration is unsupported on every host backend.
  94. """
  95. kind: ServiceManagerKind
  96. _backend: str
  97. _fn_prefix: str = ""
  98. def _backend_module(self):
  99. import importlib
  100. import hermes_cli
  101. importlib.import_module(f"hermes_cli.{self._backend}")
  102. return getattr(hermes_cli, self._backend)
  103. def _call(self, op: str) -> None:
  104. getattr(self._backend_module(), f"{self._fn_prefix}{op}")()
  105. def start(self, name: str) -> None:
  106. self._call("start")
  107. def stop(self, name: str) -> None:
  108. self._call("stop")
  109. def restart(self, name: str) -> None:
  110. self._call("restart")
  111. def supports_runtime_registration(self) -> bool:
  112. return False
  113. def _unsupported(self, verb: str) -> NotImplementedError:
  114. return NotImplementedError(
  115. f"{type(self).__name__} does not support runtime profile gateway {verb} (container-only feature)"
  116. )
  117. def register_profile_gateway(
  118. self, profile: str, *, extra_env: dict[str, str] | None = None, start_now: bool = True
  119. ) -> None:
  120. raise self._unsupported("registration")
  121. def unregister_profile_gateway(self, profile: str) -> None:
  122. raise self._unsupported("unregistration")
  123. def list_profile_gateways(self) -> list[str]:
  124. return []
  125. class SystemdServiceManager(_HostServiceManager):
  126. """Wraps the ``systemd_*`` functions in hermes_cli.gateway (host call sites still use those
  127. directly; this exists for backend-agnostic code such as the profile create/delete hooks)."""
  128. kind: ServiceManagerKind = "systemd"
  129. _backend = "gateway"
  130. _fn_prefix = "systemd_"
  131. def is_running(self, name: str) -> bool:
  132. _, running = self._backend_module()._probe_systemd_service_running()
  133. return running
  134. class LaunchdServiceManager(_HostServiceManager):
  135. """Wraps the ``launchd_*`` functions in hermes_cli.gateway."""
  136. kind: ServiceManagerKind = "launchd"
  137. _backend = "gateway"
  138. _fn_prefix = "launchd_"
  139. def is_running(self, name: str) -> bool:
  140. return self._backend_module()._probe_launchd_service_running()
  141. class WindowsServiceManager(_HostServiceManager):
  142. """Wraps ``hermes_cli.gateway_windows`` (Scheduled Task / Startup-folder fallback).
  143. Not a true init service but the lifecycle protocol is the same. ``install`` takes
  144. Windows-specific kwargs passed straight through — non-Windows callers must never call it.
  145. """
  146. kind: ServiceManagerKind = "windows"
  147. _backend = "gateway_windows"
  148. def install(
  149. self,
  150. *,
  151. force: bool = False,
  152. start_now: bool | None = None,
  153. start_on_login: bool | None = None,
  154. elevated_handoff: bool = False,
  155. ) -> None:
  156. self._backend_module().install(
  157. force=force, start_now=start_now, start_on_login=start_on_login, elevated_handoff=elevated_handoff
  158. )
  159. def is_running(self, name: str) -> bool:
  160. from hermes_cli.gateway import find_gateway_pids
  161. if not self._backend_module().is_installed():
  162. return False
  163. return bool(find_gateway_pids())
  164. def get_service_manager() -> ServiceManager:
  165. """Return the ServiceManager instance for the current environment."""
  166. cls = _MANAGER_CLASSES.get(detect_service_manager())
  167. if cls is None:
  168. raise RuntimeError("no supported service manager detected")
  169. return cls()
  170. # ---------------------------------------------------------------------------
  171. # S6ServiceManager (container-only). Per-profile gateways are registered dynamically by
  172. # `hermes profile create` inside the container. Static services (main-hermes, dashboard) live in
  173. # /etc/s6-overlay/s6-rc.d/ as part of the image and are NOT managed here.
  174. # ---------------------------------------------------------------------------
  175. # s6-overlay's root scandir, watched by the PID-1 s6-svscan: every slot in it runs under a root
  176. # s6-supervise, so it stays root-owned and holds only the image's static s6-rc services.
  177. S6_STATIC_SCANDIR = Path("/run/service")
  178. # Per-profile gateway scandir (tmpfs), watched by a nested s6-svscan running as hermes (the
  179. # ``profile-gateways`` s6-rc service); writes here trigger supervision on the next rescan.
  180. S6_DYNAMIC_SCANDIR = Path("/run/hermes-service")
  181. S6_SERVICE_PREFIX = "gateway-"
  182. def _profile_from_service(name: str) -> str:
  183. """Strip the ``gateway-`` prefix back off (matches what the user typed via ``-p``)."""
  184. return name.removeprefix(S6_SERVICE_PREFIX)
  185. def _profile_dir_for_gateway_service(name: str) -> Path:
  186. """Resolve ``gateway-<profile>`` to its persistent profile directory.
  187. s6 lifecycle commands may run from any active profile (``gateway stop --all``), so never write
  188. the caller's HERMES_HOME blindly: derive the shared profile root and map the suffix to the root
  189. default profile or ``<root>/profiles/<profile>``.
  190. """
  191. profile = _profile_from_service(name)
  192. validate_profile_name(profile)
  193. hermes_home = Path(os.environ.get("HERMES_HOME", "/opt/data"))
  194. root = hermes_home.parent.parent if hermes_home.parent.name == "profiles" else hermes_home
  195. return root if profile == "default" else root / "profiles" / profile
  196. def _write_gateway_desired_state(name: str, desired_state: str) -> None:
  197. """Persist the operator's start/stop intent (``desired_state``) next to the gateway's volatile
  198. ``gateway_state`` so boot reconciliation can restore s6 want-up/down after pod recreation.
  199. Best-effort: a failed write must not block immediate s6 lifecycle control.
  200. """
  201. profile_dir = _profile_dir_for_gateway_service(name)
  202. state_file = profile_dir / "gateway_state.json"
  203. try:
  204. if not profile_dir.exists():
  205. return
  206. try:
  207. data = json.loads(state_file.read_text(encoding="utf-8-sig")) if state_file.exists() else {}
  208. if not isinstance(data, dict):
  209. data = {}
  210. except (OSError, json.JSONDecodeError):
  211. data = {}
  212. data["desired_state"] = desired_state
  213. data["updated_at"] = int(time.time())
  214. tmp = state_file.with_suffix(state_file.suffix + ".tmp")
  215. tmp.write_text(json.dumps(data, separators=(",", ":")) + "\n", encoding="utf-8")
  216. tmp.replace(state_file)
  217. except OSError:
  218. return
  219. def register_unregistered_profile_gateway(mgr: ServiceManager, profile: str) -> bool:
  220. """Register a ``down`` s6 slot for a profile whose directory exists but was never registered.
  221. `hermes profile create` can only register a slot when it runs inside the container; created
  222. from the host against a bind-mounted home, the directory lands where the container reads it
  223. but no ``/run/hermes-service/gateway-<name>`` exists, and the boot reconciler only notices on the
  224. next container restart. Returns False without touching anything unless the directory carries
  225. ``SOUL.md`` — the reconciler's own "real profile" marker — so a mistyped ``-p`` name cannot
  226. mint a phantom slot. ``start_now=False``: the caller's ordinary ``start`` stays the single
  227. owner of the ``desired_state`` write.
  228. """
  229. profile_dir = _profile_dir_for_gateway_service(f"{S6_SERVICE_PREFIX}{profile}")
  230. if not (profile_dir / "SOUL.md").exists():
  231. return False
  232. mgr.register_profile_gateway(profile, start_now=False)
  233. return True
  234. # s6-overlay installs its binaries under /command/ and only adds it to PATH inside the supervision
  235. # tree. Out-of-tree entry points (``docker exec``, the profile create/delete hooks) inherit the base
  236. # PATH, so every s6 invocation uses this absolute prefix. Not ``/usr/bin/s6-*``: the
  237. # s6-overlay-symlinks-noarch tarball only links a subset.
  238. _S6_BIN_DIR = "/command"
  239. def _s6_run(cmd: str, *args: str, timeout: float = 5, check: bool = False):
  240. """Run an s6 binary by absolute path with the shared capture/decode settings."""
  241. return subprocess.run(
  242. [f"{_S6_BIN_DIR}/{cmd}", *args],
  243. check=check, capture_output=True, text=True, encoding='utf-8', errors='replace',
  244. timeout=timeout,
  245. )
  246. def s6_service_up(service_dir: Path) -> bool:
  247. """``s6-svstat -o up`` for any s6 service directory; False on any failure."""
  248. try:
  249. result = _s6_run("s6-svstat", "-o", "up", str(service_dir))
  250. except (OSError, subprocess.SubprocessError):
  251. return False
  252. return result.returncode == 0 and result.stdout.split() == ["true"]
  253. def _root_hermes_account():
  254. """The current (possibly remapped) hermes account when running as root, else None."""
  255. if not hasattr(os, "geteuid") or os.geteuid() != 0:
  256. return None
  257. import pwd
  258. try:
  259. return pwd.getpwnam("hermes")
  260. except KeyError:
  261. # Outside the image there may be no hermes account; retain the existing owner.
  262. return None
  263. def _chown_hermes(path: Path) -> None:
  264. """Root-created slot entries belong to the current, possibly remapped hermes account."""
  265. account = _root_hermes_account()
  266. if account is not None:
  267. os.chown(path, account.pw_uid, account.pw_gid, follow_symlinks=False)
  268. def _require_root_controlled(path: Path, *, private: bool) -> os.stat_result:
  269. """lstat ``path`` and refuse it unless only root can rename or modify it.
  270. ``private`` additionally requires a 0700-style directory that nobody else can traverse.
  271. """
  272. st = os.lstat(path)
  273. if not stat.S_ISDIR(st.st_mode) or st.st_uid != 0:
  274. raise RuntimeError(f"{path} must be a root-owned directory, not a symlink")
  275. if private and st.st_mode & 0o077:
  276. raise RuntimeError(f"{path} must not be accessible to group or others")
  277. if st.st_mode & 0o022 and not st.st_mode & stat.S_ISVTX:
  278. raise RuntimeError(f"{path} must not be writable by group or others")
  279. return st
  280. def _root_staging_dir(scandir: Path) -> Path:
  281. """Root-private build area beside ``scandir``, on the same filesystem.
  282. The gateway scandir is hermes-owned, so anything root builds inside it can be renamed and
  283. swapped for a symlink mid-build, redirecting root's chmod/chown/writes. Root therefore builds
  284. slots in ``.<scandir>.staging`` (root, 0700) and publishes them with one atomic rename. Every
  285. ancestor must be root-controlled, or an unprivileged user could rename the staging dir away.
  286. """
  287. parent = scandir.parent
  288. for ancestor in (parent, *parent.parents):
  289. _require_root_controlled(ancestor, private=False)
  290. staging = parent / f".{scandir.name}.staging"
  291. try:
  292. os.mkdir(staging, 0o700)
  293. except FileExistsError:
  294. pass
  295. st = _require_root_controlled(staging, private=True)
  296. if os.stat(scandir).st_dev != st.st_dev:
  297. raise RuntimeError(f"{staging} and {scandir} must share a filesystem for an atomic publish")
  298. return staging
  299. def _seed_supervise_skeleton(svc_dir: Path) -> None:
  300. """Pre-create hermes-owned ``supervise/`` and top-level ``event/`` inside a service directory.
  301. s6-supervise creates ``event/``/``supervise/`` 0700 and the control FIFO 0600. Root
  302. registration must seed these for the current hermes account before its unprivileged
  303. supervisor starts. s6 treats EEXIST as success and skips its chown/chmod fix-up, so
  304. seeding before ``s6-svscanctl -a`` makes s6-supervise inherit our
  305. ownership. ``log/`` gets the same skeleton (its own supervise instance) or unregister teardown
  306. EACCESes on the logger. Idempotent: existing entries (possibly live FIFOs) are left untouched.
  307. The PR #30136 review surfaced this as a real product gap: the entire S6ServiceManager lifecycle
  308. (``register/start/stop/unregister _profile_gateway``) was inert in production because every operation is
  309. dispatched as the hermes user.
  310. Reference --------- Discussed at length on the skarnet `skaware` mailing list in 2020
  311. (`<http://skarnet.org/lists/skaware/1424.html>`_); see also just-containers/s6-overlay#130. The
  312. pre-creation pattern was historically called out as forward-compatibility-fragile, but the EEXIST
  313. handling in s6-supervise has been stable since 2015 — it's the same pattern ``s6-svperms`` and
  314. ``fix-attrs.d`` rely on.
  315. """
  316. def _mkdir_owned(path: Path, mode: int) -> None:
  317. if path.exists():
  318. return
  319. path.mkdir(parents=False, exist_ok=False)
  320. path.chmod(mode)
  321. _chown_hermes(path)
  322. def _seed(root: Path) -> None:
  323. # Service-root event/ is the s6-svlisten1 subscription dir, distinct from supervise/event/.
  324. _mkdir_owned(root / "event", 0o3730)
  325. supervise = root / "supervise"
  326. _mkdir_owned(supervise, 0o755)
  327. _mkdir_owned(supervise / "event", 0o3730)
  328. # EEXIST-safe FIFO: if s6-supervise already started against this slot, leave it. The
  329. # explicit chmod is required because mkfifo honors the umask (0022 on dev hosts strips
  330. # group-write → 0o640); stage2 runs umask 0 but be defensive for any invocation context.
  331. control = supervise / "control"
  332. if not control.exists():
  333. os.mkfifo(control, 0o660)
  334. control.chmod(0o660)
  335. _chown_hermes(control)
  336. _seed(svc_dir)
  337. log_dir = svc_dir / "log"
  338. if log_dir.is_dir():
  339. _seed(log_dir)
  340. class S6Error(RuntimeError):
  341. """Base for S6ServiceManager lifecycle failures; carries the slot name so the CLI can render an
  342. actionable message instead of a raw ``CalledProcessError``."""
  343. def __init__(self, message: str, *, service: str | None = None) -> None:
  344. super().__init__(message)
  345. self.service = service
  346. class GatewayNotRegisteredError(S6Error):
  347. """A lifecycle method targeted a missing slot. ``profile`` is unprefixed so callers can phrase
  348. "no such gateway 'typo'"."""
  349. def __init__(self, profile: str) -> None:
  350. self.profile = profile
  351. super().__init__(
  352. f"no such gateway {profile!r}: register it with "
  353. f"`hermes profile create {profile}` first, or pass "
  354. "an existing profile name via `-p <name>`",
  355. service=f"gateway-{profile}",
  356. )
  357. class S6CommandError(S6Error):
  358. """An s6 command failed for a reason other than a missing slot (EACCES on the control FIFO,
  359. unexpected non-zero exit); carries the command's stderr."""
  360. def __init__(self, *, service: str, action: str, returncode: int, stderr: str) -> None:
  361. self.action = action
  362. self.returncode = returncode
  363. self.stderr = stderr
  364. message = f"s6-svc {action} on {service!r} failed (rc={returncode})"
  365. if stderr.strip():
  366. message += f": {stderr.strip()}"
  367. super().__init__(message, service=service)
  368. class S6ServiceManager:
  369. """Per-profile gateway supervision via s6-overlay, for runtime-registered services under
  370. ``S6_DYNAMIC_SCANDIR`` only (static services are managed by s6-rc at image-build time)."""
  371. kind: ServiceManagerKind = "s6"
  372. def __init__(self, scandir: Path = S6_DYNAMIC_SCANDIR) -> None:
  373. self.scandir = scandir
  374. @staticmethod
  375. def _s6(cmd: str, *args: str, timeout: float = 5, check: bool = False):
  376. """Run an s6 tool against the hermes-owned tree, as hermes when called as root.
  377. s6-svc/s6-svscanctl/s6-svwait open FIFOs and fifodirs inside that tree; as root they would
  378. follow a hermes-planted symlink and write to or create entries in any root-owned path.
  379. """
  380. if _root_hermes_account() is not None:
  381. return _s6_run("s6-setuidgid", "hermes", f"{_S6_BIN_DIR}/{cmd}", *args,
  382. timeout=timeout, check=check)
  383. return _s6_run(cmd, *args, timeout=timeout, check=check)
  384. def _service_dir(self, profile: str) -> Path:
  385. validate_profile_name(profile)
  386. return self.scandir / f"{S6_SERVICE_PREFIX}{profile}"
  387. @staticmethod
  388. def _render_run_script(profile: str, extra_env: dict[str, str]) -> str:
  389. """Run script for a profile-gateway s6 service.
  390. Sources HERMES_HOME via with-contenv (run time, not baked in), resets ``HOME`` before the
  391. privilege drop so root's HOME does not leak, activates the venv, drops to hermes.
  392. ``profile == "default"`` emits NO ``-p`` flag: it is the sentinel for the root HERMES_HOME
  393. profile and ``-p default`` would look up ``profiles/default/``. Port comes from the
  394. profile's own env (``API_SERVER_PORT``, default 8642); two profiles that both leave it
  395. unset collide.
  396. Port selection: the gateway binds the port resolved by ``gateway/config.py`` from the profile's own
  397. environment — ``API_SERVER_PORT`` (or ``platforms.api_server.extra.port`` in that profile's
  398. ``config.yaml``), defaulting to 8642. There is no ``[gateway] port`` key and no Python-side
  399. allocator: because each supervised profile gateway loads its own ``HERMES_HOME``, two profiles that
  400. both leave the port unset will both try to bind 8642 — give each profile a distinct
  401. ``API_SERVER_PORT`` in its ``.env``. Previously this method took a ``port`` parameter that was
  402. passed in but never substituted into the rendered script (carried for "API parity" with a
  403. deterministic SHA-256 allocator in ``hermes_cli.profiles._allocate_gateway_port``). PR #30136 review
  404. item I5 retired both the allocator and the parameter because they were dead code through the entire
  405. stack.
  406. """
  407. lines = [
  408. "#!/command/with-contenv sh",
  409. "# shellcheck shell=sh",
  410. "set -e",
  411. "export HOME=/opt/data",
  412. "cd /opt/data",
  413. ]
  414. for k, v in sorted(extra_env.items()):
  415. lines.append(f"export {k}={shlex.quote(v)}")
  416. # Supervised-child sentinel: without it the supervised gateway re-entering
  417. # `_gateway_command_inner` with subcmd == "run" would dispatch `gateway start` → re-exec
  418. # `gateway run --replace` → `gateway start` … (see the matching guard there).
  419. lines.append("export HERMES_S6_SUPERVISED_CHILD=1")
  420. # Generalized supervisor marker — same meaning for the profile-redirect guard in
  421. # hermes_cli.main._apply_profile_override; kept alongside the s6 one for back-compat.
  422. lines.append("export HERMES_SUPERVISED_CHILD=1")
  423. # ``--replace`` makes the supervised gateway authoritative for its HERMES_HOME. Without it
  424. # a gateway started OUTSIDE s6 (stray ``hermes gateway run``, an agent action, the Open
  425. # WebUI helper) grabs the PID lock first; the slot then hits "Another gateway instance is
  426. # already running", exits non-zero, and s6 restarts it forever — a log-flooding loop that
  427. # never binds. ``--replace`` reaps the stale holder (marker + SIGTERM→SIGKILL-with-
  428. # confirmation + scoped-lock cleanup, see gateway/run.py) so s6 always wins; the sentinel
  429. # above prevents the run→start→run recursion. s6 guarantees one supervised instance per
  430. # slot, so there is no legitimate sibling for ``--replace`` to clobber.
  431. if profile == "default":
  432. gateway_cmd = "hermes gateway run --replace"
  433. else:
  434. gateway_cmd = f"hermes -p {shlex.quote(profile)} gateway run --replace"
  435. # Skip the drop when already non-root (setgroups() lacks CAP_SETGID → s6 boot-loop).
  436. lines.append(f'[ "$(id -u)" = 0 ] || exec {gateway_cmd}')
  437. lines.append(f"exec s6-setuidgid hermes {gateway_cmd}")
  438. return "\n".join(lines) + "\n"
  439. @staticmethod
  440. def _render_finish_script() -> str:
  441. """Finish script: exit 78 (EX_CONFIG, fatal config) and clean exit 0 (intentional stop —
  442. restarting would turn every normal exit into a reconnect storm) both map to 125 so s6 stops
  443. restarting; only other non-zero exits let s6 restart normally.
  444. When the gateway exits with EX_CONFIG (78) — a fatal configuration error such as a token collision
  445. or no messaging platforms — we tell s6-supervise to stop restarting by exiting 125 (permanent
  446. failure). A clean exit 0 is an intentional stop, not a crash: restarting after it turns any normal
  447. gateway exit into a reconnect loop (the ashriel-discord storm in #76435 — 1,000+ connections and a
  448. provider token reset). See #51228, #76435.
  449. """
  450. from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE
  451. code = GATEWAY_FATAL_CONFIG_EXIT_CODE
  452. return (
  453. "#!/command/with-contenv sh\n"
  454. "# shellcheck shell=sh\n"
  455. "# $1 = exit code from the run script.\n"
  456. f"# Exit {code} (EX_CONFIG) = fatal config error — don't restart.\n"
  457. "# Exit 0 (clean stop) = intentional stop — don't restart.\n"
  458. f'if [ "$1" = "{code}" ]; then\n'
  459. " exit 125\n"
  460. "fi\n"
  461. 'if [ "$1" = "0" ]; then\n'
  462. " exit 125\n"
  463. "fi\n"
  464. "exit 0\n"
  465. )
  466. @staticmethod
  467. def _render_log_run(profile: str) -> str:
  468. """log/run script. s6-log directives apply per line in order; ``T`` (timestamp) is
  469. non-sticky and only prefixes lines for the next action directive, so it sits between
  470. ``1`` and the log dir rather than before ``1``."""
  471. prof = shlex.quote(profile)
  472. return (
  473. f"#!/command/with-contenv sh\n"
  474. f"# shellcheck shell=sh\n"
  475. f': "${{HERMES_HOME:=/opt/data}}"\n'
  476. f'log_dir="$HERMES_HOME/logs/gateways/{prof}"\n'
  477. # Create the leaf and clear a stale s6-log lock AS HERMES when starting as root. Never
  478. # chown/unlink hermes-writable volume paths from this restartable root-context script:
  479. # an unprivileged user can race a pathname op through a symlink swap (CWE-59/CWE-367).
  480. # Parent logs/gateways is seeded hermes-owned at stage2 boot (test_log_dir_seed.py).
  481. # See #45258.
  482. f'if [ "$(id -u)" = 0 ]; then\n'
  483. f' s6-setuidgid hermes mkdir -p "$log_dir"\n'
  484. f' s6-setuidgid hermes rm -f "$log_dir/lock"\n'
  485. f'else\n'
  486. f' mkdir -p "$log_dir"\n'
  487. f' rm -f "$log_dir/lock"\n'
  488. f'fi\n'
  489. # Skip the drop when already non-root (CAP_SETGID).
  490. f'[ "$(id -u)" = 0 ] || exec s6-log 1 n10 s1000000 T "$log_dir"\n'
  491. f'exec s6-setuidgid hermes s6-log 1 n10 s1000000 T "$log_dir"\n'
  492. )
  493. # -- lifecycle ---------------------------------------------------------
  494. def _run_svc(self, action_flag: str, action_label: str, name: str) -> None:
  495. """``s6-svc <action_flag>``; a missing service dir raises ``GatewayNotRegisteredError``
  496. (instead of s6-svc's opaque failure) and any other failure ``S6CommandError``."""
  497. service_dir = self.scandir / name
  498. if not service_dir.is_dir():
  499. raise GatewayNotRegisteredError(_profile_from_service(name))
  500. try:
  501. self._s6("s6-svc", action_flag, str(service_dir), check=True)
  502. except subprocess.CalledProcessError as exc:
  503. raise S6CommandError(
  504. service=name, action=action_label, returncode=exc.returncode, stderr=exc.stderr or ""
  505. ) from exc
  506. def start(self, name: str) -> None:
  507. self._run_svc("-u", "start", name)
  508. _write_gateway_desired_state(name, "running")
  509. def _svstat_fields(self, name: str, *fields: str) -> list[str] | None:
  510. """``s6-svstat -o <fields>`` values for ``name``, or None on a nonzero rc or a count mismatch.
  511. Never parse the human status line: s6 2.15 prints ``up (pid N pgid N) Ss``, which the old
  512. ``(pid N)`` regex never matched, so the planned-stop marker was silently skipped.
  513. """
  514. result = self._s6("s6-svstat", "-o", ",".join(fields), str(self.scandir / name))
  515. if result.returncode != 0:
  516. return None
  517. values = result.stdout.split()
  518. return values if len(values) == len(fields) else None
  519. def _supervised_pid(self, name: str) -> int | None:
  520. """PID of the supervised gateway per ``s6-svstat -o pid`` (``-1`` when down), or None on any
  521. failure."""
  522. try:
  523. values = self._svstat_fields(name, "pid")
  524. pid = int(values[0]) if values else 0
  525. except (OSError, subprocess.SubprocessError, ValueError):
  526. return None
  527. return pid if pid > 0 else None
  528. def stop(self, name: str) -> None:
  529. """``s6-svc -d``, after writing a planned-stop marker for the supervised PID so the gateway's
  530. shutdown handler classifies this SIGTERM as operator-initiated and persists
  531. ``gateway_state=stopped``."""
  532. pid = self._supervised_pid(name)
  533. if pid is not None:
  534. try:
  535. from gateway.status import write_planned_stop_marker
  536. write_planned_stop_marker(pid)
  537. except Exception:
  538. pass
  539. self._run_svc("-d", "stop", name)
  540. _write_gateway_desired_state(name, "stopped")
  541. def restart(self, name: str) -> None:
  542. """``s6-svc -t`` (SIGTERM)."""
  543. self._run_svc("-t", "restart", name)
  544. _write_gateway_desired_state(name, "running")
  545. def is_running(self, name: str) -> bool:
  546. values = self._svstat_fields(name, "up")
  547. return values == ["true"]
  548. # -- runtime registration ---------------------------------------------
  549. def supports_runtime_registration(self) -> bool:
  550. return True
  551. def register_profile_gateway(
  552. self, profile: str, *, extra_env: dict[str, str] | None = None, start_now: bool = True
  553. ) -> None:
  554. """Create the s6 service directory and ``s6-svscanctl -a`` so it is picked up immediately.
  555. ``start_now=False`` writes a ``down`` marker so the service stays stopped until an explicit
  556. ``gateway start``. Raises ValueError on an invalid name or existing directory, RuntimeError
  557. if ``s6-svscanctl`` fails.
  558. """
  559. svc_dir = self._service_dir(profile)
  560. if os.path.lexists(svc_dir):
  561. raise ValueError(f"profile gateway {profile!r} already registered at {svc_dir}")
  562. # Build atomically in a DOT-PREFIXED sibling (``.gateway-<profile>.tmp``) then rename:
  563. # s6-svscan skips dot entries, so a concurrent rescan (cont-init reconciler, sibling
  564. # register) cannot supervise the half-built slot. Otherwise s6-supervise would spawn AS
  565. # ROOT on the ``.tmp`` (it already has ``type``/``run``), mkdir ``supervise/`` root-owned
  566. # 0700, and our ``_seed_supervise_skeleton`` would EACCES on ``supervise/event`` — the
  567. # arm64-only CI flake on test_s6_unregister_removes_service_dir_in_live_container.
  568. #
  569. # Root never builds inside the hermes-owned scandir: hermes could swap the dot dir for a
  570. # symlink mid-build and redirect root's writes, chmods and chowns. Root builds in a
  571. # root-private staging dir instead and publishes with a single rename.
  572. if _root_hermes_account() is not None:
  573. tmp_dir = Path(tempfile.mkdtemp(prefix=f"{svc_dir.name}.",
  574. dir=_root_staging_dir(self.scandir)))
  575. else:
  576. tmp_dir = svc_dir.with_name("." + svc_dir.name + ".tmp")
  577. if tmp_dir.exists():
  578. shutil.rmtree(tmp_dir, ignore_errors=True)
  579. tmp_dir.mkdir(parents=True)
  580. def _write_script(path: Path, text: str) -> None:
  581. path.write_text(text, encoding="utf-8")
  582. path.chmod(0o755)
  583. try:
  584. (tmp_dir / "type").write_text("longrun\n", encoding="utf-8")
  585. _write_script(tmp_dir / "run", self._render_run_script(profile, extra_env or {}))
  586. _write_script(tmp_dir / "finish", self._render_finish_script())
  587. (tmp_dir / "log").mkdir()
  588. _write_script(tmp_dir / "log" / "run", self._render_log_run(profile))
  589. # Seed hermes-owned supervise/ BEFORE publishing so the hermes-user s6-svc/s6-svstat/
  590. # s6-svwait calls never hit root-owned 0700 dirs (see _seed_supervise_skeleton).
  591. _seed_supervise_skeleton(tmp_dir)
  592. # Mirrors container_boot._register_gateway_slot when start=False.
  593. if not start_now:
  594. (tmp_dir / "down").touch()
  595. # Publish a fully hermes-owned slot, not just its control skeleton. Explicit
  596. # directory modes keep root's restrictive umask from hiding run/log/run.
  597. for path in [tmp_dir, *tmp_dir.rglob("*")]:
  598. if path.is_dir() and path in (tmp_dir, tmp_dir / "log"):
  599. path.chmod(0o755)
  600. _chown_hermes(path)
  601. # rename(2) never follows a symlink at svc_dir and refuses a non-empty directory.
  602. os.rename(tmp_dir, svc_dir)
  603. except Exception:
  604. shutil.rmtree(tmp_dir, ignore_errors=True)
  605. raise
  606. result = self._s6("s6-svscanctl", "-a", str(self.scandir))
  607. if result.returncode != 0:
  608. # No supervisor is watching it — leaving the directory would be confusing.
  609. shutil.rmtree(svc_dir, ignore_errors=True)
  610. raise RuntimeError(f"s6-svscanctl failed: {result.stderr or result.stdout}")
  611. def unregister_profile_gateway(self, profile: str) -> None:
  612. """Stop the profile gateway (best effort, wait for down) and remove its directory.
  613. Idempotent: absent services are a no-op.
  614. ``s6-svscanctl -an`` fires BEFORE ``rmtree`` so s6-svscan reaps the supervise child and
  615. releases its handles on ``supervise/lock``/``status``/``death_tally``; those files are
  616. root-owned but the parent ``supervise/`` is hermes-owned (see ``_seed_supervise_skeleton``)
  617. and POSIX only needs write+execute on the parent to remove them.
  618. """
  619. svc_dir = self._service_dir(profile)
  620. if not svc_dir.exists():
  621. return
  622. self._s6("s6-svc", "-d", str(svc_dir))
  623. self._s6("s6-svwait", "-D", "-t", "10000", str(svc_dir), timeout=15)
  624. self._s6("s6-svscanctl", "-an", str(self.scandir))
  625. # No synchronous "scan completed" handshake — -a/-n just set a flag s6-svscan reads on its
  626. # next loop; 200ms is comfortably above that resolution.
  627. time.sleep(0.2)
  628. shutil.rmtree(svc_dir, ignore_errors=True)
  629. def list_profile_gateways(self) -> list[str]:
  630. """Profile names of all currently-registered gateway services."""
  631. if not self.scandir.exists():
  632. return []
  633. return [
  634. entry.name[len(S6_SERVICE_PREFIX):]
  635. for entry in self.scandir.iterdir()
  636. if not entry.name.startswith(".")
  637. and entry.is_dir()
  638. and entry.name.startswith(S6_SERVICE_PREFIX)
  639. ]
  640. _MANAGER_CLASSES: dict[str, type] = {
  641. "systemd": SystemdServiceManager,
  642. "launchd": LaunchdServiceManager,
  643. "windows": WindowsServiceManager,
  644. "s6": S6ServiceManager,
  645. }