| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773 |
- """Abstract service manager interface + systemd/launchd/Windows/s6 backends."""
- from __future__ import annotations
- import json
- import os
- import re
- import shlex
- import shutil
- import stat
- import subprocess
- import tempfile
- import time
- from pathlib import Path
- from typing import Literal, Protocol, runtime_checkable
- ServiceManagerKind = Literal["systemd", "launchd", "windows", "s6", "none"]
- # Profile names become s6 service directory names (``<scandir>/gateway-<profile>/``), so they
- # must not traverse paths, span filesystems, or break s6's own naming rules.
- _VALID_PROFILE_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
- _MAX_PROFILE_LEN = 251 # s6-svscan default name_max
- def validate_profile_name(name: str) -> None:
- """Raise ValueError unless ``name`` is a filesystem/s6-safe profile name."""
- if not name:
- raise ValueError("profile name must not be empty")
- if len(name) > _MAX_PROFILE_LEN:
- raise ValueError(f"profile name too long ({len(name)} > {_MAX_PROFILE_LEN})")
- if not _VALID_PROFILE_RE.match(name):
- raise ValueError(f"profile name must match [a-z0-9][a-z0-9_-]*, got {name!r}")
- @runtime_checkable
- class ServiceManager(Protocol):
- """Init-system-specific service operations.
- Lifecycle methods exist on every backend. Runtime registration (register/unregister/
- list_profile_gateways) is s6-only — check ``supports_runtime_registration()`` first.
- """
- kind: ServiceManagerKind
- def start(self, name: str) -> None: ...
- def stop(self, name: str) -> None: ...
- def restart(self, name: str) -> None: ...
- def is_running(self, name: str) -> bool: ...
- def supports_runtime_registration(self) -> bool: ...
- def register_profile_gateway(
- self, profile: str, *, extra_env: dict[str, str] | None = None, start_now: bool = True
- ) -> None: ...
- def unregister_profile_gateway(self, profile: str) -> None: ...
- def list_profile_gateways(self) -> list[str]: ...
- def detect_service_manager() -> ServiceManagerKind:
- """Return "s6" (s6-svscan is PID 1), "windows", "launchd", "systemd" (working bus) or "none".
- This function does NOT replace ``supports_systemd_services()`` —
- host call sites continue to use that. It exists for new backend-
- agnostic code (profile create/delete hooks, the s6 dispatch path
- in ``hermes gateway start/stop/restart``).
- """
- # Deferred so importing this module (Protocol type, validate_profile_name) doesn't drag in
- # the whole gateway dependency graph.
- from hermes_cli.gateway import is_macos, is_windows, supports_systemd_services
- # Gate on _s6_running() alone, NOT is_container(): the latter only detects Docker/Podman/lxc
- # and is False on Fly's Firecracker microVMs even though s6-overlay is PID 1 there — that
- # made the s6 dispatch inert on Fly, so `hermes gateway start` spawned a foreground gateway
- # competing with the supervised one.
- if _s6_running():
- return "s6"
- if is_windows():
- return "windows"
- if is_macos():
- return "launchd"
- if supports_systemd_services():
- return "systemd"
- return "none"
- def _s6_running() -> bool:
- """True when s6-svscan is PID 1 in this container.
- Must work for the unprivileged hermes user too: ``/proc/1/exe`` is unreadable for other UIDs
- (``resolve()`` silently yields the literal ``exe``), which made runtime registration inert in
- production. Probe the world-readable ``/proc/1/comm`` AND ``/run/s6/basedir`` — either alone
- can false-positive.
- The obvious probe — ``Path('/proc/1/exe').resolve()`` — only works as root: for any other UID, the
- symlink at ``/proc/1/exe`` is unreadable and ``resolve()`` silently returns the path unchanged, so the
- resolved name is the literal ``"exe"`` and detection always fails. Since every Hermes runtime call
- inside the container drops to hermes via ``s6-setuidgid``, that silent failure made the entire
- service-manager runtime-registration path inert in production (PR #30136 review).
- """
- try:
- comm = Path("/proc/1/comm").read_text(encoding="utf-8").strip()
- except OSError:
- return False
- return comm == "s6-svscan" and Path("/run/s6/basedir").is_dir()
- # ---------------------------------------------------------------------------
- # Host backends: thin facades over ``hermes_cli.gateway`` (systemd/launchd) and
- # ``hermes_cli.gateway_windows``. The protocol's ``name`` parameter is unused here — host backends
- # operate on the currently active profile (``hermes -p <profile>``); the shape exists for s6 where
- # each profile maps to a distinct service directory.
- # ---------------------------------------------------------------------------
- class _HostServiceManager:
- """``start``/``stop``/``restart`` resolve to ``<_fn_prefix><op>`` on ``hermes_cli.<_backend>``
- at call time (lazy import; tests monkeypatch the submodule or its functions). Runtime
- registration is unsupported on every host backend.
- """
- kind: ServiceManagerKind
- _backend: str
- _fn_prefix: str = ""
- def _backend_module(self):
- import importlib
- import hermes_cli
- importlib.import_module(f"hermes_cli.{self._backend}")
- return getattr(hermes_cli, self._backend)
- def _call(self, op: str) -> None:
- getattr(self._backend_module(), f"{self._fn_prefix}{op}")()
- def start(self, name: str) -> None:
- self._call("start")
- def stop(self, name: str) -> None:
- self._call("stop")
- def restart(self, name: str) -> None:
- self._call("restart")
- def supports_runtime_registration(self) -> bool:
- return False
- def _unsupported(self, verb: str) -> NotImplementedError:
- return NotImplementedError(
- f"{type(self).__name__} does not support runtime profile gateway {verb} (container-only feature)"
- )
- def register_profile_gateway(
- self, profile: str, *, extra_env: dict[str, str] | None = None, start_now: bool = True
- ) -> None:
- raise self._unsupported("registration")
- def unregister_profile_gateway(self, profile: str) -> None:
- raise self._unsupported("unregistration")
- def list_profile_gateways(self) -> list[str]:
- return []
- class SystemdServiceManager(_HostServiceManager):
- """Wraps the ``systemd_*`` functions in hermes_cli.gateway (host call sites still use those
- directly; this exists for backend-agnostic code such as the profile create/delete hooks)."""
- kind: ServiceManagerKind = "systemd"
- _backend = "gateway"
- _fn_prefix = "systemd_"
- def is_running(self, name: str) -> bool:
- _, running = self._backend_module()._probe_systemd_service_running()
- return running
- class LaunchdServiceManager(_HostServiceManager):
- """Wraps the ``launchd_*`` functions in hermes_cli.gateway."""
- kind: ServiceManagerKind = "launchd"
- _backend = "gateway"
- _fn_prefix = "launchd_"
- def is_running(self, name: str) -> bool:
- return self._backend_module()._probe_launchd_service_running()
- class WindowsServiceManager(_HostServiceManager):
- """Wraps ``hermes_cli.gateway_windows`` (Scheduled Task / Startup-folder fallback).
- Not a true init service but the lifecycle protocol is the same. ``install`` takes
- Windows-specific kwargs passed straight through — non-Windows callers must never call it.
- """
- kind: ServiceManagerKind = "windows"
- _backend = "gateway_windows"
- def install(
- self,
- *,
- force: bool = False,
- start_now: bool | None = None,
- start_on_login: bool | None = None,
- elevated_handoff: bool = False,
- ) -> None:
- self._backend_module().install(
- force=force, start_now=start_now, start_on_login=start_on_login, elevated_handoff=elevated_handoff
- )
- def is_running(self, name: str) -> bool:
- from hermes_cli.gateway import find_gateway_pids
- if not self._backend_module().is_installed():
- return False
- return bool(find_gateway_pids())
- def get_service_manager() -> ServiceManager:
- """Return the ServiceManager instance for the current environment."""
- cls = _MANAGER_CLASSES.get(detect_service_manager())
- if cls is None:
- raise RuntimeError("no supported service manager detected")
- return cls()
- # ---------------------------------------------------------------------------
- # S6ServiceManager (container-only). Per-profile gateways are registered dynamically by
- # `hermes profile create` inside the container. Static services (main-hermes, dashboard) live in
- # /etc/s6-overlay/s6-rc.d/ as part of the image and are NOT managed here.
- # ---------------------------------------------------------------------------
- # s6-overlay's root scandir, watched by the PID-1 s6-svscan: every slot in it runs under a root
- # s6-supervise, so it stays root-owned and holds only the image's static s6-rc services.
- S6_STATIC_SCANDIR = Path("/run/service")
- # Per-profile gateway scandir (tmpfs), watched by a nested s6-svscan running as hermes (the
- # ``profile-gateways`` s6-rc service); writes here trigger supervision on the next rescan.
- S6_DYNAMIC_SCANDIR = Path("/run/hermes-service")
- S6_SERVICE_PREFIX = "gateway-"
- def _profile_from_service(name: str) -> str:
- """Strip the ``gateway-`` prefix back off (matches what the user typed via ``-p``)."""
- return name.removeprefix(S6_SERVICE_PREFIX)
- def _profile_dir_for_gateway_service(name: str) -> Path:
- """Resolve ``gateway-<profile>`` to its persistent profile directory.
- s6 lifecycle commands may run from any active profile (``gateway stop --all``), so never write
- the caller's HERMES_HOME blindly: derive the shared profile root and map the suffix to the root
- default profile or ``<root>/profiles/<profile>``.
- """
- profile = _profile_from_service(name)
- validate_profile_name(profile)
- hermes_home = Path(os.environ.get("HERMES_HOME", "/opt/data"))
- root = hermes_home.parent.parent if hermes_home.parent.name == "profiles" else hermes_home
- return root if profile == "default" else root / "profiles" / profile
- def _write_gateway_desired_state(name: str, desired_state: str) -> None:
- """Persist the operator's start/stop intent (``desired_state``) next to the gateway's volatile
- ``gateway_state`` so boot reconciliation can restore s6 want-up/down after pod recreation.
- Best-effort: a failed write must not block immediate s6 lifecycle control.
- """
- profile_dir = _profile_dir_for_gateway_service(name)
- state_file = profile_dir / "gateway_state.json"
- try:
- if not profile_dir.exists():
- return
- try:
- data = json.loads(state_file.read_text(encoding="utf-8-sig")) if state_file.exists() else {}
- if not isinstance(data, dict):
- data = {}
- except (OSError, json.JSONDecodeError):
- data = {}
- data["desired_state"] = desired_state
- data["updated_at"] = int(time.time())
- tmp = state_file.with_suffix(state_file.suffix + ".tmp")
- tmp.write_text(json.dumps(data, separators=(",", ":")) + "\n", encoding="utf-8")
- tmp.replace(state_file)
- except OSError:
- return
- def register_unregistered_profile_gateway(mgr: ServiceManager, profile: str) -> bool:
- """Register a ``down`` s6 slot for a profile whose directory exists but was never registered.
- `hermes profile create` can only register a slot when it runs inside the container; created
- from the host against a bind-mounted home, the directory lands where the container reads it
- but no ``/run/hermes-service/gateway-<name>`` exists, and the boot reconciler only notices on the
- next container restart. Returns False without touching anything unless the directory carries
- ``SOUL.md`` — the reconciler's own "real profile" marker — so a mistyped ``-p`` name cannot
- mint a phantom slot. ``start_now=False``: the caller's ordinary ``start`` stays the single
- owner of the ``desired_state`` write.
- """
- profile_dir = _profile_dir_for_gateway_service(f"{S6_SERVICE_PREFIX}{profile}")
- if not (profile_dir / "SOUL.md").exists():
- return False
- mgr.register_profile_gateway(profile, start_now=False)
- return True
- # s6-overlay installs its binaries under /command/ and only adds it to PATH inside the supervision
- # tree. Out-of-tree entry points (``docker exec``, the profile create/delete hooks) inherit the base
- # PATH, so every s6 invocation uses this absolute prefix. Not ``/usr/bin/s6-*``: the
- # s6-overlay-symlinks-noarch tarball only links a subset.
- _S6_BIN_DIR = "/command"
- def _s6_run(cmd: str, *args: str, timeout: float = 5, check: bool = False):
- """Run an s6 binary by absolute path with the shared capture/decode settings."""
- return subprocess.run(
- [f"{_S6_BIN_DIR}/{cmd}", *args],
- check=check, capture_output=True, text=True, encoding='utf-8', errors='replace',
- timeout=timeout,
- )
- def s6_service_up(service_dir: Path) -> bool:
- """``s6-svstat -o up`` for any s6 service directory; False on any failure."""
- try:
- result = _s6_run("s6-svstat", "-o", "up", str(service_dir))
- except (OSError, subprocess.SubprocessError):
- return False
- return result.returncode == 0 and result.stdout.split() == ["true"]
- def _root_hermes_account():
- """The current (possibly remapped) hermes account when running as root, else None."""
- if not hasattr(os, "geteuid") or os.geteuid() != 0:
- return None
- import pwd
- try:
- return pwd.getpwnam("hermes")
- except KeyError:
- # Outside the image there may be no hermes account; retain the existing owner.
- return None
- def _chown_hermes(path: Path) -> None:
- """Root-created slot entries belong to the current, possibly remapped hermes account."""
- account = _root_hermes_account()
- if account is not None:
- os.chown(path, account.pw_uid, account.pw_gid, follow_symlinks=False)
- def _require_root_controlled(path: Path, *, private: bool) -> os.stat_result:
- """lstat ``path`` and refuse it unless only root can rename or modify it.
- ``private`` additionally requires a 0700-style directory that nobody else can traverse.
- """
- st = os.lstat(path)
- if not stat.S_ISDIR(st.st_mode) or st.st_uid != 0:
- raise RuntimeError(f"{path} must be a root-owned directory, not a symlink")
- if private and st.st_mode & 0o077:
- raise RuntimeError(f"{path} must not be accessible to group or others")
- if st.st_mode & 0o022 and not st.st_mode & stat.S_ISVTX:
- raise RuntimeError(f"{path} must not be writable by group or others")
- return st
- def _root_staging_dir(scandir: Path) -> Path:
- """Root-private build area beside ``scandir``, on the same filesystem.
- The gateway scandir is hermes-owned, so anything root builds inside it can be renamed and
- swapped for a symlink mid-build, redirecting root's chmod/chown/writes. Root therefore builds
- slots in ``.<scandir>.staging`` (root, 0700) and publishes them with one atomic rename. Every
- ancestor must be root-controlled, or an unprivileged user could rename the staging dir away.
- """
- parent = scandir.parent
- for ancestor in (parent, *parent.parents):
- _require_root_controlled(ancestor, private=False)
- staging = parent / f".{scandir.name}.staging"
- try:
- os.mkdir(staging, 0o700)
- except FileExistsError:
- pass
- st = _require_root_controlled(staging, private=True)
- if os.stat(scandir).st_dev != st.st_dev:
- raise RuntimeError(f"{staging} and {scandir} must share a filesystem for an atomic publish")
- return staging
- def _seed_supervise_skeleton(svc_dir: Path) -> None:
- """Pre-create hermes-owned ``supervise/`` and top-level ``event/`` inside a service directory.
- s6-supervise creates ``event/``/``supervise/`` 0700 and the control FIFO 0600. Root
- registration must seed these for the current hermes account before its unprivileged
- supervisor starts. s6 treats EEXIST as success and skips its chown/chmod fix-up, so
- seeding before ``s6-svscanctl -a`` makes s6-supervise inherit our
- ownership. ``log/`` gets the same skeleton (its own supervise instance) or unregister teardown
- EACCESes on the logger. Idempotent: existing entries (possibly live FIFOs) are left untouched.
- The PR #30136 review surfaced this as a real product gap: the entire S6ServiceManager lifecycle
- (``register/start/stop/unregister _profile_gateway``) was inert in production because every operation is
- dispatched as the hermes user.
- Reference --------- Discussed at length on the skarnet `skaware` mailing list in 2020
- (`<http://skarnet.org/lists/skaware/1424.html>`_); see also just-containers/s6-overlay#130. The
- pre-creation pattern was historically called out as forward-compatibility-fragile, but the EEXIST
- handling in s6-supervise has been stable since 2015 — it's the same pattern ``s6-svperms`` and
- ``fix-attrs.d`` rely on.
- """
- def _mkdir_owned(path: Path, mode: int) -> None:
- if path.exists():
- return
- path.mkdir(parents=False, exist_ok=False)
- path.chmod(mode)
- _chown_hermes(path)
- def _seed(root: Path) -> None:
- # Service-root event/ is the s6-svlisten1 subscription dir, distinct from supervise/event/.
- _mkdir_owned(root / "event", 0o3730)
- supervise = root / "supervise"
- _mkdir_owned(supervise, 0o755)
- _mkdir_owned(supervise / "event", 0o3730)
- # EEXIST-safe FIFO: if s6-supervise already started against this slot, leave it. The
- # explicit chmod is required because mkfifo honors the umask (0022 on dev hosts strips
- # group-write → 0o640); stage2 runs umask 0 but be defensive for any invocation context.
- control = supervise / "control"
- if not control.exists():
- os.mkfifo(control, 0o660)
- control.chmod(0o660)
- _chown_hermes(control)
- _seed(svc_dir)
- log_dir = svc_dir / "log"
- if log_dir.is_dir():
- _seed(log_dir)
- class S6Error(RuntimeError):
- """Base for S6ServiceManager lifecycle failures; carries the slot name so the CLI can render an
- actionable message instead of a raw ``CalledProcessError``."""
- def __init__(self, message: str, *, service: str | None = None) -> None:
- super().__init__(message)
- self.service = service
- class GatewayNotRegisteredError(S6Error):
- """A lifecycle method targeted a missing slot. ``profile`` is unprefixed so callers can phrase
- "no such gateway 'typo'"."""
- def __init__(self, profile: str) -> None:
- self.profile = profile
- super().__init__(
- f"no such gateway {profile!r}: register it with "
- f"`hermes profile create {profile}` first, or pass "
- "an existing profile name via `-p <name>`",
- service=f"gateway-{profile}",
- )
- class S6CommandError(S6Error):
- """An s6 command failed for a reason other than a missing slot (EACCES on the control FIFO,
- unexpected non-zero exit); carries the command's stderr."""
- def __init__(self, *, service: str, action: str, returncode: int, stderr: str) -> None:
- self.action = action
- self.returncode = returncode
- self.stderr = stderr
- message = f"s6-svc {action} on {service!r} failed (rc={returncode})"
- if stderr.strip():
- message += f": {stderr.strip()}"
- super().__init__(message, service=service)
- class S6ServiceManager:
- """Per-profile gateway supervision via s6-overlay, for runtime-registered services under
- ``S6_DYNAMIC_SCANDIR`` only (static services are managed by s6-rc at image-build time)."""
- kind: ServiceManagerKind = "s6"
- def __init__(self, scandir: Path = S6_DYNAMIC_SCANDIR) -> None:
- self.scandir = scandir
- @staticmethod
- def _s6(cmd: str, *args: str, timeout: float = 5, check: bool = False):
- """Run an s6 tool against the hermes-owned tree, as hermes when called as root.
- s6-svc/s6-svscanctl/s6-svwait open FIFOs and fifodirs inside that tree; as root they would
- follow a hermes-planted symlink and write to or create entries in any root-owned path.
- """
- if _root_hermes_account() is not None:
- return _s6_run("s6-setuidgid", "hermes", f"{_S6_BIN_DIR}/{cmd}", *args,
- timeout=timeout, check=check)
- return _s6_run(cmd, *args, timeout=timeout, check=check)
- def _service_dir(self, profile: str) -> Path:
- validate_profile_name(profile)
- return self.scandir / f"{S6_SERVICE_PREFIX}{profile}"
- @staticmethod
- def _render_run_script(profile: str, extra_env: dict[str, str]) -> str:
- """Run script for a profile-gateway s6 service.
- Sources HERMES_HOME via with-contenv (run time, not baked in), resets ``HOME`` before the
- privilege drop so root's HOME does not leak, activates the venv, drops to hermes.
- ``profile == "default"`` emits NO ``-p`` flag: it is the sentinel for the root HERMES_HOME
- profile and ``-p default`` would look up ``profiles/default/``. Port comes from the
- profile's own env (``API_SERVER_PORT``, default 8642); two profiles that both leave it
- unset collide.
- Port selection: the gateway binds the port resolved by ``gateway/config.py`` from the profile's own
- environment — ``API_SERVER_PORT`` (or ``platforms.api_server.extra.port`` in that profile's
- ``config.yaml``), defaulting to 8642. There is no ``[gateway] port`` key and no Python-side
- allocator: because each supervised profile gateway loads its own ``HERMES_HOME``, two profiles that
- both leave the port unset will both try to bind 8642 — give each profile a distinct
- ``API_SERVER_PORT`` in its ``.env``. Previously this method took a ``port`` parameter that was
- passed in but never substituted into the rendered script (carried for "API parity" with a
- deterministic SHA-256 allocator in ``hermes_cli.profiles._allocate_gateway_port``). PR #30136 review
- item I5 retired both the allocator and the parameter because they were dead code through the entire
- stack.
- """
- lines = [
- "#!/command/with-contenv sh",
- "# shellcheck shell=sh",
- "set -e",
- "export HOME=/opt/data",
- "cd /opt/data",
- ]
- for k, v in sorted(extra_env.items()):
- lines.append(f"export {k}={shlex.quote(v)}")
- # Supervised-child sentinel: without it the supervised gateway re-entering
- # `_gateway_command_inner` with subcmd == "run" would dispatch `gateway start` → re-exec
- # `gateway run --replace` → `gateway start` … (see the matching guard there).
- lines.append("export HERMES_S6_SUPERVISED_CHILD=1")
- # Generalized supervisor marker — same meaning for the profile-redirect guard in
- # hermes_cli.main._apply_profile_override; kept alongside the s6 one for back-compat.
- lines.append("export HERMES_SUPERVISED_CHILD=1")
- # ``--replace`` makes the supervised gateway authoritative for its HERMES_HOME. Without it
- # a gateway started OUTSIDE s6 (stray ``hermes gateway run``, an agent action, the Open
- # WebUI helper) grabs the PID lock first; the slot then hits "Another gateway instance is
- # already running", exits non-zero, and s6 restarts it forever — a log-flooding loop that
- # never binds. ``--replace`` reaps the stale holder (marker + SIGTERM→SIGKILL-with-
- # confirmation + scoped-lock cleanup, see gateway/run.py) so s6 always wins; the sentinel
- # above prevents the run→start→run recursion. s6 guarantees one supervised instance per
- # slot, so there is no legitimate sibling for ``--replace`` to clobber.
- if profile == "default":
- gateway_cmd = "hermes gateway run --replace"
- else:
- gateway_cmd = f"hermes -p {shlex.quote(profile)} gateway run --replace"
- # Skip the drop when already non-root (setgroups() lacks CAP_SETGID → s6 boot-loop).
- lines.append(f'[ "$(id -u)" = 0 ] || exec {gateway_cmd}')
- lines.append(f"exec s6-setuidgid hermes {gateway_cmd}")
- return "\n".join(lines) + "\n"
- @staticmethod
- def _render_finish_script() -> str:
- """Finish script: exit 78 (EX_CONFIG, fatal config) and clean exit 0 (intentional stop —
- restarting would turn every normal exit into a reconnect storm) both map to 125 so s6 stops
- restarting; only other non-zero exits let s6 restart normally.
- When the gateway exits with EX_CONFIG (78) — a fatal configuration error such as a token collision
- or no messaging platforms — we tell s6-supervise to stop restarting by exiting 125 (permanent
- failure). A clean exit 0 is an intentional stop, not a crash: restarting after it turns any normal
- gateway exit into a reconnect loop (the ashriel-discord storm in #76435 — 1,000+ connections and a
- provider token reset). See #51228, #76435.
- """
- from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE
- code = GATEWAY_FATAL_CONFIG_EXIT_CODE
- return (
- "#!/command/with-contenv sh\n"
- "# shellcheck shell=sh\n"
- "# $1 = exit code from the run script.\n"
- f"# Exit {code} (EX_CONFIG) = fatal config error — don't restart.\n"
- "# Exit 0 (clean stop) = intentional stop — don't restart.\n"
- f'if [ "$1" = "{code}" ]; then\n'
- " exit 125\n"
- "fi\n"
- 'if [ "$1" = "0" ]; then\n'
- " exit 125\n"
- "fi\n"
- "exit 0\n"
- )
- @staticmethod
- def _render_log_run(profile: str) -> str:
- """log/run script. s6-log directives apply per line in order; ``T`` (timestamp) is
- non-sticky and only prefixes lines for the next action directive, so it sits between
- ``1`` and the log dir rather than before ``1``."""
- prof = shlex.quote(profile)
- return (
- f"#!/command/with-contenv sh\n"
- f"# shellcheck shell=sh\n"
- f': "${{HERMES_HOME:=/opt/data}}"\n'
- f'log_dir="$HERMES_HOME/logs/gateways/{prof}"\n'
- # Create the leaf and clear a stale s6-log lock AS HERMES when starting as root. Never
- # chown/unlink hermes-writable volume paths from this restartable root-context script:
- # an unprivileged user can race a pathname op through a symlink swap (CWE-59/CWE-367).
- # Parent logs/gateways is seeded hermes-owned at stage2 boot (test_log_dir_seed.py).
- # See #45258.
- f'if [ "$(id -u)" = 0 ]; then\n'
- f' s6-setuidgid hermes mkdir -p "$log_dir"\n'
- f' s6-setuidgid hermes rm -f "$log_dir/lock"\n'
- f'else\n'
- f' mkdir -p "$log_dir"\n'
- f' rm -f "$log_dir/lock"\n'
- f'fi\n'
- # Skip the drop when already non-root (CAP_SETGID).
- f'[ "$(id -u)" = 0 ] || exec s6-log 1 n10 s1000000 T "$log_dir"\n'
- f'exec s6-setuidgid hermes s6-log 1 n10 s1000000 T "$log_dir"\n'
- )
- # -- lifecycle ---------------------------------------------------------
- def _run_svc(self, action_flag: str, action_label: str, name: str) -> None:
- """``s6-svc <action_flag>``; a missing service dir raises ``GatewayNotRegisteredError``
- (instead of s6-svc's opaque failure) and any other failure ``S6CommandError``."""
- service_dir = self.scandir / name
- if not service_dir.is_dir():
- raise GatewayNotRegisteredError(_profile_from_service(name))
- try:
- self._s6("s6-svc", action_flag, str(service_dir), check=True)
- except subprocess.CalledProcessError as exc:
- raise S6CommandError(
- service=name, action=action_label, returncode=exc.returncode, stderr=exc.stderr or ""
- ) from exc
- def start(self, name: str) -> None:
- self._run_svc("-u", "start", name)
- _write_gateway_desired_state(name, "running")
- def _svstat_fields(self, name: str, *fields: str) -> list[str] | None:
- """``s6-svstat -o <fields>`` values for ``name``, or None on a nonzero rc or a count mismatch.
- Never parse the human status line: s6 2.15 prints ``up (pid N pgid N) Ss``, which the old
- ``(pid N)`` regex never matched, so the planned-stop marker was silently skipped.
- """
- result = self._s6("s6-svstat", "-o", ",".join(fields), str(self.scandir / name))
- if result.returncode != 0:
- return None
- values = result.stdout.split()
- return values if len(values) == len(fields) else None
- def _supervised_pid(self, name: str) -> int | None:
- """PID of the supervised gateway per ``s6-svstat -o pid`` (``-1`` when down), or None on any
- failure."""
- try:
- values = self._svstat_fields(name, "pid")
- pid = int(values[0]) if values else 0
- except (OSError, subprocess.SubprocessError, ValueError):
- return None
- return pid if pid > 0 else None
- def stop(self, name: str) -> None:
- """``s6-svc -d``, after writing a planned-stop marker for the supervised PID so the gateway's
- shutdown handler classifies this SIGTERM as operator-initiated and persists
- ``gateway_state=stopped``."""
- pid = self._supervised_pid(name)
- if pid is not None:
- try:
- from gateway.status import write_planned_stop_marker
- write_planned_stop_marker(pid)
- except Exception:
- pass
- self._run_svc("-d", "stop", name)
- _write_gateway_desired_state(name, "stopped")
- def restart(self, name: str) -> None:
- """``s6-svc -t`` (SIGTERM)."""
- self._run_svc("-t", "restart", name)
- _write_gateway_desired_state(name, "running")
- def is_running(self, name: str) -> bool:
- values = self._svstat_fields(name, "up")
- return values == ["true"]
- # -- runtime registration ---------------------------------------------
- def supports_runtime_registration(self) -> bool:
- return True
- def register_profile_gateway(
- self, profile: str, *, extra_env: dict[str, str] | None = None, start_now: bool = True
- ) -> None:
- """Create the s6 service directory and ``s6-svscanctl -a`` so it is picked up immediately.
- ``start_now=False`` writes a ``down`` marker so the service stays stopped until an explicit
- ``gateway start``. Raises ValueError on an invalid name or existing directory, RuntimeError
- if ``s6-svscanctl`` fails.
- """
- svc_dir = self._service_dir(profile)
- if os.path.lexists(svc_dir):
- raise ValueError(f"profile gateway {profile!r} already registered at {svc_dir}")
- # Build atomically in a DOT-PREFIXED sibling (``.gateway-<profile>.tmp``) then rename:
- # s6-svscan skips dot entries, so a concurrent rescan (cont-init reconciler, sibling
- # register) cannot supervise the half-built slot. Otherwise s6-supervise would spawn AS
- # ROOT on the ``.tmp`` (it already has ``type``/``run``), mkdir ``supervise/`` root-owned
- # 0700, and our ``_seed_supervise_skeleton`` would EACCES on ``supervise/event`` — the
- # arm64-only CI flake on test_s6_unregister_removes_service_dir_in_live_container.
- #
- # Root never builds inside the hermes-owned scandir: hermes could swap the dot dir for a
- # symlink mid-build and redirect root's writes, chmods and chowns. Root builds in a
- # root-private staging dir instead and publishes with a single rename.
- if _root_hermes_account() is not None:
- tmp_dir = Path(tempfile.mkdtemp(prefix=f"{svc_dir.name}.",
- dir=_root_staging_dir(self.scandir)))
- else:
- tmp_dir = svc_dir.with_name("." + svc_dir.name + ".tmp")
- if tmp_dir.exists():
- shutil.rmtree(tmp_dir, ignore_errors=True)
- tmp_dir.mkdir(parents=True)
- def _write_script(path: Path, text: str) -> None:
- path.write_text(text, encoding="utf-8")
- path.chmod(0o755)
- try:
- (tmp_dir / "type").write_text("longrun\n", encoding="utf-8")
- _write_script(tmp_dir / "run", self._render_run_script(profile, extra_env or {}))
- _write_script(tmp_dir / "finish", self._render_finish_script())
- (tmp_dir / "log").mkdir()
- _write_script(tmp_dir / "log" / "run", self._render_log_run(profile))
- # Seed hermes-owned supervise/ BEFORE publishing so the hermes-user s6-svc/s6-svstat/
- # s6-svwait calls never hit root-owned 0700 dirs (see _seed_supervise_skeleton).
- _seed_supervise_skeleton(tmp_dir)
- # Mirrors container_boot._register_gateway_slot when start=False.
- if not start_now:
- (tmp_dir / "down").touch()
- # Publish a fully hermes-owned slot, not just its control skeleton. Explicit
- # directory modes keep root's restrictive umask from hiding run/log/run.
- for path in [tmp_dir, *tmp_dir.rglob("*")]:
- if path.is_dir() and path in (tmp_dir, tmp_dir / "log"):
- path.chmod(0o755)
- _chown_hermes(path)
- # rename(2) never follows a symlink at svc_dir and refuses a non-empty directory.
- os.rename(tmp_dir, svc_dir)
- except Exception:
- shutil.rmtree(tmp_dir, ignore_errors=True)
- raise
- result = self._s6("s6-svscanctl", "-a", str(self.scandir))
- if result.returncode != 0:
- # No supervisor is watching it — leaving the directory would be confusing.
- shutil.rmtree(svc_dir, ignore_errors=True)
- raise RuntimeError(f"s6-svscanctl failed: {result.stderr or result.stdout}")
- def unregister_profile_gateway(self, profile: str) -> None:
- """Stop the profile gateway (best effort, wait for down) and remove its directory.
- Idempotent: absent services are a no-op.
- ``s6-svscanctl -an`` fires BEFORE ``rmtree`` so s6-svscan reaps the supervise child and
- releases its handles on ``supervise/lock``/``status``/``death_tally``; those files are
- root-owned but the parent ``supervise/`` is hermes-owned (see ``_seed_supervise_skeleton``)
- and POSIX only needs write+execute on the parent to remove them.
- """
- svc_dir = self._service_dir(profile)
- if not svc_dir.exists():
- return
- self._s6("s6-svc", "-d", str(svc_dir))
- self._s6("s6-svwait", "-D", "-t", "10000", str(svc_dir), timeout=15)
- self._s6("s6-svscanctl", "-an", str(self.scandir))
- # No synchronous "scan completed" handshake — -a/-n just set a flag s6-svscan reads on its
- # next loop; 200ms is comfortably above that resolution.
- time.sleep(0.2)
- shutil.rmtree(svc_dir, ignore_errors=True)
- def list_profile_gateways(self) -> list[str]:
- """Profile names of all currently-registered gateway services."""
- if not self.scandir.exists():
- return []
- return [
- entry.name[len(S6_SERVICE_PREFIX):]
- for entry in self.scandir.iterdir()
- if not entry.name.startswith(".")
- and entry.is_dir()
- and entry.name.startswith(S6_SERVICE_PREFIX)
- ]
- _MANAGER_CLASSES: dict[str, type] = {
- "systemd": SystemdServiceManager,
- "launchd": LaunchdServiceManager,
- "windows": WindowsServiceManager,
- "s6": S6ServiceManager,
- }
|