container_boot.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378
  1. """Container-boot reconciliation of per-profile gateway s6 services.
  2. Wired into the image as /etc/cont-init.d/02-reconcile-profiles. Runs as root after
  3. 01-hermes-setup (the stage2 hook) has chowned the volume and seeded $HERMES_HOME, but
  4. before s6-rc starts user services.
  5. """
  6. from __future__ import annotations
  7. import json
  8. import logging
  9. import os
  10. from dataclasses import dataclass
  11. from pathlib import Path
  12. from typing import Literal, Sequence
  13. from hermes_cli.gateway_multiplex_s6 import AUTOSTART_STATES as _AUTOSTART_STATES, fold_named_slot_intent
  14. log = logging.getLogger(__name__)
  15. # Older installs only have gateway_state; newer lifecycle commands persist desired_state separately
  16. # so a transient runtime state can't erase operator intent.
  17. # Transient sub-states of a RUNNING gateway (not an operator stop, not a failed boot). A gateway
  18. # hard-killed in one of them with no `desired_state` would otherwise stay DOWN on every later boot
  19. # (observed: staging stranded at `draining`); map them to `running`, mirroring gateway/run.py.
  20. # `starting` / `startup_failed` are excluded: auto-restarting a mid-boot death is the crash-loop.
  21. # A gateway only ever reaches these while it is up and serving, so they are NOT an operator stop and NOT a
  22. # failed boot: - `draining` — written by the drain watcher / scale-to-zero go-dormant path when an
  23. # in-flight quiesce begins (gateway/run.py). - `degraded` — written when the gateway comes up with some
  24. # platforms queued for retry, then "falls through to the normal running state" (gateway/run.py #5196): the
  25. # process is up, serving cron + whatever platforms connected, and the reconnect watcher takes the rest from
  26. # there. When a gateway is hard-killed *while in one of these states* (a container/VM recreate SIGTERMs it
  27. # before `_stop_impl` reaches its terminal-state persist), the last value left in gateway_state.json is the
  28. # transient sub-state. With no explicit `desired_state` to fall back to, treating that literal value as the
  29. # autostart intent would leave the gateway DOWN on every subsequent boot — the gateway never comes back, the
  30. # dashboard is up but messaging stays dark (observed on a relay-opted-in staging instance stranded at
  31. # `draining`, 2026-06; `degraded` is the same wedge class). Map these transient sub-states to `running` so a
  32. # stranded marker reads as the run-intent it actually represents. This mirrors gateway/run.py's #42675
  33. # handling, which persists `running` (not the mid-shutdown `draining`) when an unexpected signal tears the
  34. # gateway down — extended here to the case where the gateway died before it could persist anything at all.
  35. _TRANSIENT_RUNNING_STATES = frozenset({"draining", "degraded"})
  36. # Container-namespaced state is garbage post-restart (an equal PID is a different process).
  37. _STALE_RUNTIME_FILES = ("gateway.pid", "processes.json")
  38. ReconcileActionLabel = Literal["started", "registered", "skipped"]
  39. @dataclass(frozen=True)
  40. class ReconcileAction:
  41. """One profile's outcome from a single reconciliation pass."""
  42. profile: str
  43. prior_state: str | None
  44. action: ReconcileActionLabel
  45. # "clean" / "unclean" (sentinel still says running — SIGKILL/OOM/VM death) / "unknown". Boot
  46. # is the one place that can stamp a violent death into the volume-persisted log.
  47. prior_exit: str = "unknown"
  48. #: A named slot whose autostart intent was inherited by the root slot (multiplex-only), or —
  49. #: on the root slot — True when it is starting because of one.
  50. folded_into_root: bool = False
  51. def _slot_action(profile: str, profile_dir: Path, prior_state: str | None, start: bool,
  52. *, folded_into_root: bool = False) -> ReconcileAction:
  53. return ReconcileAction(profile=profile, prior_state=prior_state,
  54. action="started" if start else "registered",
  55. prior_exit=_read_prior_exit_label(profile_dir),
  56. folded_into_root=folded_into_root)
  57. def _named_profile_dirs(hermes_home: Path) -> list[tuple[str, Path]]:
  58. """Every real named profile under ``$HERMES_HOME/profiles`` (``SOUL.md`` is the marker)."""
  59. profiles_root = hermes_home / "profiles"
  60. if not profiles_root.is_dir():
  61. return []
  62. found: list[tuple[str, Path]] = []
  63. for entry in sorted(profiles_root.iterdir()):
  64. if not entry.is_dir() or not (entry / "SOUL.md").exists():
  65. continue
  66. # "default" is reserved for the root profile slot.
  67. if entry.name == "default":
  68. log.warning("profiles/default/ exists — skipping to avoid colliding with the "
  69. "reserved root-profile s6 slot")
  70. continue
  71. found.append((entry.name, entry))
  72. return found
  73. def reconcile_profile_gateways(
  74. *, hermes_home: Path, scandir: Path, dry_run: bool = False,
  75. container_argv: Sequence[str] | None = None) -> list[ReconcileAction]:
  76. """Recreate s6 service registrations for every persistent profile.
  77. Always registers a ``gateway-default`` slot for the root profile (the implicit profile at
  78. the top of ``$HERMES_HOME``): ``hermes_cli.gateway`` maps an empty profile suffix to it,
  79. so it is what ``hermes gateway start`` (no ``-p``) targets.
  80. Without it, bare ``hermes gateway start`` inside the container would land on ``s6-svc -u
  81. /run/hermes-service/gateway-default`` → uncaught ``CalledProcessError`` → traceback to the user (PR #30136
  82. review).
  83. """
  84. actions: list[ReconcileAction] = []
  85. # ONE multiplexing gateway per container: named slots are registered (so `hermes -p X gateway
  86. # start` has a target and `s6-svstat` can report them) but are never booted from their persisted
  87. # run intent, except a `gateway.standalone` profile's (below).
  88. # This is the s6 leg of the multiplex-only convergence: an image upgraded from a release that
  89. # booted N per-profile slots comes back up with one multiplexing root gateway and no manual
  90. # step, instead of N processes fighting over the same profiles.
  91. #
  92. # It used to be gated on `gateway.multiplex_profiles`, which made the UNSET default (on) boot
  93. # the slots anyway — the container shipped the opt-out topology by accident. The key is retired
  94. # as a topology switch (hermes_cli/gateway_multiplex_mode.py), so there is nothing to read.
  95. named = [(name, entry, _read_desired_state(entry)) for name, entry in _named_profile_dirs(hermes_home)]
  96. # The exception is a `gateway.standalone` profile: the root multiplexer never serves it
  97. # (profiles_to_serve excludes it), so it boots its own slot from its own intent, beside the root.
  98. from hermes_cli.profiles import profile_is_standalone
  99. standalone = {name for name, entry, _prior in named if profile_is_standalone(entry)}
  100. # A legacy `gateway run` container with no state yet seeds `running` (pre-s6 behavior).
  101. legacy_default_state = _maybe_migrate_legacy_gateway_run_state(
  102. hermes_home, container_argv=container_argv, dry_run=dry_run)
  103. default_prior_state = legacy_default_state or _read_desired_state(hermes_home)
  104. # The root slot INHERITS every named slot's autostart intent, because it is the one process
  105. # that serves them. Without this an image only ever driven as `hermes -p coder gateway start`
  106. # booted with ZERO gateways: it has no root state (or "stopped"), every named slot is now
  107. # registered down unconditionally, and every action reported "registered" — a container that
  108. # looks healthy while nothing is listening.
  109. fold = fold_named_slot_intent(
  110. default_prior_state, ((name, prior) for name, _dir, prior in named if name not in standalone))
  111. folded, default_should_start = list(fold.folded), fold.root_should_start
  112. if folded and default_prior_state not in _AUTOSTART_STATES:
  113. log.warning("%s", boot_notice(folded))
  114. if not dry_run:
  115. _cleanup_stale_runtime_files(hermes_home)
  116. _register_service(scandir, "default", start=default_should_start)
  117. actions.append(_slot_action("default", hermes_home, default_prior_state, default_should_start,
  118. folded_into_root=bool(folded)))
  119. for name, entry, prior_state in named:
  120. # Registered down unless standalone: any other started named slot IS a second gateway on this host.
  121. should_start = name in standalone and prior_state in _AUTOSTART_STATES
  122. if not dry_run:
  123. _cleanup_stale_runtime_files(entry)
  124. _register_service(scandir, name, start=should_start)
  125. actions.append(_slot_action(name, entry, prior_state, should_start,
  126. folded_into_root=name in folded))
  127. if not dry_run:
  128. _write_reconcile_log(hermes_home, actions)
  129. return actions
  130. def boot_notice(folded: Sequence[str]) -> str:
  131. """One line naming the profiles whose autostart intent the root slot took over."""
  132. return ("reconcile: profile gateway(s) " + ", ".join(sorted(folded)) +
  133. " asked to autostart; one gateway per container serves every non-standalone profile, so the ROOT "
  134. "slot (gateway-default) was started instead and multiplexes them.")
  135. def _maybe_migrate_legacy_gateway_run_state(
  136. hermes_home: Path, *, container_argv: Sequence[str] | None, dry_run: bool) -> str | None:
  137. """Seed root gateway_state for pre-s6 `gateway run` containers (the tini image let users run
  138. the gateway as the container command; post-s6 it would register down and never start)."""
  139. state_file = hermes_home / "gateway_state.json"
  140. if state_file.exists():
  141. return None
  142. if os.environ.get("HERMES_GATEWAY_NO_SUPERVISE", "").lower() in ("1", "true", "yes"):
  143. return None
  144. argv = tuple(container_argv) if container_argv is not None else _read_container_argv()
  145. if not _is_legacy_gateway_run_request(argv):
  146. return None
  147. if not dry_run:
  148. import time
  149. state_file.write_text(json.dumps({
  150. "gateway_state": "running",
  151. "desired_state": "running",
  152. "timestamp": int(time.time()),
  153. "migrated_from": "legacy-container-cmd",
  154. }) + "\n", encoding="utf-8")
  155. return "running"
  156. def _cmdline_argv(cmdline: Path) -> tuple[str, ...]:
  157. raw = cmdline.read_bytes()
  158. return tuple(part.decode("utf-8", "replace") for part in raw.split(b"\0") if part)
  159. def _read_container_argv() -> tuple[str, ...]:
  160. """Best-effort argv of the container's main program (the one holding ``main-wrapper.sh``):
  161. PID 1 first (s6 v2 ``/init``), then every ``/proc/*/cmdline`` (s6 v3 ``s6-svscan``)."""
  162. def _cmdlines():
  163. yield Path("/proc/1/cmdline")
  164. try:
  165. for entry in Path("/proc").iterdir():
  166. if entry.name.isdigit():
  167. yield entry / "cmdline"
  168. except OSError:
  169. return
  170. for cmdline in _cmdlines():
  171. try:
  172. argv = _cmdline_argv(cmdline)
  173. except OSError:
  174. continue
  175. if any("main-wrapper.sh" in part for part in argv):
  176. return argv
  177. return ()
  178. def _strip_container_argv_prefix(argv: Sequence[str]) -> list[str]:
  179. """Strip the s6/wrapper prefix off the container argv, leaving the hermes args.
  180. Drops everything through the ``main-wrapper.sh`` token — the stable boundary the image
  181. owns — rather than peeling tokens positionally (which broke on the s6 v2→v3 bump).
  182. """
  183. args = list(argv)
  184. wrapper_idx = next((i for i, a in enumerate(args) if a.endswith("main-wrapper.sh")), None)
  185. if wrapper_idx is not None:
  186. args = args[wrapper_idx + 1 :]
  187. elif args and Path(args[0]).name == "init": # defensive: `init` with no wrapper token
  188. args = args[1:]
  189. if args and args[0].endswith("entrypoint-dispatch.sh"): # non-PID-1 dispatch shim
  190. args = args[1:]
  191. if args and Path(args[0]).name == "hermes": # the wrapper re-execs `hermes <subcommand>`
  192. args = args[1:]
  193. return args
  194. def _is_legacy_gateway_run_request(argv: Sequence[str]) -> bool:
  195. """True for Docker commands equivalent to `gateway run`."""
  196. args = _strip_container_argv_prefix(argv)
  197. if "--no-supervise" in args:
  198. return False
  199. return len(args) >= 2 and args[0] == "gateway" and args[1] == "run"
  200. def _is_dashboard_container(argv: Sequence[str]) -> bool:
  201. """True when the container's command is the dashboard (which never supervises gateways)."""
  202. args = _strip_container_argv_prefix(argv)
  203. return bool(args) and args[0] == "dashboard"
  204. def _read_desired_state(profile_dir: Path) -> str | None:
  205. """Persisted ``desired_state`` (operator intent), else legacy ``gateway_state``; missing or
  206. unparseable files count as "no state" so a corrupt file can't bork boot."""
  207. state_file = profile_dir / "gateway_state.json"
  208. if not state_file.exists():
  209. return None
  210. try:
  211. data = json.loads(state_file.read_text(encoding="utf-8-sig"))
  212. except (OSError, json.JSONDecodeError):
  213. log.warning("could not read %s; treating as no prior state", state_file)
  214. return None
  215. desired_state = data.get("desired_state")
  216. if desired_state is not None:
  217. return desired_state
  218. gateway_state = data.get("gateway_state")
  219. return "running" if gateway_state in _TRANSIENT_RUNNING_STATES else gateway_state
  220. def _cleanup_stale_runtime_files(profile_dir: Path) -> None:
  221. """Remove PID-namespace-bound runtime files that would confuse the new gateway's checks."""
  222. for name in _STALE_RUNTIME_FILES:
  223. (profile_dir / name).unlink(missing_ok=True)
  224. def _read_prior_exit_label(profile_dir: Path) -> str:
  225. """Exception-free ``lifecycle_ledger.read_prior_exit_label`` — forensics never block boot."""
  226. try:
  227. from gateway.lifecycle_ledger import read_prior_exit_label
  228. return read_prior_exit_label(profile_dir)
  229. except Exception:
  230. return "unknown"
  231. def _write_exec(path: Path, content: str) -> None:
  232. path.write_text(content, encoding="utf-8")
  233. path.chmod(0o755)
  234. def _register_service(scandir: Path, profile: str, *, start: bool) -> None:
  235. """Recreate the s6 service slot for one profile.
  236. Mirrors ``S6ServiceManager.register_profile_gateway`` but sets start state via the ``down``
  237. marker (cont-init.d runs before s6-svscan has a control socket). Built in a sibling temp
  238. dir and ``Path.replace``d into place so an interrupted write never leaves a half-built dir.
  239. This matches :meth:`S6ServiceManager.register_profile_gateway` (PR #30136 review item O4) — even though
  240. cont-init.d runs before s6-svscan starts scanning, an atomic publication keeps the contract uniform
  241. between the two registration paths and protects against a half-populated dir if the script is
  242. interrupted mid-write.
  243. """
  244. import shutil
  245. from hermes_cli.service_manager import (
  246. S6ServiceManager, _seed_supervise_skeleton, validate_profile_name)
  247. validate_profile_name(profile)
  248. service_dir = scandir / f"gateway-{profile}"
  249. # Dot-prefixed so s6-svscan skips the staging dir: a non-dotted name gets supervised AS ROOT
  250. # by a concurrent rescan, creating a root-owned ``supervise/`` → EACCES in the seed below.
  251. tmp_dir = service_dir.with_name("." + service_dir.name + ".tmp")
  252. if tmp_dir.exists():
  253. shutil.rmtree(tmp_dir, ignore_errors=True)
  254. tmp_dir.mkdir(parents=True)
  255. try:
  256. (tmp_dir / "type").write_text("longrun\n", encoding="utf-8")
  257. # Manager's own rendering keeps both registration paths consistent; per-profile env
  258. # comes from the profile's config.yaml, so extra_env is empty.
  259. _write_exec(tmp_dir / "run", S6ServiceManager._render_run_script(profile, extra_env={}))
  260. _write_exec(tmp_dir / "finish", S6ServiceManager._render_finish_script())
  261. (tmp_dir / "log").mkdir()
  262. _write_exec(tmp_dir / "log" / "run", S6ServiceManager._render_log_run(profile))
  263. if not start: # `hermes -p <profile> gateway start` brings it up later (s6-svc -u)
  264. (tmp_dir / "down").touch()
  265. # Pre-create supervise/ with hermes ownership BEFORE publishing so s6-supervise inherits
  266. # it and runtime s6-svc calls as the hermes user won't EACCES.
  267. _seed_supervise_skeleton(tmp_dir)
  268. if service_dir.exists():
  269. shutil.rmtree(service_dir)
  270. tmp_dir.replace(service_dir) # atomic publish
  271. except Exception:
  272. shutil.rmtree(tmp_dir, ignore_errors=True)
  273. raise
  274. # ~3000 lines ≈ a year of daily reboots on a 5-profile container; rotated to .1 when crossed.
  275. _LOG_ROTATE_BYTES = 256 * 1024
  276. def _write_reconcile_log(hermes_home: Path, actions: list[ReconcileAction]) -> None:
  277. """Append one line per profile to $HERMES_HOME/logs/container-boot.log (rotated to ``.1``) —
  278. a separate greppable file for "why didn't my profile come back up".
  279. Size-bounded: when the file exceeds ``_LOG_ROTATE_BYTES`` (defaults to 256 KiB ≈ 3000 reconcile lines),
  280. the current file is renamed to ``container-boot.log.1`` (replacing any previous rotation) before the new
  281. entries are appended. This gives long- lived containers a soft cap of ~512 KiB across the two files
  282. without pulling in logrotate or s6-log machinery just for this one append-only file (PR #30136 review
  283. item O3).
  284. """
  285. import time
  286. log_dir = hermes_home / "logs"
  287. log_dir.mkdir(parents=True, exist_ok=True)
  288. log_path = log_dir / "container-boot.log"
  289. try:
  290. if log_path.exists() and log_path.stat().st_size >= _LOG_ROTATE_BYTES:
  291. log_path.replace(log_dir / "container-boot.log.1")
  292. except OSError as exc: # non-fatal — keep appending rather than lose the entry
  293. log.warning("could not rotate %s: %s", log_path, exc)
  294. ts = time.strftime("%Y-%m-%dT%H:%M:%S%z")
  295. with log_path.open("a", encoding="utf-8") as f:
  296. for a in actions:
  297. f.write(f"{ts} profile={a.profile} prior_state={a.prior_state} "
  298. f"action={a.action} prior_exit={a.prior_exit} "
  299. f"folded_into_root={a.folded_into_root}\n")
  300. def main() -> int:
  301. """Entry point invoked from /etc/cont-init.d/02-reconcile-profiles."""
  302. # A dashboard-only container must not reconcile: with a shared bind-mounted HERMES_HOME both
  303. # containers race to flock() the same s6-log files → "Resource busy" restart storm. Detected
  304. # from PID 1 argv, not an operator flag (a flag can be forgotten in a hand-written manifest).
  305. if _is_dashboard_container(_read_container_argv()):
  306. print("reconcile: skipping (dashboard container — does not need per-profile gateways)")
  307. return 0
  308. from hermes_cli.service_manager import S6_DYNAMIC_SCANDIR
  309. hermes_home = Path(os.environ.get("HERMES_HOME", "/opt/data"))
  310. scandir = Path(os.environ.get("S6_PROFILE_GATEWAY_SCANDIR", str(S6_DYNAMIC_SCANDIR)))
  311. actions = reconcile_profile_gateways(hermes_home=hermes_home, scandir=scandir)
  312. folded = [a.profile for a in actions if a.profile != "default" and a.folded_into_root]
  313. if folded:
  314. print(boot_notice(folded))
  315. for a in actions:
  316. print(f"reconcile: profile={a.profile} prior_state={a.prior_state} action={a.action}")
  317. return 0
  318. if __name__ == "__main__":
  319. raise SystemExit(main())