host_rendezvous.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812
  1. """Host-wide singleton rendezvous: one lock + one record per ROLE per OS user.
  2. Multiplex-only (Teknium ruling): exactly ONE ``hermes serve`` and ONE ``hermes gateway run``
  3. per host, each multiplexing every profile. The per-``HERMES_HOME`` gateway lock/PID files
  4. (``gateway.status``) cannot express that — N profiles are N homes, so N processes each take
  5. their own flock and none of them ever sees the others. This module adds the missing layer:
  6. * a **host lock** (flock/``msvcrt``) held for the lifetime of the winning process, and
  7. * a **rendezvous record** the winner publishes so a second invocation can find it, prove it
  8. is the same live process, and ATTACH instead of binding a second port.
  9. Both live in :func:`gateway.status._get_lock_dir` — the only cross-profile lock root already
  10. in the tree (``$HERMES_GATEWAY_LOCK_DIR`` else ``$XDG_STATE_HOME/hermes/gateway-locks``),
  11. which scopes to the **OS user**. That is the correct granularity: separate OS users have
  12. separate ``$HOME``s, separate ``~/.hermes`` profile roots, separate ports-by-convention and
  13. separate credentials, so "one per host" means "one per host per OS user".
  14. **Staleness is proved, never assumed.** New records pair the PID with the repository's stable
  15. process-start fingerprint (boot-relative on Linux/WSL); ``createTime`` remains the legacy fallback.
  16. A dead PID or a mismatched process incarnation is STALE and ignored — an attaching client must
  17. never dial a recycled PID's port.
  18. **Relationship to ``spawn-ledger.json``** (``hermes_cli/process_identity.py``): the ledger stays
  19. the append-only machine roster of every long-lived Hermes process (Desktop's attach ladder reads
  20. it) and is still written unchanged. It cannot be the host record: it has no lock, no
  21. single-writer semantics, no removal on exit, and no place to publish a protocol version or
  22. an authentication handle. The record here is authoritative for "who owns this host role"; the
  23. ledger remains authoritative for "what is running". Both are written. New host records pair the
  24. ledger's live-PID proof with ``gateway.status``'s canonical process-start fingerprint; legacy
  25. records without that fingerprint keep the old ``(pid, create_time)`` proof.
  26. """
  27. from __future__ import annotations
  28. import contextlib
  29. import enum
  30. import hashlib
  31. import json
  32. import logging
  33. import os
  34. import stat
  35. import sys
  36. from dataclasses import dataclass
  37. from datetime import datetime, timezone, UTC
  38. from pathlib import Path
  39. from typing import Any, Optional, Sequence
  40. from hermes_dir_acl import make_user_dir
  41. from utils import atomic_json_write
  42. logger = logging.getLogger(__name__)
  43. #: Bumped when the record's shape or the attach handshake changes incompatibly. A reader that
  44. #: does not recognise the version refuses to attach instead of guessing.
  45. HOST_PROTOCOL_VERSION = 1
  46. #: Token-gated endpoint the owner answers with its own identity. An attaching client dials the
  47. #: RECORDED port and only attaches when the answer is the recorded owner — a record alone proves
  48. #: nothing about who holds that port right now.
  49. HOST_IDENTITY_PATH = "/api/host/identity"
  50. #: Bounded: a closed port, a black-holed one or a foreign listener must cost a second, not a hang.
  51. PROBE_TIMEOUT_S = 2.0
  52. ROLE_GATEWAY = "gateway"
  53. ROLE_SERVE = "serve"
  54. #: A Desktop-owned pool child (loopback, random port, per-profile lifecycle). It is NOT a host
  55. #: owner — the attach/refuse ladder reads ``ROLE_SERVE`` only, so a supervised public dashboard
  56. #: never stands down behind it (#119824) — but ``hermes plugins install`` from a terminal still
  57. #: has to reach the backend hosting the open chats (#119644), and this record + 0600 token is
  58. #: how it dials one on a Desktop-only box.
  59. ROLE_DESKTOP_SERVE = "desktop-serve"
  60. _ROLES = (ROLE_GATEWAY, ROLE_SERVE, ROLE_DESKTOP_SERVE)
  61. # Open lock handles, keyed by (role, resolved lock path): the OS releases the flock when this
  62. # process dies, which is what makes a crashed owner's host lock re-acquirable without a reaper.
  63. # The PATH is part of the key because the lock dir is env-derived (HERMES_GATEWAY_LOCK_DIR):
  64. # keyed by role alone, a second call after the dir changed returned "already held" without ever
  65. # creating the new lock file, so owns_host_lock() lied and one pytest process leaked the handle
  66. # across tests.
  67. _lock_handles: dict[tuple[str, str], Any] = {}
  68. @dataclass(frozen=True)
  69. class HostRecord:
  70. """A published host-role owner. ``profiles`` is the SERVED set, not the launch profile."""
  71. role: str
  72. pid: int
  73. create_time: Optional[float]
  74. host: str
  75. port: Optional[int]
  76. protocol_version: int
  77. token_fingerprint: str
  78. profiles: tuple[str, ...]
  79. updated_at: str
  80. #: HERMES_HOME the owner was launched from. The attach channel (``gateway.control_socket``) is
  81. #: keyed by home, so without it a client can only guess the default root — wrong as soon as a
  82. #: named profile launches the host process. Absent in records written before this field; added
  83. #: WITHOUT a protocol bump on purpose, because a bump would make every live owner's record read
  84. #: as stale and a second gateway would start.
  85. home: str = ""
  86. #: Stable process-start fingerprint. On Linux/WSL this is boot-relative /proc start ticks,
  87. #: so a wall-clock resync cannot make a live owner look like a recycled PID. Optional for
  88. #: records written by older Hermes versions, which still fall back to create_time.
  89. #: NOT unique across reboots — pair it with ``boot_epoch``.
  90. start_time: Optional[int] = None
  91. #: Instantiation epoch (boot id + PID 1 start). Distinguishes a pre-reboot owner from an
  92. #: early-boot process that reused its PID and landed inside the start-tick drift tolerance.
  93. #: Absent on older records; added WITHOUT a protocol bump, because a bump would make every
  94. #: live owner's record read as stale and a second gateway would start.
  95. boot_epoch: str = ""
  96. def to_json(self) -> dict[str, Any]:
  97. return {
  98. "role": self.role,
  99. "home": self.home,
  100. "pid": self.pid,
  101. "createTime": self.create_time,
  102. "startTime": self.start_time,
  103. "bootEpoch": self.boot_epoch,
  104. "host": self.host,
  105. "port": self.port,
  106. "protocolVersion": self.protocol_version,
  107. "tokenFingerprint": self.token_fingerprint,
  108. "profiles": list(self.profiles),
  109. "updatedAt": self.updated_at,
  110. }
  111. @classmethod
  112. def from_json(cls, payload: Any) -> Optional[HostRecord]:
  113. if not isinstance(payload, dict):
  114. return None
  115. pid = payload.get("pid")
  116. role = payload.get("role")
  117. if not isinstance(pid, int) or pid <= 0 or role not in _ROLES:
  118. return None
  119. create = payload.get("createTime")
  120. start = payload.get("startTime")
  121. port = payload.get("port")
  122. profiles = payload.get("profiles")
  123. version = payload.get("protocolVersion")
  124. return cls(
  125. role=role,
  126. pid=pid,
  127. create_time=float(create) if isinstance(create, (int, float)) else None,
  128. start_time=(
  129. int(start)
  130. if isinstance(start, int) and not isinstance(start, bool) and start > 0
  131. else None
  132. ),
  133. boot_epoch=str(payload.get("bootEpoch") or ""),
  134. host=str(payload.get("host") or ""),
  135. port=int(port) if isinstance(port, int) and 0 < port <= 65535 else None,
  136. protocol_version=version if isinstance(version, int) else 0,
  137. token_fingerprint=str(payload.get("tokenFingerprint") or ""),
  138. profiles=tuple(str(p) for p in profiles if isinstance(p, str)) if isinstance(profiles, list) else (),
  139. updated_at=str(payload.get("updatedAt") or ""),
  140. home=str(payload.get("home") or ""),
  141. )
  142. def host_state_dir() -> Path:
  143. """Per-OS-USER rendezvous dir (shared by every profile of this user)."""
  144. from gateway.status import _get_lock_dir
  145. return _get_lock_dir()
  146. def ensure_host_state_dir() -> Path:
  147. """The rendezvous dir, created owner-only (``0o700``) and tightened if it is not.
  148. A bare ``mkdir`` under the common ``umask 002`` leaves the dir group-writable, and the record
  149. inside it is what every lifecycle verb believes: a same-group process could unlink+replace it
  150. and choose this host's ATTACH answers (which home to dial, which profiles are "served").
  151. Ownership of the dir is ours, so widening is repaired rather than refused.
  152. """
  153. directory = host_state_dir()
  154. make_user_dir(directory, parents=True, exist_ok=True, mode=0o700)
  155. if sys.platform != "win32":
  156. with contextlib.suppress(OSError):
  157. if stat.S_IMODE(directory.stat().st_mode) & 0o077:
  158. os.chmod(directory, 0o700)
  159. return directory
  160. def _record_is_own(path: Path) -> bool:
  161. """True when ``path`` was written by THIS OS user inside a dir this user owns.
  162. ``read_record`` hands its result straight to :mod:`gateway.host_attach`, which dials the home
  163. it names and believes the served set it carries. A hand-written record therefore buys an
  164. attacker the lifecycle verdict for every profile on the host, so the file's ``st_uid`` — not
  165. its contents — is what makes it a record at all. Windows sets no ACLs from mode bits; there
  166. the dir already lives under the user's own state root.
  167. """
  168. if sys.platform == "win32":
  169. return True
  170. try:
  171. info = path.stat()
  172. parent = path.parent.stat()
  173. except OSError:
  174. return False
  175. uid = os.getuid() # windows-footgun: ok — unreachable on Windows (early return above)
  176. if info.st_uid != uid or parent.st_uid != uid:
  177. logger.warning(
  178. "ignoring host record %s: owned by uid %s (expected %s)", path, info.st_uid, uid)
  179. return False
  180. return True
  181. def _validated_role(role: str) -> str:
  182. if role not in _ROLES:
  183. raise ValueError(f"unknown host role: {role!r}")
  184. return role
  185. def record_path(role: str) -> Path:
  186. return host_state_dir() / f"host-{_validated_role(role)}.json"
  187. def lock_path(role: str) -> Path:
  188. return host_state_dir() / f"host-{_validated_role(role)}.lock"
  189. def token_path(role: str) -> Path:
  190. return host_state_dir() / f"host-{_validated_role(role)}.token"
  191. def token_fingerprint(token: str) -> str:
  192. """Short, non-reversible handle for a session token (safe to publish in the record)."""
  193. return hashlib.sha256(token.encode("utf-8", "replace")).hexdigest()[:16] if token else ""
  194. def process_create_time(pid: Optional[int] = None) -> Optional[float]:
  195. """Creation time of ``pid`` (default: this process); ``None`` when unknowable."""
  196. from hermes_cli.process_identity import _process_create_time
  197. return _process_create_time(pid)
  198. def _pid_incarnation_matches(pid: int, create_time: Optional[float]) -> Optional[bool]:
  199. """Reuse the spawn ledger's proof: True/False when provable, ``None`` when it cannot say."""
  200. from hermes_cli.process_identity import _pid_alive_matches
  201. return _pid_alive_matches(pid, create_time)
  202. # A record published before ``bootEpoch`` existed cannot name its boot. Its ``createTime`` older
  203. # than this boot's start (``psutil.boot_time()``, minus this slack) means EITHER a reboot (the
  204. # owner is gone and its PID was recycled) OR a wall-clock step since the owner started (boot time
  205. # is wall clock minus uptime, so a forward step moves it past a running owner's saved start;
  206. # #117505). Clocks cannot tell those apart, so the host lock decides: a live gateway holds it.
  207. _LEGACY_BOOT_SLACK_S = 120.0
  208. def _boot_time() -> Optional[float]:
  209. try:
  210. import psutil
  211. return float(psutil.boot_time())
  212. except Exception:
  213. logger.debug("boot time unreadable; legacy record age check skipped", exc_info=True)
  214. return None
  215. def _host_lock_is_free(role: str) -> Optional[bool]:
  216. """True when no process holds the host lock for ``role``; ``None`` when it cannot be probed.
  217. Read-only callers (status, dashboard) reach this, so it never creates the lock dir or file: a
  218. missing lock file means nobody holds the lock. The probe takes the lock for an instant, so a
  219. gateway claiming it in that instant gets HELD_BY_OTHER and its supervisor retries.
  220. """
  221. if owns_host_lock(role):
  222. return False
  223. from gateway.status import _probe_lock_file
  224. try:
  225. fd = os.open(str(lock_path(role)), os.O_RDWR)
  226. except FileNotFoundError:
  227. return True
  228. except OSError:
  229. logger.debug("host %s lock could not be probed", role, exc_info=True)
  230. return None
  231. return not _probe_lock_file(os.fdopen(fd, "r+", encoding="utf-8"))
  232. def _current_boot_epoch() -> str:
  233. """Identity of this boot. Empty when /proc is unreadable — a recorded epoch is then unprovable."""
  234. try:
  235. from gateway.drain_control import current_instantiation_epoch
  236. return current_instantiation_epoch() or ""
  237. except Exception:
  238. logger.debug("boot epoch unreadable; host liveness will not use it", exc_info=True)
  239. return ""
  240. def _record_incarnation_matches(record: HostRecord) -> Optional[bool]:
  241. """Whether the record still names the same live process incarnation.
  242. New records prefer the canonical start fingerprint: on Linux/WSL it comes from /proc start
  243. ticks and is immune to wall-clock shifts that move psutil.create_time(). Those ticks are
  244. NOT unique across reboots — they reset, and an early-boot PID reused within the drift
  245. tolerance looks like the dead owner. A published boot epoch that differs from this boot's is a
  246. mismatch; one that cannot be compared (this boot's epoch is unreadable) is unprovable, ``None``.
  247. A gateway record with no epoch (published before ``bootEpoch`` existed, or on any platform
  248. without /proc, where the epoch is always empty) whose create_time predates this boot is a
  249. mismatch only while nobody holds the host lock. A record with neither create_time nor
  250. start_time proves nothing about its pid, so it never vouches for one. Legacy records without
  251. a start fingerprint keep the create-time check. Do not infer identity from argv.
  252. """
  253. recorded_epoch = record.boot_epoch
  254. epoch = _current_boot_epoch()
  255. if recorded_epoch:
  256. if not epoch:
  257. return None # boots can't be compared; matching ticks alone prove nothing
  258. if recorded_epoch != epoch:
  259. return False
  260. if record.start_time is None and record.create_time is None:
  261. return False
  262. if record.start_time is None:
  263. matched = _pid_incarnation_matches(record.pid, record.create_time)
  264. else:
  265. alive = _pid_incarnation_matches(record.pid, None)
  266. if alive is not True:
  267. return alive
  268. from gateway.status import get_process_start_time, start_time_fingerprints_match
  269. current = get_process_start_time(record.pid)
  270. matched = None if current is None else start_time_fingerprints_match(record.start_time, current)
  271. if matched is not True:
  272. return matched
  273. if record.role == ROLE_GATEWAY and not recorded_epoch and record.create_time is not None:
  274. boot = _boot_time()
  275. if boot is not None and record.create_time < boot - _LEGACY_BOOT_SLACK_S:
  276. lock_free = _host_lock_is_free(record.role)
  277. if lock_free is None:
  278. return None
  279. if lock_free:
  280. return False
  281. return True
  282. def record_is_stale(record: Optional[HostRecord]) -> bool:
  283. """A record nobody may attach to: absent, unknown protocol, dead PID, or PID reuse.
  284. ``None`` from the liveness probe (no psutil, permission denied, an unexpected psutil error)
  285. means UNPROVABLE, and an unprovable record is a CANDIDATE, never an owner: it survives this
  286. predicate only to be handed to :func:`probe_owner`, which dials the recorded port and makes
  287. the owner prove itself. Treating unprovable as dead would let a second process bind a second
  288. port; treating it as alive *without the probe* made a record for a long-dead PID a permanent
  289. silent outage (attach exited 0 forever, with nothing listening).
  290. """
  291. if record is None:
  292. return True
  293. if record.protocol_version != HOST_PROTOCOL_VERSION:
  294. return True
  295. return _record_incarnation_matches(record) is False
  296. def liveness_is_proven(record: HostRecord) -> bool:
  297. """True only when the record's process-incarnation probe positively matched (never on ``None``).
  298. A record with neither ``createTime`` nor ``startTime`` cannot prove incarnation:
  299. ``_same_incarnation`` treats a missing create_time as a match, so a recycled pid would
  300. otherwise look alive forever. Topology already refused that shape; attach did not.
  301. """
  302. if record.create_time is None and record.start_time is None:
  303. return False
  304. return _record_incarnation_matches(record) is True
  305. def dial_host(record: HostRecord) -> str:
  306. """Address to dial for ``record``: a wildcard bind is reached over loopback."""
  307. host = record.host or "127.0.0.1"
  308. return "127.0.0.1" if host in ("0.0.0.0", "::", "*", "") else host
  309. def probe_owner(record: HostRecord, *, timeout: float = PROBE_TIMEOUT_S) -> Optional[dict]:
  310. """Make the recorded endpoint prove it is this record's owner; ``None`` when it does not.
  311. Two gates, both required: a bounded TCP connect (nothing listening → the owner is gone, even
  312. though its record and PID may still look alive during a graceful-shutdown window) and an
  313. identity GET authenticated with the 0600 token (a foreign listener that inherited the port
  314. answers the connect but cannot answer as PID N of role R).
  315. Returns the owner's identity payload (``pid``/``role``/``servesSpa``) on success.
  316. """
  317. if not record.port:
  318. return None
  319. import socket
  320. host = dial_host(record)
  321. try:
  322. with socket.create_connection((host, record.port), timeout=timeout):
  323. pass
  324. except OSError:
  325. logger.debug("host %s owner does not answer at %s:%s", record.role, host, record.port)
  326. return None
  327. import urllib.request
  328. token = read_token(record.role)
  329. headers = {"X-Hermes-Token": token, "Authorization": f"Bearer {token}"} if token else {}
  330. request = urllib.request.Request(
  331. f"http://{host}:{record.port}{HOST_IDENTITY_PATH}", headers=headers)
  332. try:
  333. with urllib.request.urlopen(request, timeout=timeout) as response:
  334. if response.status != 200:
  335. return None
  336. payload = json.loads(response.read(65536).decode("utf-8", "replace"))
  337. except Exception:
  338. logger.debug("host %s identity probe failed at %s:%s", record.role, host, record.port, exc_info=True)
  339. return None
  340. if not isinstance(payload, dict):
  341. return None
  342. if payload.get("role") != record.role or payload.get("pid") != record.pid:
  343. return None
  344. return payload
  345. def read_record(role: str, *, include_stale: bool = False) -> Optional[HostRecord]:
  346. """Published record for ``role``; ``None`` when absent, foreign, corrupt or (by default) stale."""
  347. path = record_path(role)
  348. if not _record_is_own(path):
  349. return None
  350. try:
  351. raw = path.read_text(encoding="utf-8-sig")
  352. except (OSError, UnicodeDecodeError):
  353. return None
  354. try:
  355. record = HostRecord.from_json(json.loads(raw))
  356. except (ValueError, TypeError):
  357. return None
  358. if record is None:
  359. return None
  360. return record if include_stale or not record_is_stale(record) else None
  361. def read_token(role: str) -> str:
  362. """Owner-written session token for ``role`` (``""`` when absent/unreadable).
  363. This is the handle an attaching client uses when the backend is auth-gated and therefore
  364. withholds its token from an unauthenticated ``GET /``. Confidentiality is enforced at WRITE
  365. time by :func:`_write_private_text` (POSIX ``0o600``; an owner+SYSTEM-only protected DACL on
  366. Windows, where mode bits set no ACLs at all). Reading it is therefore evidence of, not proof
  367. of, same-OS-user authority — the authority boundary the host lock is scoped to.
  368. """
  369. try:
  370. return token_path(role).read_text(encoding="utf-8-sig").strip()
  371. except (OSError, UnicodeDecodeError):
  372. return ""
  373. def record_token_is_consistent(record: HostRecord) -> bool:
  374. """Does the on-disk token still hash to the record's fingerprint?
  375. A record published without a token (the gateway) has an empty fingerprint and is consistent
  376. by definition. A mismatch means the record and the token file come from different
  377. incarnations (a torn restart) — discovery must not attach with a token the owner rejects.
  378. """
  379. if not record.token_fingerprint:
  380. return True
  381. return token_fingerprint(read_token(record.role)) == record.token_fingerprint
  382. def _write_private_text(path: Path, text: str) -> None:
  383. """Create/replace ``path`` with owner-only content.
  384. POSIX: ``0o600`` via tmp + atomic replace. Windows: mode bits set NO ACLs, so the same
  385. ``os.open`` would leave a live session token readable by whatever the inherited DACL grants;
  386. the SSH runtime's protected owner+SYSTEM DACL writer is the repo's primitive for exactly this
  387. credential class and is reused here. It also replaces in place, because ``os.replace`` onto a
  388. token file another process still holds open fails on Windows.
  389. """
  390. if sys.platform == "win32":
  391. from hermes_cli.windows_ssh_runtime import write_private_file
  392. write_private_file(path, text.encode("utf-8"))
  393. return
  394. tmp = path.with_name(path.name + ".tmp")
  395. fd = os.open(str(tmp), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
  396. try:
  397. with os.fdopen(fd, "w", encoding="utf-8") as handle:
  398. handle.write(text)
  399. handle.flush()
  400. with contextlib.suppress(OSError):
  401. os.fsync(handle.fileno())
  402. except BaseException:
  403. with contextlib.suppress(OSError):
  404. os.unlink(str(tmp))
  405. raise
  406. os.replace(str(tmp), str(path))
  407. class HostLockOutcome(enum.Enum):
  408. """Why a host-lock claim ended the way it did.
  409. ``COULD_NOT_OPEN`` is NOT contention: a read-only/undeletable lock dir (EROFS, EACCES,
  410. ENOSPC) collapsed into the same "another owner holds it" log and sent operators hunting a
  411. second gateway that never existed.
  412. """
  413. ACQUIRED = "acquired"
  414. HELD_BY_OTHER = "held-by-other"
  415. COULD_NOT_OPEN = "could-not-open"
  416. def _lock_key(role: str) -> tuple[str, str]:
  417. path = lock_path(role)
  418. with contextlib.suppress(OSError):
  419. return (role, os.path.abspath(str(path)))
  420. return (role, str(path))
  421. def claim_host_lock(role: str) -> tuple[HostLockOutcome, Optional[OSError]]:
  422. """Take the host-wide lock for ``role``. Idempotent per (role, lock path).
  423. Returns the outcome and, for ``COULD_NOT_OPEN``, the OSError that explains it.
  424. """
  425. role = _validated_role(role)
  426. key = _lock_key(role)
  427. if _lock_handles.get(key) is not None:
  428. return (HostLockOutcome.ACQUIRED, None)
  429. from gateway.status import _try_acquire_file_lock
  430. path = Path(key[1])
  431. try:
  432. ensure_host_state_dir()
  433. handle = open(path, "a+", encoding="utf-8")
  434. except OSError as exc:
  435. logger.debug("host %s lock could not be opened at %s", role, path, exc_info=True)
  436. return (HostLockOutcome.COULD_NOT_OPEN, exc)
  437. if not _try_acquire_file_lock(handle):
  438. with contextlib.suppress(OSError):
  439. handle.close()
  440. return (HostLockOutcome.HELD_BY_OTHER, None)
  441. _lock_handles[key] = handle
  442. return (HostLockOutcome.ACQUIRED, None)
  443. def release_host_lock(role: str) -> None:
  444. """Release the host lock for ``role`` when this process holds it."""
  445. handle = _lock_handles.pop(_lock_key(_validated_role(role)), None)
  446. if handle is None:
  447. return
  448. from gateway.status import _release_file_lock
  449. _release_file_lock(handle)
  450. with contextlib.suppress(OSError):
  451. handle.close()
  452. def owns_host_lock(role: str) -> bool:
  453. """True when THIS process holds the host lock for ``role`` (re-probing our own flock lies)."""
  454. return _lock_handles.get(_lock_key(_validated_role(role))) is not None
  455. def host_lock_is_held(role: str) -> Optional[bool]:
  456. """Whether any process currently holds the host lock for ``role``.
  457. ``True`` / ``False`` when the flock can be observed; ``None`` when the lock file cannot
  458. be opened. Never retains a lock. This process's handle is authoritative: re-probing our
  459. own flock succeeds on POSIX, so a second open would lie.
  460. """
  461. role = _validated_role(role)
  462. if owns_host_lock(role):
  463. return True
  464. path = lock_path(role)
  465. if not path.exists():
  466. return False
  467. try:
  468. handle = open(path, "a+", encoding="utf-8")
  469. except OSError:
  470. return None
  471. from gateway.status import _probe_lock_file
  472. return _probe_lock_file(handle)
  473. def record_names_live_owner(record: HostRecord) -> bool:
  474. """True when ``record``'s incarnation is proven AND its host lock is not observed free.
  475. The publisher holds the host lock for its whole life and the OS drops it on death, so a free
  476. flock falsifies a record whose pid merely looks alive (recycle, start-tick collision after a
  477. reboot — #138343, #133744). Every reader that trusts a record as a running owner uses this.
  478. """
  479. if not liveness_is_proven(record):
  480. return False
  481. if host_lock_is_held(record.role) is False:
  482. logger.debug("host %s record names pid %s but the host lock is free", record.role, record.pid)
  483. return False
  484. return True
  485. def publish_record(
  486. role: str,
  487. *,
  488. host: str = "",
  489. port: Optional[int] = None,
  490. profiles: Sequence[str] = (),
  491. token: Optional[str] = None,
  492. home: str = "",
  493. ) -> Optional[HostRecord]:
  494. """Publish this process as the host owner of ``role``. ``None`` when the write failed.
  495. ``token`` (serve) is persisted 0600 next to the record and only its fingerprint is published.
  496. ``home`` is the launch HERMES_HOME — the key an attaching client needs to reach this owner's
  497. control socket.
  498. """
  499. role = _validated_role(role)
  500. from gateway.status import get_process_start_time
  501. record = HostRecord(
  502. role=role,
  503. pid=os.getpid(),
  504. create_time=process_create_time(),
  505. start_time=get_process_start_time(os.getpid()),
  506. boot_epoch=_current_boot_epoch(),
  507. host=str(host or ""),
  508. port=int(port) if isinstance(port, int) and port > 0 else None,
  509. protocol_version=HOST_PROTOCOL_VERSION,
  510. token_fingerprint=token_fingerprint(token or ""),
  511. profiles=tuple(str(p) for p in profiles),
  512. updated_at=datetime.now(UTC).isoformat(),
  513. home=str(home or ""),
  514. )
  515. try:
  516. ensure_host_state_dir()
  517. if token:
  518. # No record without its token: publishing one an attaching client cannot
  519. # authenticate against would degrade to a silent "attach refused forever".
  520. _write_private_text(token_path(role), token)
  521. atomic_json_write(record_path(role), record.to_json(), mode=0o600)
  522. except OSError:
  523. logger.warning("host %s record could not be published; discovery will not find it", role, exc_info=True)
  524. return None
  525. _invalidate_attach_cache()
  526. return record
  527. def _invalidate_attach_cache() -> None:
  528. """Drop :mod:`gateway.host_attach`'s memo: the record it summarises just changed."""
  529. with contextlib.suppress(Exception):
  530. from gateway.host_attach import invalidate_host_gateway_cache
  531. invalidate_host_gateway_cache()
  532. def discard_dead_record(role: str) -> bool:
  533. """Retract the record for ``role`` when its owner is provably gone; True when one was removed.
  534. A confirmed stop must retract the record too. Leaving it made ``gateway restart --all`` a
  535. silent no-op: the re-entered ``gateway run`` read the corpse's record, decided ATTACH and
  536. exited 0, so the host ended up with no gateway at all.
  537. """
  538. role = _validated_role(role)
  539. record = read_record(role, include_stale=True)
  540. if record is None:
  541. return False
  542. guard = None
  543. if record.pid != os.getpid() and _record_incarnation_matches(record) is not False:
  544. # A recycled pid can still look like the same incarnation. The publisher holds the
  545. # host lock for its life, so winning it proves the recorded owner is gone. Hold it
  546. # across the re-read and unlink: a free-lock probe released first let a new gateway
  547. # claim and publish in the gap, and we deleted ITS record and token. A private handle,
  548. # not claim_host_lock: cleanup must not enter this process's own claim/release ledger.
  549. if owns_host_lock(role) or (guard := _hold_lock_briefly(role)) is None:
  550. return False
  551. try:
  552. if guard is not None and read_record(role, include_stale=True) is None:
  553. return False
  554. for path in (record_path(role), token_path(role)):
  555. with contextlib.suppress(OSError):
  556. path.unlink(missing_ok=True)
  557. finally:
  558. if guard is not None:
  559. from gateway.status import _release_file_lock
  560. _release_file_lock(guard)
  561. with contextlib.suppress(OSError):
  562. guard.close()
  563. _invalidate_attach_cache()
  564. return True
  565. def _hold_lock_briefly(role: str):
  566. """Open and flock ``role``'s host lock without registering it; ``None`` when not won."""
  567. from gateway.status import _try_acquire_file_lock
  568. path = lock_path(role)
  569. try:
  570. handle = open(path, "a+", encoding="utf-8")
  571. except OSError:
  572. return None
  573. if _try_acquire_file_lock(handle):
  574. return handle
  575. with contextlib.suppress(OSError):
  576. handle.close()
  577. return None
  578. def clear_record(role: str) -> None:
  579. """Remove this process's record + token on exit (never another owner's)."""
  580. role = _validated_role(role)
  581. existing = read_record(role, include_stale=True)
  582. if existing is not None and existing.pid != os.getpid():
  583. return
  584. for path in (record_path(role), token_path(role)):
  585. with contextlib.suppress(OSError):
  586. path.unlink(missing_ok=True)
  587. _invalidate_attach_cache()
  588. # Roles this process must clean up on the way out, and the signal handlers we prepended.
  589. _cleanup_roles: list[str] = []
  590. _prev_signal_handlers: dict[int, Any] = {}
  591. def _cleanup_role(role: str) -> None:
  592. with contextlib.suppress(Exception):
  593. clear_record(role)
  594. with contextlib.suppress(Exception):
  595. release_host_lock(role)
  596. def _handle_terminating_signal(signum, frame) -> None:
  597. """Drop the record + token, then hand off to the handler we prepended to."""
  598. import signal as _signal
  599. for role in tuple(_cleanup_roles):
  600. _cleanup_role(role)
  601. prev = _prev_signal_handlers.get(signum)
  602. if callable(prev):
  603. prev(signum, frame)
  604. return
  605. if prev is _signal.SIG_IGN:
  606. return
  607. with contextlib.suppress(Exception):
  608. _signal.signal(signum, _signal.SIG_DFL)
  609. os.kill(os.getpid(), signum)
  610. raise SystemExit(128 + int(signum))
  611. def cleanup_on_exit(role: str) -> None:
  612. """Clear ``role``'s record + token and release its lock on exit — SIGTERM included.
  613. ``atexit`` alone is false advertising for the NORMAL stop: systemd stop, ``docker stop`` and
  614. the update relaunch all send SIGTERM, and every terminating SIGTERM path here ends in the
  615. default disposition (uvicorn's ``capture_signals`` re-raises it after its graceful shutdown),
  616. which kills the process without running ``atexit``. The record then outlived its process and
  617. the 0600 token kept a LIVE session token on disk indefinitely.
  618. The handler only PREPENDS cleanup: whatever handler was installed before (uvicorn's graceful
  619. shutdown, the exit-flush chain, a supervisor's) still runs, so the shutdown sequence is
  620. unchanged.
  621. """
  622. import atexit
  623. import signal as _signal
  624. import threading
  625. role = _validated_role(role)
  626. if role not in _cleanup_roles:
  627. _cleanup_roles.append(role)
  628. atexit.register(_cleanup_role, role)
  629. if threading.current_thread() is not threading.main_thread():
  630. return
  631. for name in ("SIGTERM", "SIGBREAK"):
  632. signum = getattr(_signal, name, None)
  633. if signum is None or signum in _prev_signal_handlers:
  634. continue
  635. with contextlib.suppress(ValueError, OSError, RuntimeError):
  636. prev = _signal.getsignal(signum)
  637. _signal.signal(signum, _handle_terminating_signal)
  638. _prev_signal_handlers[signum] = prev
  639. def _multiplex_profiles_enabled() -> bool:
  640. """Will THIS process multiplex? An explicit ``true`` and an unset key both say yes, and an
  641. explicit ``false`` is RETIRED (``hermes_cli.gateway_multiplex_mode``) — it is warned about and
  642. ignored at boot, so it must not make the claim-time record advertise a narrower roster than
  643. the process actually serves. Reading it here was the last place the retired flag still decided
  644. topology, and it made CLI/dashboard report "standalone, serving default" while the runtime
  645. multiplexed. The RUNTIME verdict (a boot-time guard refusal) narrows the record afterwards, in
  646. ``gateway.run._refresh_host_gateway_record``, which republishes the SETTLED set.
  647. """
  648. return True
  649. def served_profiles(*, multiplex: Optional[bool] = None) -> tuple[str, ...]:
  650. """Profiles this process multiplexes; ``()`` when the roster cannot be read.
  651. ``multiplex`` defaults to what this process's own config says. Hard-coding ``True`` here
  652. published a record claiming EVERY profile from a gateway that would only ever serve its own,
  653. and a second profile's supervised unit then stood down against a set nobody serves.
  654. """
  655. try:
  656. from hermes_cli.profiles import profiles_to_serve
  657. enabled = _multiplex_profiles_enabled() if multiplex is None else bool(multiplex)
  658. return tuple(name for name, _ in profiles_to_serve(multiplex=enabled))
  659. except Exception:
  660. logger.debug("served profile roster unavailable", exc_info=True)
  661. return ()
  662. def describe(record: HostRecord) -> str:
  663. """One-line human description used by attach messages and conflict logs."""
  664. where = f"{record.host or '127.0.0.1'}:{record.port}" if record.port else "no bound port"
  665. profiles = ", ".join(record.profiles) if record.profiles else "unknown"
  666. return f"PID {record.pid} ({where}; profiles: {profiles})"