| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812 |
- """Host-wide singleton rendezvous: one lock + one record per ROLE per OS user.
- Multiplex-only (Teknium ruling): exactly ONE ``hermes serve`` and ONE ``hermes gateway run``
- per host, each multiplexing every profile. The per-``HERMES_HOME`` gateway lock/PID files
- (``gateway.status``) cannot express that — N profiles are N homes, so N processes each take
- their own flock and none of them ever sees the others. This module adds the missing layer:
- * a **host lock** (flock/``msvcrt``) held for the lifetime of the winning process, and
- * a **rendezvous record** the winner publishes so a second invocation can find it, prove it
- is the same live process, and ATTACH instead of binding a second port.
- Both live in :func:`gateway.status._get_lock_dir` — the only cross-profile lock root already
- in the tree (``$HERMES_GATEWAY_LOCK_DIR`` else ``$XDG_STATE_HOME/hermes/gateway-locks``),
- which scopes to the **OS user**. That is the correct granularity: separate OS users have
- separate ``$HOME``s, separate ``~/.hermes`` profile roots, separate ports-by-convention and
- separate credentials, so "one per host" means "one per host per OS user".
- **Staleness is proved, never assumed.** New records pair the PID with the repository's stable
- process-start fingerprint (boot-relative on Linux/WSL); ``createTime`` remains the legacy fallback.
- A dead PID or a mismatched process incarnation is STALE and ignored — an attaching client must
- never dial a recycled PID's port.
- **Relationship to ``spawn-ledger.json``** (``hermes_cli/process_identity.py``): the ledger stays
- the append-only machine roster of every long-lived Hermes process (Desktop's attach ladder reads
- it) and is still written unchanged. It cannot be the host record: it has no lock, no
- single-writer semantics, no removal on exit, and no place to publish a protocol version or
- an authentication handle. The record here is authoritative for "who owns this host role"; the
- ledger remains authoritative for "what is running". Both are written. New host records pair the
- ledger's live-PID proof with ``gateway.status``'s canonical process-start fingerprint; legacy
- records without that fingerprint keep the old ``(pid, create_time)`` proof.
- """
- from __future__ import annotations
- import contextlib
- import enum
- import hashlib
- import json
- import logging
- import os
- import stat
- import sys
- from dataclasses import dataclass
- from datetime import datetime, timezone, UTC
- from pathlib import Path
- from typing import Any, Optional, Sequence
- from hermes_dir_acl import make_user_dir
- from utils import atomic_json_write
- logger = logging.getLogger(__name__)
- #: Bumped when the record's shape or the attach handshake changes incompatibly. A reader that
- #: does not recognise the version refuses to attach instead of guessing.
- HOST_PROTOCOL_VERSION = 1
- #: Token-gated endpoint the owner answers with its own identity. An attaching client dials the
- #: RECORDED port and only attaches when the answer is the recorded owner — a record alone proves
- #: nothing about who holds that port right now.
- HOST_IDENTITY_PATH = "/api/host/identity"
- #: Bounded: a closed port, a black-holed one or a foreign listener must cost a second, not a hang.
- PROBE_TIMEOUT_S = 2.0
- ROLE_GATEWAY = "gateway"
- ROLE_SERVE = "serve"
- #: A Desktop-owned pool child (loopback, random port, per-profile lifecycle). It is NOT a host
- #: owner — the attach/refuse ladder reads ``ROLE_SERVE`` only, so a supervised public dashboard
- #: never stands down behind it (#119824) — but ``hermes plugins install`` from a terminal still
- #: has to reach the backend hosting the open chats (#119644), and this record + 0600 token is
- #: how it dials one on a Desktop-only box.
- ROLE_DESKTOP_SERVE = "desktop-serve"
- _ROLES = (ROLE_GATEWAY, ROLE_SERVE, ROLE_DESKTOP_SERVE)
- # Open lock handles, keyed by (role, resolved lock path): the OS releases the flock when this
- # process dies, which is what makes a crashed owner's host lock re-acquirable without a reaper.
- # The PATH is part of the key because the lock dir is env-derived (HERMES_GATEWAY_LOCK_DIR):
- # keyed by role alone, a second call after the dir changed returned "already held" without ever
- # creating the new lock file, so owns_host_lock() lied and one pytest process leaked the handle
- # across tests.
- _lock_handles: dict[tuple[str, str], Any] = {}
- @dataclass(frozen=True)
- class HostRecord:
- """A published host-role owner. ``profiles`` is the SERVED set, not the launch profile."""
- role: str
- pid: int
- create_time: Optional[float]
- host: str
- port: Optional[int]
- protocol_version: int
- token_fingerprint: str
- profiles: tuple[str, ...]
- updated_at: str
- #: HERMES_HOME the owner was launched from. The attach channel (``gateway.control_socket``) is
- #: keyed by home, so without it a client can only guess the default root — wrong as soon as a
- #: named profile launches the host process. Absent in records written before this field; added
- #: WITHOUT a protocol bump on purpose, because a bump would make every live owner's record read
- #: as stale and a second gateway would start.
- home: str = ""
- #: Stable process-start fingerprint. On Linux/WSL this is boot-relative /proc start ticks,
- #: so a wall-clock resync cannot make a live owner look like a recycled PID. Optional for
- #: records written by older Hermes versions, which still fall back to create_time.
- #: NOT unique across reboots — pair it with ``boot_epoch``.
- start_time: Optional[int] = None
- #: Instantiation epoch (boot id + PID 1 start). Distinguishes a pre-reboot owner from an
- #: early-boot process that reused its PID and landed inside the start-tick drift tolerance.
- #: Absent on older records; added WITHOUT a protocol bump, because a bump would make every
- #: live owner's record read as stale and a second gateway would start.
- boot_epoch: str = ""
- def to_json(self) -> dict[str, Any]:
- return {
- "role": self.role,
- "home": self.home,
- "pid": self.pid,
- "createTime": self.create_time,
- "startTime": self.start_time,
- "bootEpoch": self.boot_epoch,
- "host": self.host,
- "port": self.port,
- "protocolVersion": self.protocol_version,
- "tokenFingerprint": self.token_fingerprint,
- "profiles": list(self.profiles),
- "updatedAt": self.updated_at,
- }
- @classmethod
- def from_json(cls, payload: Any) -> Optional[HostRecord]:
- if not isinstance(payload, dict):
- return None
- pid = payload.get("pid")
- role = payload.get("role")
- if not isinstance(pid, int) or pid <= 0 or role not in _ROLES:
- return None
- create = payload.get("createTime")
- start = payload.get("startTime")
- port = payload.get("port")
- profiles = payload.get("profiles")
- version = payload.get("protocolVersion")
- return cls(
- role=role,
- pid=pid,
- create_time=float(create) if isinstance(create, (int, float)) else None,
- start_time=(
- int(start)
- if isinstance(start, int) and not isinstance(start, bool) and start > 0
- else None
- ),
- boot_epoch=str(payload.get("bootEpoch") or ""),
- host=str(payload.get("host") or ""),
- port=int(port) if isinstance(port, int) and 0 < port <= 65535 else None,
- protocol_version=version if isinstance(version, int) else 0,
- token_fingerprint=str(payload.get("tokenFingerprint") or ""),
- profiles=tuple(str(p) for p in profiles if isinstance(p, str)) if isinstance(profiles, list) else (),
- updated_at=str(payload.get("updatedAt") or ""),
- home=str(payload.get("home") or ""),
- )
- def host_state_dir() -> Path:
- """Per-OS-USER rendezvous dir (shared by every profile of this user)."""
- from gateway.status import _get_lock_dir
- return _get_lock_dir()
- def ensure_host_state_dir() -> Path:
- """The rendezvous dir, created owner-only (``0o700``) and tightened if it is not.
- A bare ``mkdir`` under the common ``umask 002`` leaves the dir group-writable, and the record
- inside it is what every lifecycle verb believes: a same-group process could unlink+replace it
- and choose this host's ATTACH answers (which home to dial, which profiles are "served").
- Ownership of the dir is ours, so widening is repaired rather than refused.
- """
- directory = host_state_dir()
- make_user_dir(directory, parents=True, exist_ok=True, mode=0o700)
- if sys.platform != "win32":
- with contextlib.suppress(OSError):
- if stat.S_IMODE(directory.stat().st_mode) & 0o077:
- os.chmod(directory, 0o700)
- return directory
- def _record_is_own(path: Path) -> bool:
- """True when ``path`` was written by THIS OS user inside a dir this user owns.
- ``read_record`` hands its result straight to :mod:`gateway.host_attach`, which dials the home
- it names and believes the served set it carries. A hand-written record therefore buys an
- attacker the lifecycle verdict for every profile on the host, so the file's ``st_uid`` — not
- its contents — is what makes it a record at all. Windows sets no ACLs from mode bits; there
- the dir already lives under the user's own state root.
- """
- if sys.platform == "win32":
- return True
- try:
- info = path.stat()
- parent = path.parent.stat()
- except OSError:
- return False
- uid = os.getuid() # windows-footgun: ok — unreachable on Windows (early return above)
- if info.st_uid != uid or parent.st_uid != uid:
- logger.warning(
- "ignoring host record %s: owned by uid %s (expected %s)", path, info.st_uid, uid)
- return False
- return True
- def _validated_role(role: str) -> str:
- if role not in _ROLES:
- raise ValueError(f"unknown host role: {role!r}")
- return role
- def record_path(role: str) -> Path:
- return host_state_dir() / f"host-{_validated_role(role)}.json"
- def lock_path(role: str) -> Path:
- return host_state_dir() / f"host-{_validated_role(role)}.lock"
- def token_path(role: str) -> Path:
- return host_state_dir() / f"host-{_validated_role(role)}.token"
- def token_fingerprint(token: str) -> str:
- """Short, non-reversible handle for a session token (safe to publish in the record)."""
- return hashlib.sha256(token.encode("utf-8", "replace")).hexdigest()[:16] if token else ""
- def process_create_time(pid: Optional[int] = None) -> Optional[float]:
- """Creation time of ``pid`` (default: this process); ``None`` when unknowable."""
- from hermes_cli.process_identity import _process_create_time
- return _process_create_time(pid)
- def _pid_incarnation_matches(pid: int, create_time: Optional[float]) -> Optional[bool]:
- """Reuse the spawn ledger's proof: True/False when provable, ``None`` when it cannot say."""
- from hermes_cli.process_identity import _pid_alive_matches
- return _pid_alive_matches(pid, create_time)
- # A record published before ``bootEpoch`` existed cannot name its boot. Its ``createTime`` older
- # than this boot's start (``psutil.boot_time()``, minus this slack) means EITHER a reboot (the
- # owner is gone and its PID was recycled) OR a wall-clock step since the owner started (boot time
- # is wall clock minus uptime, so a forward step moves it past a running owner's saved start;
- # #117505). Clocks cannot tell those apart, so the host lock decides: a live gateway holds it.
- _LEGACY_BOOT_SLACK_S = 120.0
- def _boot_time() -> Optional[float]:
- try:
- import psutil
- return float(psutil.boot_time())
- except Exception:
- logger.debug("boot time unreadable; legacy record age check skipped", exc_info=True)
- return None
- def _host_lock_is_free(role: str) -> Optional[bool]:
- """True when no process holds the host lock for ``role``; ``None`` when it cannot be probed.
- Read-only callers (status, dashboard) reach this, so it never creates the lock dir or file: a
- missing lock file means nobody holds the lock. The probe takes the lock for an instant, so a
- gateway claiming it in that instant gets HELD_BY_OTHER and its supervisor retries.
- """
- if owns_host_lock(role):
- return False
- from gateway.status import _probe_lock_file
- try:
- fd = os.open(str(lock_path(role)), os.O_RDWR)
- except FileNotFoundError:
- return True
- except OSError:
- logger.debug("host %s lock could not be probed", role, exc_info=True)
- return None
- return not _probe_lock_file(os.fdopen(fd, "r+", encoding="utf-8"))
- def _current_boot_epoch() -> str:
- """Identity of this boot. Empty when /proc is unreadable — a recorded epoch is then unprovable."""
- try:
- from gateway.drain_control import current_instantiation_epoch
- return current_instantiation_epoch() or ""
- except Exception:
- logger.debug("boot epoch unreadable; host liveness will not use it", exc_info=True)
- return ""
- def _record_incarnation_matches(record: HostRecord) -> Optional[bool]:
- """Whether the record still names the same live process incarnation.
- New records prefer the canonical start fingerprint: on Linux/WSL it comes from /proc start
- ticks and is immune to wall-clock shifts that move psutil.create_time(). Those ticks are
- NOT unique across reboots — they reset, and an early-boot PID reused within the drift
- tolerance looks like the dead owner. A published boot epoch that differs from this boot's is a
- mismatch; one that cannot be compared (this boot's epoch is unreadable) is unprovable, ``None``.
- A gateway record with no epoch (published before ``bootEpoch`` existed, or on any platform
- without /proc, where the epoch is always empty) whose create_time predates this boot is a
- mismatch only while nobody holds the host lock. A record with neither create_time nor
- start_time proves nothing about its pid, so it never vouches for one. Legacy records without
- a start fingerprint keep the create-time check. Do not infer identity from argv.
- """
- recorded_epoch = record.boot_epoch
- epoch = _current_boot_epoch()
- if recorded_epoch:
- if not epoch:
- return None # boots can't be compared; matching ticks alone prove nothing
- if recorded_epoch != epoch:
- return False
- if record.start_time is None and record.create_time is None:
- return False
- if record.start_time is None:
- matched = _pid_incarnation_matches(record.pid, record.create_time)
- else:
- alive = _pid_incarnation_matches(record.pid, None)
- if alive is not True:
- return alive
- from gateway.status import get_process_start_time, start_time_fingerprints_match
- current = get_process_start_time(record.pid)
- matched = None if current is None else start_time_fingerprints_match(record.start_time, current)
- if matched is not True:
- return matched
- if record.role == ROLE_GATEWAY and not recorded_epoch and record.create_time is not None:
- boot = _boot_time()
- if boot is not None and record.create_time < boot - _LEGACY_BOOT_SLACK_S:
- lock_free = _host_lock_is_free(record.role)
- if lock_free is None:
- return None
- if lock_free:
- return False
- return True
- def record_is_stale(record: Optional[HostRecord]) -> bool:
- """A record nobody may attach to: absent, unknown protocol, dead PID, or PID reuse.
- ``None`` from the liveness probe (no psutil, permission denied, an unexpected psutil error)
- means UNPROVABLE, and an unprovable record is a CANDIDATE, never an owner: it survives this
- predicate only to be handed to :func:`probe_owner`, which dials the recorded port and makes
- the owner prove itself. Treating unprovable as dead would let a second process bind a second
- port; treating it as alive *without the probe* made a record for a long-dead PID a permanent
- silent outage (attach exited 0 forever, with nothing listening).
- """
- if record is None:
- return True
- if record.protocol_version != HOST_PROTOCOL_VERSION:
- return True
- return _record_incarnation_matches(record) is False
- def liveness_is_proven(record: HostRecord) -> bool:
- """True only when the record's process-incarnation probe positively matched (never on ``None``).
- A record with neither ``createTime`` nor ``startTime`` cannot prove incarnation:
- ``_same_incarnation`` treats a missing create_time as a match, so a recycled pid would
- otherwise look alive forever. Topology already refused that shape; attach did not.
- """
- if record.create_time is None and record.start_time is None:
- return False
- return _record_incarnation_matches(record) is True
- def dial_host(record: HostRecord) -> str:
- """Address to dial for ``record``: a wildcard bind is reached over loopback."""
- host = record.host or "127.0.0.1"
- return "127.0.0.1" if host in ("0.0.0.0", "::", "*", "") else host
- def probe_owner(record: HostRecord, *, timeout: float = PROBE_TIMEOUT_S) -> Optional[dict]:
- """Make the recorded endpoint prove it is this record's owner; ``None`` when it does not.
- Two gates, both required: a bounded TCP connect (nothing listening → the owner is gone, even
- though its record and PID may still look alive during a graceful-shutdown window) and an
- identity GET authenticated with the 0600 token (a foreign listener that inherited the port
- answers the connect but cannot answer as PID N of role R).
- Returns the owner's identity payload (``pid``/``role``/``servesSpa``) on success.
- """
- if not record.port:
- return None
- import socket
- host = dial_host(record)
- try:
- with socket.create_connection((host, record.port), timeout=timeout):
- pass
- except OSError:
- logger.debug("host %s owner does not answer at %s:%s", record.role, host, record.port)
- return None
- import urllib.request
- token = read_token(record.role)
- headers = {"X-Hermes-Token": token, "Authorization": f"Bearer {token}"} if token else {}
- request = urllib.request.Request(
- f"http://{host}:{record.port}{HOST_IDENTITY_PATH}", headers=headers)
- try:
- with urllib.request.urlopen(request, timeout=timeout) as response:
- if response.status != 200:
- return None
- payload = json.loads(response.read(65536).decode("utf-8", "replace"))
- except Exception:
- logger.debug("host %s identity probe failed at %s:%s", record.role, host, record.port, exc_info=True)
- return None
- if not isinstance(payload, dict):
- return None
- if payload.get("role") != record.role or payload.get("pid") != record.pid:
- return None
- return payload
- def read_record(role: str, *, include_stale: bool = False) -> Optional[HostRecord]:
- """Published record for ``role``; ``None`` when absent, foreign, corrupt or (by default) stale."""
- path = record_path(role)
- if not _record_is_own(path):
- return None
- try:
- raw = path.read_text(encoding="utf-8-sig")
- except (OSError, UnicodeDecodeError):
- return None
- try:
- record = HostRecord.from_json(json.loads(raw))
- except (ValueError, TypeError):
- return None
- if record is None:
- return None
- return record if include_stale or not record_is_stale(record) else None
- def read_token(role: str) -> str:
- """Owner-written session token for ``role`` (``""`` when absent/unreadable).
- This is the handle an attaching client uses when the backend is auth-gated and therefore
- withholds its token from an unauthenticated ``GET /``. Confidentiality is enforced at WRITE
- time by :func:`_write_private_text` (POSIX ``0o600``; an owner+SYSTEM-only protected DACL on
- Windows, where mode bits set no ACLs at all). Reading it is therefore evidence of, not proof
- of, same-OS-user authority — the authority boundary the host lock is scoped to.
- """
- try:
- return token_path(role).read_text(encoding="utf-8-sig").strip()
- except (OSError, UnicodeDecodeError):
- return ""
- def record_token_is_consistent(record: HostRecord) -> bool:
- """Does the on-disk token still hash to the record's fingerprint?
- A record published without a token (the gateway) has an empty fingerprint and is consistent
- by definition. A mismatch means the record and the token file come from different
- incarnations (a torn restart) — discovery must not attach with a token the owner rejects.
- """
- if not record.token_fingerprint:
- return True
- return token_fingerprint(read_token(record.role)) == record.token_fingerprint
- def _write_private_text(path: Path, text: str) -> None:
- """Create/replace ``path`` with owner-only content.
- POSIX: ``0o600`` via tmp + atomic replace. Windows: mode bits set NO ACLs, so the same
- ``os.open`` would leave a live session token readable by whatever the inherited DACL grants;
- the SSH runtime's protected owner+SYSTEM DACL writer is the repo's primitive for exactly this
- credential class and is reused here. It also replaces in place, because ``os.replace`` onto a
- token file another process still holds open fails on Windows.
- """
- if sys.platform == "win32":
- from hermes_cli.windows_ssh_runtime import write_private_file
- write_private_file(path, text.encode("utf-8"))
- return
- tmp = path.with_name(path.name + ".tmp")
- fd = os.open(str(tmp), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
- try:
- with os.fdopen(fd, "w", encoding="utf-8") as handle:
- handle.write(text)
- handle.flush()
- with contextlib.suppress(OSError):
- os.fsync(handle.fileno())
- except BaseException:
- with contextlib.suppress(OSError):
- os.unlink(str(tmp))
- raise
- os.replace(str(tmp), str(path))
- class HostLockOutcome(enum.Enum):
- """Why a host-lock claim ended the way it did.
- ``COULD_NOT_OPEN`` is NOT contention: a read-only/undeletable lock dir (EROFS, EACCES,
- ENOSPC) collapsed into the same "another owner holds it" log and sent operators hunting a
- second gateway that never existed.
- """
- ACQUIRED = "acquired"
- HELD_BY_OTHER = "held-by-other"
- COULD_NOT_OPEN = "could-not-open"
- def _lock_key(role: str) -> tuple[str, str]:
- path = lock_path(role)
- with contextlib.suppress(OSError):
- return (role, os.path.abspath(str(path)))
- return (role, str(path))
- def claim_host_lock(role: str) -> tuple[HostLockOutcome, Optional[OSError]]:
- """Take the host-wide lock for ``role``. Idempotent per (role, lock path).
- Returns the outcome and, for ``COULD_NOT_OPEN``, the OSError that explains it.
- """
- role = _validated_role(role)
- key = _lock_key(role)
- if _lock_handles.get(key) is not None:
- return (HostLockOutcome.ACQUIRED, None)
- from gateway.status import _try_acquire_file_lock
- path = Path(key[1])
- try:
- ensure_host_state_dir()
- handle = open(path, "a+", encoding="utf-8")
- except OSError as exc:
- logger.debug("host %s lock could not be opened at %s", role, path, exc_info=True)
- return (HostLockOutcome.COULD_NOT_OPEN, exc)
- if not _try_acquire_file_lock(handle):
- with contextlib.suppress(OSError):
- handle.close()
- return (HostLockOutcome.HELD_BY_OTHER, None)
- _lock_handles[key] = handle
- return (HostLockOutcome.ACQUIRED, None)
- def release_host_lock(role: str) -> None:
- """Release the host lock for ``role`` when this process holds it."""
- handle = _lock_handles.pop(_lock_key(_validated_role(role)), None)
- if handle is None:
- return
- from gateway.status import _release_file_lock
- _release_file_lock(handle)
- with contextlib.suppress(OSError):
- handle.close()
- def owns_host_lock(role: str) -> bool:
- """True when THIS process holds the host lock for ``role`` (re-probing our own flock lies)."""
- return _lock_handles.get(_lock_key(_validated_role(role))) is not None
- def host_lock_is_held(role: str) -> Optional[bool]:
- """Whether any process currently holds the host lock for ``role``.
- ``True`` / ``False`` when the flock can be observed; ``None`` when the lock file cannot
- be opened. Never retains a lock. This process's handle is authoritative: re-probing our
- own flock succeeds on POSIX, so a second open would lie.
- """
- role = _validated_role(role)
- if owns_host_lock(role):
- return True
- path = lock_path(role)
- if not path.exists():
- return False
- try:
- handle = open(path, "a+", encoding="utf-8")
- except OSError:
- return None
- from gateway.status import _probe_lock_file
- return _probe_lock_file(handle)
- def record_names_live_owner(record: HostRecord) -> bool:
- """True when ``record``'s incarnation is proven AND its host lock is not observed free.
- The publisher holds the host lock for its whole life and the OS drops it on death, so a free
- flock falsifies a record whose pid merely looks alive (recycle, start-tick collision after a
- reboot — #138343, #133744). Every reader that trusts a record as a running owner uses this.
- """
- if not liveness_is_proven(record):
- return False
- if host_lock_is_held(record.role) is False:
- logger.debug("host %s record names pid %s but the host lock is free", record.role, record.pid)
- return False
- return True
- def publish_record(
- role: str,
- *,
- host: str = "",
- port: Optional[int] = None,
- profiles: Sequence[str] = (),
- token: Optional[str] = None,
- home: str = "",
- ) -> Optional[HostRecord]:
- """Publish this process as the host owner of ``role``. ``None`` when the write failed.
- ``token`` (serve) is persisted 0600 next to the record and only its fingerprint is published.
- ``home`` is the launch HERMES_HOME — the key an attaching client needs to reach this owner's
- control socket.
- """
- role = _validated_role(role)
- from gateway.status import get_process_start_time
- record = HostRecord(
- role=role,
- pid=os.getpid(),
- create_time=process_create_time(),
- start_time=get_process_start_time(os.getpid()),
- boot_epoch=_current_boot_epoch(),
- host=str(host or ""),
- port=int(port) if isinstance(port, int) and port > 0 else None,
- protocol_version=HOST_PROTOCOL_VERSION,
- token_fingerprint=token_fingerprint(token or ""),
- profiles=tuple(str(p) for p in profiles),
- updated_at=datetime.now(UTC).isoformat(),
- home=str(home or ""),
- )
- try:
- ensure_host_state_dir()
- if token:
- # No record without its token: publishing one an attaching client cannot
- # authenticate against would degrade to a silent "attach refused forever".
- _write_private_text(token_path(role), token)
- atomic_json_write(record_path(role), record.to_json(), mode=0o600)
- except OSError:
- logger.warning("host %s record could not be published; discovery will not find it", role, exc_info=True)
- return None
- _invalidate_attach_cache()
- return record
- def _invalidate_attach_cache() -> None:
- """Drop :mod:`gateway.host_attach`'s memo: the record it summarises just changed."""
- with contextlib.suppress(Exception):
- from gateway.host_attach import invalidate_host_gateway_cache
- invalidate_host_gateway_cache()
- def discard_dead_record(role: str) -> bool:
- """Retract the record for ``role`` when its owner is provably gone; True when one was removed.
- A confirmed stop must retract the record too. Leaving it made ``gateway restart --all`` a
- silent no-op: the re-entered ``gateway run`` read the corpse's record, decided ATTACH and
- exited 0, so the host ended up with no gateway at all.
- """
- role = _validated_role(role)
- record = read_record(role, include_stale=True)
- if record is None:
- return False
- guard = None
- if record.pid != os.getpid() and _record_incarnation_matches(record) is not False:
- # A recycled pid can still look like the same incarnation. The publisher holds the
- # host lock for its life, so winning it proves the recorded owner is gone. Hold it
- # across the re-read and unlink: a free-lock probe released first let a new gateway
- # claim and publish in the gap, and we deleted ITS record and token. A private handle,
- # not claim_host_lock: cleanup must not enter this process's own claim/release ledger.
- if owns_host_lock(role) or (guard := _hold_lock_briefly(role)) is None:
- return False
- try:
- if guard is not None and read_record(role, include_stale=True) is None:
- return False
- for path in (record_path(role), token_path(role)):
- with contextlib.suppress(OSError):
- path.unlink(missing_ok=True)
- finally:
- if guard is not None:
- from gateway.status import _release_file_lock
- _release_file_lock(guard)
- with contextlib.suppress(OSError):
- guard.close()
- _invalidate_attach_cache()
- return True
- def _hold_lock_briefly(role: str):
- """Open and flock ``role``'s host lock without registering it; ``None`` when not won."""
- from gateway.status import _try_acquire_file_lock
- path = lock_path(role)
- try:
- handle = open(path, "a+", encoding="utf-8")
- except OSError:
- return None
- if _try_acquire_file_lock(handle):
- return handle
- with contextlib.suppress(OSError):
- handle.close()
- return None
- def clear_record(role: str) -> None:
- """Remove this process's record + token on exit (never another owner's)."""
- role = _validated_role(role)
- existing = read_record(role, include_stale=True)
- if existing is not None and existing.pid != os.getpid():
- return
- for path in (record_path(role), token_path(role)):
- with contextlib.suppress(OSError):
- path.unlink(missing_ok=True)
- _invalidate_attach_cache()
- # Roles this process must clean up on the way out, and the signal handlers we prepended.
- _cleanup_roles: list[str] = []
- _prev_signal_handlers: dict[int, Any] = {}
- def _cleanup_role(role: str) -> None:
- with contextlib.suppress(Exception):
- clear_record(role)
- with contextlib.suppress(Exception):
- release_host_lock(role)
- def _handle_terminating_signal(signum, frame) -> None:
- """Drop the record + token, then hand off to the handler we prepended to."""
- import signal as _signal
- for role in tuple(_cleanup_roles):
- _cleanup_role(role)
- prev = _prev_signal_handlers.get(signum)
- if callable(prev):
- prev(signum, frame)
- return
- if prev is _signal.SIG_IGN:
- return
- with contextlib.suppress(Exception):
- _signal.signal(signum, _signal.SIG_DFL)
- os.kill(os.getpid(), signum)
- raise SystemExit(128 + int(signum))
- def cleanup_on_exit(role: str) -> None:
- """Clear ``role``'s record + token and release its lock on exit — SIGTERM included.
- ``atexit`` alone is false advertising for the NORMAL stop: systemd stop, ``docker stop`` and
- the update relaunch all send SIGTERM, and every terminating SIGTERM path here ends in the
- default disposition (uvicorn's ``capture_signals`` re-raises it after its graceful shutdown),
- which kills the process without running ``atexit``. The record then outlived its process and
- the 0600 token kept a LIVE session token on disk indefinitely.
- The handler only PREPENDS cleanup: whatever handler was installed before (uvicorn's graceful
- shutdown, the exit-flush chain, a supervisor's) still runs, so the shutdown sequence is
- unchanged.
- """
- import atexit
- import signal as _signal
- import threading
- role = _validated_role(role)
- if role not in _cleanup_roles:
- _cleanup_roles.append(role)
- atexit.register(_cleanup_role, role)
- if threading.current_thread() is not threading.main_thread():
- return
- for name in ("SIGTERM", "SIGBREAK"):
- signum = getattr(_signal, name, None)
- if signum is None or signum in _prev_signal_handlers:
- continue
- with contextlib.suppress(ValueError, OSError, RuntimeError):
- prev = _signal.getsignal(signum)
- _signal.signal(signum, _handle_terminating_signal)
- _prev_signal_handlers[signum] = prev
- def _multiplex_profiles_enabled() -> bool:
- """Will THIS process multiplex? An explicit ``true`` and an unset key both say yes, and an
- explicit ``false`` is RETIRED (``hermes_cli.gateway_multiplex_mode``) — it is warned about and
- ignored at boot, so it must not make the claim-time record advertise a narrower roster than
- the process actually serves. Reading it here was the last place the retired flag still decided
- topology, and it made CLI/dashboard report "standalone, serving default" while the runtime
- multiplexed. The RUNTIME verdict (a boot-time guard refusal) narrows the record afterwards, in
- ``gateway.run._refresh_host_gateway_record``, which republishes the SETTLED set.
- """
- return True
- def served_profiles(*, multiplex: Optional[bool] = None) -> tuple[str, ...]:
- """Profiles this process multiplexes; ``()`` when the roster cannot be read.
- ``multiplex`` defaults to what this process's own config says. Hard-coding ``True`` here
- published a record claiming EVERY profile from a gateway that would only ever serve its own,
- and a second profile's supervised unit then stood down against a set nobody serves.
- """
- try:
- from hermes_cli.profiles import profiles_to_serve
- enabled = _multiplex_profiles_enabled() if multiplex is None else bool(multiplex)
- return tuple(name for name, _ in profiles_to_serve(multiplex=enabled))
- except Exception:
- logger.debug("served profile roster unavailable", exc_info=True)
- return ()
- def describe(record: HostRecord) -> str:
- """One-line human description used by attach messages and conflict logs."""
- where = f"{record.host or '127.0.0.1'}:{record.port}" if record.port else "no bound port"
- profiles = ", ".join(record.profiles) if record.profiles else "unknown"
- return f"PID {record.pid} ({where}; profiles: {profiles})"
|