"""Host-wide singleton rendezvous: one lock + one record per ROLE per OS user. Multiplex-only (Teknium ruling): exactly ONE ``hermes serve`` and ONE ``hermes gateway run`` per host, each multiplexing every profile. The per-``HERMES_HOME`` gateway lock/PID files (``gateway.status``) cannot express that — N profiles are N homes, so N processes each take their own flock and none of them ever sees the others. This module adds the missing layer: * a **host lock** (flock/``msvcrt``) held for the lifetime of the winning process, and * a **rendezvous record** the winner publishes so a second invocation can find it, prove it is the same live process, and ATTACH instead of binding a second port. Both live in :func:`gateway.status._get_lock_dir` — the only cross-profile lock root already in the tree (``$HERMES_GATEWAY_LOCK_DIR`` else ``$XDG_STATE_HOME/hermes/gateway-locks``), which scopes to the **OS user**. That is the correct granularity: separate OS users have separate ``$HOME``s, separate ``~/.hermes`` profile roots, separate ports-by-convention and separate credentials, so "one per host" means "one per host per OS user". **Staleness is proved, never assumed.** New records pair the PID with the repository's stable process-start fingerprint (boot-relative on Linux/WSL); ``createTime`` remains the legacy fallback. A dead PID or a mismatched process incarnation is STALE and ignored — an attaching client must never dial a recycled PID's port. **Relationship to ``spawn-ledger.json``** (``hermes_cli/process_identity.py``): the ledger stays the append-only machine roster of every long-lived Hermes process (Desktop's attach ladder reads it) and is still written unchanged. It cannot be the host record: it has no lock, no single-writer semantics, no removal on exit, and no place to publish a protocol version or an authentication handle. The record here is authoritative for "who owns this host role"; the ledger remains authoritative for "what is running". Both are written. New host records pair the ledger's live-PID proof with ``gateway.status``'s canonical process-start fingerprint; legacy records without that fingerprint keep the old ``(pid, create_time)`` proof. """ from __future__ import annotations import contextlib import enum import hashlib import json import logging import os import stat import sys from dataclasses import dataclass from datetime import datetime, timezone, UTC from pathlib import Path from typing import Any, Optional, Sequence from hermes_dir_acl import make_user_dir from utils import atomic_json_write logger = logging.getLogger(__name__) #: Bumped when the record's shape or the attach handshake changes incompatibly. A reader that #: does not recognise the version refuses to attach instead of guessing. HOST_PROTOCOL_VERSION = 1 #: Token-gated endpoint the owner answers with its own identity. An attaching client dials the #: RECORDED port and only attaches when the answer is the recorded owner — a record alone proves #: nothing about who holds that port right now. HOST_IDENTITY_PATH = "/api/host/identity" #: Bounded: a closed port, a black-holed one or a foreign listener must cost a second, not a hang. PROBE_TIMEOUT_S = 2.0 ROLE_GATEWAY = "gateway" ROLE_SERVE = "serve" #: A Desktop-owned pool child (loopback, random port, per-profile lifecycle). It is NOT a host #: owner — the attach/refuse ladder reads ``ROLE_SERVE`` only, so a supervised public dashboard #: never stands down behind it (#119824) — but ``hermes plugins install`` from a terminal still #: has to reach the backend hosting the open chats (#119644), and this record + 0600 token is #: how it dials one on a Desktop-only box. ROLE_DESKTOP_SERVE = "desktop-serve" _ROLES = (ROLE_GATEWAY, ROLE_SERVE, ROLE_DESKTOP_SERVE) # Open lock handles, keyed by (role, resolved lock path): the OS releases the flock when this # process dies, which is what makes a crashed owner's host lock re-acquirable without a reaper. # The PATH is part of the key because the lock dir is env-derived (HERMES_GATEWAY_LOCK_DIR): # keyed by role alone, a second call after the dir changed returned "already held" without ever # creating the new lock file, so owns_host_lock() lied and one pytest process leaked the handle # across tests. _lock_handles: dict[tuple[str, str], Any] = {} @dataclass(frozen=True) class HostRecord: """A published host-role owner. ``profiles`` is the SERVED set, not the launch profile.""" role: str pid: int create_time: Optional[float] host: str port: Optional[int] protocol_version: int token_fingerprint: str profiles: tuple[str, ...] updated_at: str #: HERMES_HOME the owner was launched from. The attach channel (``gateway.control_socket``) is #: keyed by home, so without it a client can only guess the default root — wrong as soon as a #: named profile launches the host process. Absent in records written before this field; added #: WITHOUT a protocol bump on purpose, because a bump would make every live owner's record read #: as stale and a second gateway would start. home: str = "" #: Stable process-start fingerprint. On Linux/WSL this is boot-relative /proc start ticks, #: so a wall-clock resync cannot make a live owner look like a recycled PID. Optional for #: records written by older Hermes versions, which still fall back to create_time. #: NOT unique across reboots — pair it with ``boot_epoch``. start_time: Optional[int] = None #: Instantiation epoch (boot id + PID 1 start). Distinguishes a pre-reboot owner from an #: early-boot process that reused its PID and landed inside the start-tick drift tolerance. #: Absent on older records; added WITHOUT a protocol bump, because a bump would make every #: live owner's record read as stale and a second gateway would start. boot_epoch: str = "" def to_json(self) -> dict[str, Any]: return { "role": self.role, "home": self.home, "pid": self.pid, "createTime": self.create_time, "startTime": self.start_time, "bootEpoch": self.boot_epoch, "host": self.host, "port": self.port, "protocolVersion": self.protocol_version, "tokenFingerprint": self.token_fingerprint, "profiles": list(self.profiles), "updatedAt": self.updated_at, } @classmethod def from_json(cls, payload: Any) -> Optional[HostRecord]: if not isinstance(payload, dict): return None pid = payload.get("pid") role = payload.get("role") if not isinstance(pid, int) or pid <= 0 or role not in _ROLES: return None create = payload.get("createTime") start = payload.get("startTime") port = payload.get("port") profiles = payload.get("profiles") version = payload.get("protocolVersion") return cls( role=role, pid=pid, create_time=float(create) if isinstance(create, (int, float)) else None, start_time=( int(start) if isinstance(start, int) and not isinstance(start, bool) and start > 0 else None ), boot_epoch=str(payload.get("bootEpoch") or ""), host=str(payload.get("host") or ""), port=int(port) if isinstance(port, int) and 0 < port <= 65535 else None, protocol_version=version if isinstance(version, int) else 0, token_fingerprint=str(payload.get("tokenFingerprint") or ""), profiles=tuple(str(p) for p in profiles if isinstance(p, str)) if isinstance(profiles, list) else (), updated_at=str(payload.get("updatedAt") or ""), home=str(payload.get("home") or ""), ) def host_state_dir() -> Path: """Per-OS-USER rendezvous dir (shared by every profile of this user).""" from gateway.status import _get_lock_dir return _get_lock_dir() def ensure_host_state_dir() -> Path: """The rendezvous dir, created owner-only (``0o700``) and tightened if it is not. A bare ``mkdir`` under the common ``umask 002`` leaves the dir group-writable, and the record inside it is what every lifecycle verb believes: a same-group process could unlink+replace it and choose this host's ATTACH answers (which home to dial, which profiles are "served"). Ownership of the dir is ours, so widening is repaired rather than refused. """ directory = host_state_dir() make_user_dir(directory, parents=True, exist_ok=True, mode=0o700) if sys.platform != "win32": with contextlib.suppress(OSError): if stat.S_IMODE(directory.stat().st_mode) & 0o077: os.chmod(directory, 0o700) return directory def _record_is_own(path: Path) -> bool: """True when ``path`` was written by THIS OS user inside a dir this user owns. ``read_record`` hands its result straight to :mod:`gateway.host_attach`, which dials the home it names and believes the served set it carries. A hand-written record therefore buys an attacker the lifecycle verdict for every profile on the host, so the file's ``st_uid`` — not its contents — is what makes it a record at all. Windows sets no ACLs from mode bits; there the dir already lives under the user's own state root. """ if sys.platform == "win32": return True try: info = path.stat() parent = path.parent.stat() except OSError: return False uid = os.getuid() # windows-footgun: ok — unreachable on Windows (early return above) if info.st_uid != uid or parent.st_uid != uid: logger.warning( "ignoring host record %s: owned by uid %s (expected %s)", path, info.st_uid, uid) return False return True def _validated_role(role: str) -> str: if role not in _ROLES: raise ValueError(f"unknown host role: {role!r}") return role def record_path(role: str) -> Path: return host_state_dir() / f"host-{_validated_role(role)}.json" def lock_path(role: str) -> Path: return host_state_dir() / f"host-{_validated_role(role)}.lock" def token_path(role: str) -> Path: return host_state_dir() / f"host-{_validated_role(role)}.token" def token_fingerprint(token: str) -> str: """Short, non-reversible handle for a session token (safe to publish in the record).""" return hashlib.sha256(token.encode("utf-8", "replace")).hexdigest()[:16] if token else "" def process_create_time(pid: Optional[int] = None) -> Optional[float]: """Creation time of ``pid`` (default: this process); ``None`` when unknowable.""" from hermes_cli.process_identity import _process_create_time return _process_create_time(pid) def _pid_incarnation_matches(pid: int, create_time: Optional[float]) -> Optional[bool]: """Reuse the spawn ledger's proof: True/False when provable, ``None`` when it cannot say.""" from hermes_cli.process_identity import _pid_alive_matches return _pid_alive_matches(pid, create_time) # A record published before ``bootEpoch`` existed cannot name its boot. Its ``createTime`` older # than this boot's start (``psutil.boot_time()``, minus this slack) means EITHER a reboot (the # owner is gone and its PID was recycled) OR a wall-clock step since the owner started (boot time # is wall clock minus uptime, so a forward step moves it past a running owner's saved start; # #117505). Clocks cannot tell those apart, so the host lock decides: a live gateway holds it. _LEGACY_BOOT_SLACK_S = 120.0 def _boot_time() -> Optional[float]: try: import psutil return float(psutil.boot_time()) except Exception: logger.debug("boot time unreadable; legacy record age check skipped", exc_info=True) return None def _host_lock_is_free(role: str) -> Optional[bool]: """True when no process holds the host lock for ``role``; ``None`` when it cannot be probed. Read-only callers (status, dashboard) reach this, so it never creates the lock dir or file: a missing lock file means nobody holds the lock. The probe takes the lock for an instant, so a gateway claiming it in that instant gets HELD_BY_OTHER and its supervisor retries. """ if owns_host_lock(role): return False from gateway.status import _probe_lock_file try: fd = os.open(str(lock_path(role)), os.O_RDWR) except FileNotFoundError: return True except OSError: logger.debug("host %s lock could not be probed", role, exc_info=True) return None return not _probe_lock_file(os.fdopen(fd, "r+", encoding="utf-8")) def _current_boot_epoch() -> str: """Identity of this boot. Empty when /proc is unreadable — a recorded epoch is then unprovable.""" try: from gateway.drain_control import current_instantiation_epoch return current_instantiation_epoch() or "" except Exception: logger.debug("boot epoch unreadable; host liveness will not use it", exc_info=True) return "" def _record_incarnation_matches(record: HostRecord) -> Optional[bool]: """Whether the record still names the same live process incarnation. New records prefer the canonical start fingerprint: on Linux/WSL it comes from /proc start ticks and is immune to wall-clock shifts that move psutil.create_time(). Those ticks are NOT unique across reboots — they reset, and an early-boot PID reused within the drift tolerance looks like the dead owner. A published boot epoch that differs from this boot's is a mismatch; one that cannot be compared (this boot's epoch is unreadable) is unprovable, ``None``. A gateway record with no epoch (published before ``bootEpoch`` existed, or on any platform without /proc, where the epoch is always empty) whose create_time predates this boot is a mismatch only while nobody holds the host lock. A record with neither create_time nor start_time proves nothing about its pid, so it never vouches for one. Legacy records without a start fingerprint keep the create-time check. Do not infer identity from argv. """ recorded_epoch = record.boot_epoch epoch = _current_boot_epoch() if recorded_epoch: if not epoch: return None # boots can't be compared; matching ticks alone prove nothing if recorded_epoch != epoch: return False if record.start_time is None and record.create_time is None: return False if record.start_time is None: matched = _pid_incarnation_matches(record.pid, record.create_time) else: alive = _pid_incarnation_matches(record.pid, None) if alive is not True: return alive from gateway.status import get_process_start_time, start_time_fingerprints_match current = get_process_start_time(record.pid) matched = None if current is None else start_time_fingerprints_match(record.start_time, current) if matched is not True: return matched if record.role == ROLE_GATEWAY and not recorded_epoch and record.create_time is not None: boot = _boot_time() if boot is not None and record.create_time < boot - _LEGACY_BOOT_SLACK_S: lock_free = _host_lock_is_free(record.role) if lock_free is None: return None if lock_free: return False return True def record_is_stale(record: Optional[HostRecord]) -> bool: """A record nobody may attach to: absent, unknown protocol, dead PID, or PID reuse. ``None`` from the liveness probe (no psutil, permission denied, an unexpected psutil error) means UNPROVABLE, and an unprovable record is a CANDIDATE, never an owner: it survives this predicate only to be handed to :func:`probe_owner`, which dials the recorded port and makes the owner prove itself. Treating unprovable as dead would let a second process bind a second port; treating it as alive *without the probe* made a record for a long-dead PID a permanent silent outage (attach exited 0 forever, with nothing listening). """ if record is None: return True if record.protocol_version != HOST_PROTOCOL_VERSION: return True return _record_incarnation_matches(record) is False def liveness_is_proven(record: HostRecord) -> bool: """True only when the record's process-incarnation probe positively matched (never on ``None``). A record with neither ``createTime`` nor ``startTime`` cannot prove incarnation: ``_same_incarnation`` treats a missing create_time as a match, so a recycled pid would otherwise look alive forever. Topology already refused that shape; attach did not. """ if record.create_time is None and record.start_time is None: return False return _record_incarnation_matches(record) is True def dial_host(record: HostRecord) -> str: """Address to dial for ``record``: a wildcard bind is reached over loopback.""" host = record.host or "127.0.0.1" return "127.0.0.1" if host in ("0.0.0.0", "::", "*", "") else host def probe_owner(record: HostRecord, *, timeout: float = PROBE_TIMEOUT_S) -> Optional[dict]: """Make the recorded endpoint prove it is this record's owner; ``None`` when it does not. Two gates, both required: a bounded TCP connect (nothing listening → the owner is gone, even though its record and PID may still look alive during a graceful-shutdown window) and an identity GET authenticated with the 0600 token (a foreign listener that inherited the port answers the connect but cannot answer as PID N of role R). Returns the owner's identity payload (``pid``/``role``/``servesSpa``) on success. """ if not record.port: return None import socket host = dial_host(record) try: with socket.create_connection((host, record.port), timeout=timeout): pass except OSError: logger.debug("host %s owner does not answer at %s:%s", record.role, host, record.port) return None import urllib.request token = read_token(record.role) headers = {"X-Hermes-Token": token, "Authorization": f"Bearer {token}"} if token else {} request = urllib.request.Request( f"http://{host}:{record.port}{HOST_IDENTITY_PATH}", headers=headers) try: with urllib.request.urlopen(request, timeout=timeout) as response: if response.status != 200: return None payload = json.loads(response.read(65536).decode("utf-8", "replace")) except Exception: logger.debug("host %s identity probe failed at %s:%s", record.role, host, record.port, exc_info=True) return None if not isinstance(payload, dict): return None if payload.get("role") != record.role or payload.get("pid") != record.pid: return None return payload def read_record(role: str, *, include_stale: bool = False) -> Optional[HostRecord]: """Published record for ``role``; ``None`` when absent, foreign, corrupt or (by default) stale.""" path = record_path(role) if not _record_is_own(path): return None try: raw = path.read_text(encoding="utf-8-sig") except (OSError, UnicodeDecodeError): return None try: record = HostRecord.from_json(json.loads(raw)) except (ValueError, TypeError): return None if record is None: return None return record if include_stale or not record_is_stale(record) else None def read_token(role: str) -> str: """Owner-written session token for ``role`` (``""`` when absent/unreadable). This is the handle an attaching client uses when the backend is auth-gated and therefore withholds its token from an unauthenticated ``GET /``. Confidentiality is enforced at WRITE time by :func:`_write_private_text` (POSIX ``0o600``; an owner+SYSTEM-only protected DACL on Windows, where mode bits set no ACLs at all). Reading it is therefore evidence of, not proof of, same-OS-user authority — the authority boundary the host lock is scoped to. """ try: return token_path(role).read_text(encoding="utf-8-sig").strip() except (OSError, UnicodeDecodeError): return "" def record_token_is_consistent(record: HostRecord) -> bool: """Does the on-disk token still hash to the record's fingerprint? A record published without a token (the gateway) has an empty fingerprint and is consistent by definition. A mismatch means the record and the token file come from different incarnations (a torn restart) — discovery must not attach with a token the owner rejects. """ if not record.token_fingerprint: return True return token_fingerprint(read_token(record.role)) == record.token_fingerprint def _write_private_text(path: Path, text: str) -> None: """Create/replace ``path`` with owner-only content. POSIX: ``0o600`` via tmp + atomic replace. Windows: mode bits set NO ACLs, so the same ``os.open`` would leave a live session token readable by whatever the inherited DACL grants; the SSH runtime's protected owner+SYSTEM DACL writer is the repo's primitive for exactly this credential class and is reused here. It also replaces in place, because ``os.replace`` onto a token file another process still holds open fails on Windows. """ if sys.platform == "win32": from hermes_cli.windows_ssh_runtime import write_private_file write_private_file(path, text.encode("utf-8")) return tmp = path.with_name(path.name + ".tmp") fd = os.open(str(tmp), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) try: with os.fdopen(fd, "w", encoding="utf-8") as handle: handle.write(text) handle.flush() with contextlib.suppress(OSError): os.fsync(handle.fileno()) except BaseException: with contextlib.suppress(OSError): os.unlink(str(tmp)) raise os.replace(str(tmp), str(path)) class HostLockOutcome(enum.Enum): """Why a host-lock claim ended the way it did. ``COULD_NOT_OPEN`` is NOT contention: a read-only/undeletable lock dir (EROFS, EACCES, ENOSPC) collapsed into the same "another owner holds it" log and sent operators hunting a second gateway that never existed. """ ACQUIRED = "acquired" HELD_BY_OTHER = "held-by-other" COULD_NOT_OPEN = "could-not-open" def _lock_key(role: str) -> tuple[str, str]: path = lock_path(role) with contextlib.suppress(OSError): return (role, os.path.abspath(str(path))) return (role, str(path)) def claim_host_lock(role: str) -> tuple[HostLockOutcome, Optional[OSError]]: """Take the host-wide lock for ``role``. Idempotent per (role, lock path). Returns the outcome and, for ``COULD_NOT_OPEN``, the OSError that explains it. """ role = _validated_role(role) key = _lock_key(role) if _lock_handles.get(key) is not None: return (HostLockOutcome.ACQUIRED, None) from gateway.status import _try_acquire_file_lock path = Path(key[1]) try: ensure_host_state_dir() handle = open(path, "a+", encoding="utf-8") except OSError as exc: logger.debug("host %s lock could not be opened at %s", role, path, exc_info=True) return (HostLockOutcome.COULD_NOT_OPEN, exc) if not _try_acquire_file_lock(handle): with contextlib.suppress(OSError): handle.close() return (HostLockOutcome.HELD_BY_OTHER, None) _lock_handles[key] = handle return (HostLockOutcome.ACQUIRED, None) def release_host_lock(role: str) -> None: """Release the host lock for ``role`` when this process holds it.""" handle = _lock_handles.pop(_lock_key(_validated_role(role)), None) if handle is None: return from gateway.status import _release_file_lock _release_file_lock(handle) with contextlib.suppress(OSError): handle.close() def owns_host_lock(role: str) -> bool: """True when THIS process holds the host lock for ``role`` (re-probing our own flock lies).""" return _lock_handles.get(_lock_key(_validated_role(role))) is not None def host_lock_is_held(role: str) -> Optional[bool]: """Whether any process currently holds the host lock for ``role``. ``True`` / ``False`` when the flock can be observed; ``None`` when the lock file cannot be opened. Never retains a lock. This process's handle is authoritative: re-probing our own flock succeeds on POSIX, so a second open would lie. """ role = _validated_role(role) if owns_host_lock(role): return True path = lock_path(role) if not path.exists(): return False try: handle = open(path, "a+", encoding="utf-8") except OSError: return None from gateway.status import _probe_lock_file return _probe_lock_file(handle) def record_names_live_owner(record: HostRecord) -> bool: """True when ``record``'s incarnation is proven AND its host lock is not observed free. The publisher holds the host lock for its whole life and the OS drops it on death, so a free flock falsifies a record whose pid merely looks alive (recycle, start-tick collision after a reboot — #138343, #133744). Every reader that trusts a record as a running owner uses this. """ if not liveness_is_proven(record): return False if host_lock_is_held(record.role) is False: logger.debug("host %s record names pid %s but the host lock is free", record.role, record.pid) return False return True def publish_record( role: str, *, host: str = "", port: Optional[int] = None, profiles: Sequence[str] = (), token: Optional[str] = None, home: str = "", ) -> Optional[HostRecord]: """Publish this process as the host owner of ``role``. ``None`` when the write failed. ``token`` (serve) is persisted 0600 next to the record and only its fingerprint is published. ``home`` is the launch HERMES_HOME — the key an attaching client needs to reach this owner's control socket. """ role = _validated_role(role) from gateway.status import get_process_start_time record = HostRecord( role=role, pid=os.getpid(), create_time=process_create_time(), start_time=get_process_start_time(os.getpid()), boot_epoch=_current_boot_epoch(), host=str(host or ""), port=int(port) if isinstance(port, int) and port > 0 else None, protocol_version=HOST_PROTOCOL_VERSION, token_fingerprint=token_fingerprint(token or ""), profiles=tuple(str(p) for p in profiles), updated_at=datetime.now(UTC).isoformat(), home=str(home or ""), ) try: ensure_host_state_dir() if token: # No record without its token: publishing one an attaching client cannot # authenticate against would degrade to a silent "attach refused forever". _write_private_text(token_path(role), token) atomic_json_write(record_path(role), record.to_json(), mode=0o600) except OSError: logger.warning("host %s record could not be published; discovery will not find it", role, exc_info=True) return None _invalidate_attach_cache() return record def _invalidate_attach_cache() -> None: """Drop :mod:`gateway.host_attach`'s memo: the record it summarises just changed.""" with contextlib.suppress(Exception): from gateway.host_attach import invalidate_host_gateway_cache invalidate_host_gateway_cache() def discard_dead_record(role: str) -> bool: """Retract the record for ``role`` when its owner is provably gone; True when one was removed. A confirmed stop must retract the record too. Leaving it made ``gateway restart --all`` a silent no-op: the re-entered ``gateway run`` read the corpse's record, decided ATTACH and exited 0, so the host ended up with no gateway at all. """ role = _validated_role(role) record = read_record(role, include_stale=True) if record is None: return False guard = None if record.pid != os.getpid() and _record_incarnation_matches(record) is not False: # A recycled pid can still look like the same incarnation. The publisher holds the # host lock for its life, so winning it proves the recorded owner is gone. Hold it # across the re-read and unlink: a free-lock probe released first let a new gateway # claim and publish in the gap, and we deleted ITS record and token. A private handle, # not claim_host_lock: cleanup must not enter this process's own claim/release ledger. if owns_host_lock(role) or (guard := _hold_lock_briefly(role)) is None: return False try: if guard is not None and read_record(role, include_stale=True) is None: return False for path in (record_path(role), token_path(role)): with contextlib.suppress(OSError): path.unlink(missing_ok=True) finally: if guard is not None: from gateway.status import _release_file_lock _release_file_lock(guard) with contextlib.suppress(OSError): guard.close() _invalidate_attach_cache() return True def _hold_lock_briefly(role: str): """Open and flock ``role``'s host lock without registering it; ``None`` when not won.""" from gateway.status import _try_acquire_file_lock path = lock_path(role) try: handle = open(path, "a+", encoding="utf-8") except OSError: return None if _try_acquire_file_lock(handle): return handle with contextlib.suppress(OSError): handle.close() return None def clear_record(role: str) -> None: """Remove this process's record + token on exit (never another owner's).""" role = _validated_role(role) existing = read_record(role, include_stale=True) if existing is not None and existing.pid != os.getpid(): return for path in (record_path(role), token_path(role)): with contextlib.suppress(OSError): path.unlink(missing_ok=True) _invalidate_attach_cache() # Roles this process must clean up on the way out, and the signal handlers we prepended. _cleanup_roles: list[str] = [] _prev_signal_handlers: dict[int, Any] = {} def _cleanup_role(role: str) -> None: with contextlib.suppress(Exception): clear_record(role) with contextlib.suppress(Exception): release_host_lock(role) def _handle_terminating_signal(signum, frame) -> None: """Drop the record + token, then hand off to the handler we prepended to.""" import signal as _signal for role in tuple(_cleanup_roles): _cleanup_role(role) prev = _prev_signal_handlers.get(signum) if callable(prev): prev(signum, frame) return if prev is _signal.SIG_IGN: return with contextlib.suppress(Exception): _signal.signal(signum, _signal.SIG_DFL) os.kill(os.getpid(), signum) raise SystemExit(128 + int(signum)) def cleanup_on_exit(role: str) -> None: """Clear ``role``'s record + token and release its lock on exit — SIGTERM included. ``atexit`` alone is false advertising for the NORMAL stop: systemd stop, ``docker stop`` and the update relaunch all send SIGTERM, and every terminating SIGTERM path here ends in the default disposition (uvicorn's ``capture_signals`` re-raises it after its graceful shutdown), which kills the process without running ``atexit``. The record then outlived its process and the 0600 token kept a LIVE session token on disk indefinitely. The handler only PREPENDS cleanup: whatever handler was installed before (uvicorn's graceful shutdown, the exit-flush chain, a supervisor's) still runs, so the shutdown sequence is unchanged. """ import atexit import signal as _signal import threading role = _validated_role(role) if role not in _cleanup_roles: _cleanup_roles.append(role) atexit.register(_cleanup_role, role) if threading.current_thread() is not threading.main_thread(): return for name in ("SIGTERM", "SIGBREAK"): signum = getattr(_signal, name, None) if signum is None or signum in _prev_signal_handlers: continue with contextlib.suppress(ValueError, OSError, RuntimeError): prev = _signal.getsignal(signum) _signal.signal(signum, _handle_terminating_signal) _prev_signal_handlers[signum] = prev def _multiplex_profiles_enabled() -> bool: """Will THIS process multiplex? An explicit ``true`` and an unset key both say yes, and an explicit ``false`` is RETIRED (``hermes_cli.gateway_multiplex_mode``) — it is warned about and ignored at boot, so it must not make the claim-time record advertise a narrower roster than the process actually serves. Reading it here was the last place the retired flag still decided topology, and it made CLI/dashboard report "standalone, serving default" while the runtime multiplexed. The RUNTIME verdict (a boot-time guard refusal) narrows the record afterwards, in ``gateway.run._refresh_host_gateway_record``, which republishes the SETTLED set. """ return True def served_profiles(*, multiplex: Optional[bool] = None) -> tuple[str, ...]: """Profiles this process multiplexes; ``()`` when the roster cannot be read. ``multiplex`` defaults to what this process's own config says. Hard-coding ``True`` here published a record claiming EVERY profile from a gateway that would only ever serve its own, and a second profile's supervised unit then stood down against a set nobody serves. """ try: from hermes_cli.profiles import profiles_to_serve enabled = _multiplex_profiles_enabled() if multiplex is None else bool(multiplex) return tuple(name for name, _ in profiles_to_serve(multiplex=enabled)) except Exception: logger.debug("served profile roster unavailable", exc_info=True) return () def describe(record: HostRecord) -> str: """One-line human description used by attach messages and conflict logs.""" where = f"{record.host or '127.0.0.1'}:{record.port}" if record.port else "no bound port" profiles = ", ".join(record.profiles) if record.profiles else "unknown" return f"PID {record.pid} ({where}; profiles: {profiles})"