"""Gateway restart/drain, Hermes update and background-action status dashboard routes. Extracted from ``hermes_cli.web_server``; helpers/state that tests monkeypatch on ``web_server`` stay there and are late-bound (cycle-safe). """ import asyncio import contextlib import logging import re import secrets import subprocess import time from pathlib import Path from typing import Any, Dict, List, Optional from fastapi import APIRouter, HTTPException, Request from hermes_cli.config import format_docker_update_message, recommended_update_command_for_method from hermes_cli.update_contract import COMMIT_BUILD_UPDATE_MESSAGE, is_commit_build from hermes_cli.version_info import get_version_info from hermes_cli.web_deps import LateState, late from hermes_cli.web_server_gateway import _ACTION_LOG_FILES from hermes_cli.web_routers._common import http_failure _log = logging.getLogger("hermes_cli.web_server") router = APIRouter() status_router = APIRouter() # Late-bound so a test's monkeypatch on the owning module wins at call time. _dashboard_local_update_managed_externally = late("_dashboard_local_update_managed_externally", "hermes_cli.web_server_files") _spawn_gateway_restart = late("_spawn_gateway_restart") _spawn_hermes_action = late("_spawn_hermes_action", "hermes_cli.web_server_gateway") detect_install_method = late("detect_install_method", "hermes_cli.config") get_hermes_home = late("get_hermes_home", "hermes_cli.config") _config_profile_scope = late("_config_profile_scope", "hermes_cli.web_server_profiles") _ACTION_COMMANDS = LateState("_ACTION_COMMANDS", "hermes_cli.web_server_gateway") _ACTION_IDS = LateState("_ACTION_IDS", "hermes_cli.web_server_gateway") _ACTION_PROCS = LateState("_ACTION_PROCS", "hermes_cli.web_server_gateway") _ACTION_RESULTS = LateState("_ACTION_RESULTS", "hermes_cli.web_server_gateway") def _server_path(name: str) -> Path: """Live Path value (``_ACTION_LOG_DIR`` on web_server_gateway, ``PROJECT_ROOT`` on web_server; plain values are not proxied by LateState).""" import hermes_cli.web_server as ws import hermes_cli.web_server_gateway as gw return getattr(gw if name == "_ACTION_LOG_DIR" else ws, name) _ACTION_LOG_TAIL_MAX_BYTES = 256 * 1024 _ACTION_LOG_TAIL_INITIAL_CHUNK_BYTES = 8 * 1024 _ACTION_LOG_TAIL_MAX_CHUNK_BYTES = 64 * 1024 _UPDATE_ACTION_COMPLETED_RE = re.compile(r"^=== hermes-update completed ([0-9a-f]{32}) ===$") _UPDATE_ACTION_STARTED_RE = re.compile(r"^=== hermes-update started .* ([0-9a-f]{32}) ===$") _MANAGED_EXTERNALLY_MESSAGE = "Hermes updates are managed outside this dashboard in containerized environments." # Per-kind dashboard error codes the UI keys on, by admission-refusal code. _UPDATE_REFUSAL_ERROR_CODES = { "docker": "docker_update_unsupported", "image-marker": "docker_update_unsupported", "image-marker-invalid": "docker_update_unsupported", "apt": "apt_update_required", "nix": "nix_update_unsupported", } def _finish_action(name: str, exit_code: Optional[int], pid: Optional[int]) -> None: """Record a terminal result and drop the live-process registries for ``name``.""" _ACTION_RESULTS[name] = {"exit_code": exit_code, "pid": pid} for registry in (_ACTION_PROCS, _ACTION_COMMANDS, _ACTION_IDS): registry.pop(name, None) def _record_completed_action(name: str, message: str, exit_code: int = 1) -> None: """Record a non-spawned action result and write it to the action log.""" log_dir = _server_path("_ACTION_LOG_DIR") log_dir.mkdir(parents=True, exist_ok=True) with open(log_dir / _ACTION_LOG_FILES[name], "ab", buffering=0) as log_file: log_file.write(f"\n=== {name} completed {time.strftime('%Y-%m-%d %H:%M:%S')} ===\n".encode()) log_file.write(message.encode("utf-8", errors="replace")) if not message.endswith("\n"): log_file.write(b"\n") _finish_action(name, exit_code, None) def _tail_lines(path: Path, n: int) -> list[str]: """Return the last ``n`` lines of ``path`` without loading huge logs.""" try: size = path.stat().st_size except OSError: return [] if n <= 0 or size <= 0: return [] min_offset = max(0, size - _ACTION_LOG_TAIL_MAX_BYTES) offset = size chunk_size = _ACTION_LOG_TAIL_INITIAL_CHUNK_BYTES newline_count = 0 chunks: list[bytes] = [] drop_partial_first_line = False try: with path.open("rb") as handle: while offset > min_offset and newline_count <= n: read_size = min(chunk_size, offset - min_offset) offset -= read_size handle.seek(offset) chunk = handle.read(read_size) chunks.append(chunk) newline_count += chunk.count(b"\n") chunk_size = min(chunk_size * 2, _ACTION_LOG_TAIL_MAX_CHUNK_BYTES) if offset > 0: handle.seek(offset - 1) drop_partial_first_line = handle.read(1) != b"\n" except OSError: return [] lines = b"".join(reversed(chunks)).decode("utf-8", errors="replace").splitlines() if drop_partial_first_line and lines: lines = lines[1:] return lines[-n:] def _durable_completed_update_action_id(lines: list[str]) -> Optional[str]: """Latest successful update id from ``update.log`` — the durable record that survives the update restarting the dashboard (losing the in-memory ``Popen``/result registries). Only a completion marker after the latest start marker counts, so a stale success cannot mask a newer failed attempt.""" last_start = last_completed = -1 completed_action_id: Optional[str] = None for index, line in enumerate(lines): if line.startswith("=== hermes update started "): last_start = index match = _UPDATE_ACTION_COMPLETED_RE.fullmatch(line.strip()) if match: last_completed = index completed_action_id = match.group(1) return completed_action_id if completed_action_id and last_completed > last_start else None def _latest_spawned_update_action_id(lines: list[str]) -> Optional[str]: """Action id named by the latest ``hermes-update started`` header of THIS dashboard's log.""" for line in reversed(lines): if line.startswith("=== hermes-update started "): match = _UPDATE_ACTION_STARTED_RE.fullmatch(line.strip()) return match.group(1) if match else None return None def _persisted_action_id(log_dir: Path, name: str) -> Optional[str]: """Action id of the latest spawn of ``name`` from its sidecar (survives the log's tail bound and rotation). A visible start header wins: a dashboard predating the sidecar wrote only the header, and the newest header IS the latest spawn.""" with contextlib.suppress(OSError, UnicodeDecodeError): value = (log_dir / f"{name}.action_id").read_text(encoding="utf-8-sig").strip() return value if re.fullmatch(r"[0-9a-f]{32}", value) else None return None @router.post("/api/gateway/restart") async def restart_gateway(profile: Optional[str] = None): """Kick off a ``hermes gateway restart --interrupt`` in the background: the user-commanded force path that supersedes a drain (D6, 19b26244046), so live sessions do not refuse it.""" with http_failure("Failed to spawn gateway restart", 500, "Failed to restart gateway"): proc, _reused = _spawn_gateway_restart(profile, interrupt=True) return {"ok": True, "pid": proc.pid, "name": "gateway-restart"} @router.get("/api/gateway/migrate/plan") async def gateway_migrate_plan(): """Preflight for folding per-profile gateways into one multiplexer (same JSON as the CLI plan).""" from hermes_cli.gateway_migrate import build_migration_plan plan = await asyncio.to_thread(build_migration_plan) return plan.to_dict() @router.post("/api/gateway/migrate") async def gateway_migrate(): """Run ``hermes gateway migrate --multiplex --yes`` detached; the CLI re-runs the preflight and refuses (exit 1 into the action log) when blocked, so the UI should gate on the plan first.""" from hermes_cli.web_server_gateway import _spawn_hermes_action with http_failure("Failed to spawn gateway migrate", 500, "Failed to start gateway migration"): proc = _spawn_hermes_action(["gateway", "migrate", "--multiplex", "--yes"], "gateway-migrate") return {"ok": True, "pid": proc.pid, "name": "gateway-migrate"} @router.post("/api/gateway/drain") async def gateway_drain(request: Request): """Begin or cancel an external (NAS-driven) gateway drain. Authenticated by the non-interactive token-auth seam (the ``dashboard_auth/drain`` plugin registers this path as a token route and verifies the bearer secret); without that plugin the cookie gate covers a gated bind and the legacy session-token gate a loopback bind — never unauthenticated on a network bind. Body ``{"action": "drain"|"cancel"}``. Only the ``.drain_request.json`` marker is written/removed here — the gateway's ``_drain_control_watcher`` owns the state transition (the marker IS the control channel). Idempotent on both sides; ``POST /api/gateway/restart`` is the force-override that supersedes a drain. """ from gateway.drain_control import clear_drain_request, drain_requested, write_drain_request try: body = await request.json() except Exception: body = {} body = body or {} action = str(body.get("action", "drain")).strip().lower() # Attribute to the verified token principal when the token-auth seam attached one. principal = getattr(getattr(request.state, "token_principal", None), "principal", None) or "dashboard" if action == "cancel": existed = clear_drain_request() _log.info("Gateway drain CANCEL requested by %s (existed=%s)", principal, existed) return {"ok": True, "action": "cancel", "was_draining": existed} if action != "drain": raise HTTPException(status_code=400, detail=f"Unknown drain action {action!r}; expected 'drain' or 'cancel'") payload = write_drain_request( principal=str(principal), suppress_notification=bool(body.get("suppress_notification", False)), ) _log.info( "Gateway drain BEGIN requested by %s (suppress_notification=%s)", principal, payload["suppress_notification"], ) return { "ok": True, "action": "drain", "requested_at": payload["requested_at"], # Echo so a caller polling /api/status knows the marker is now set; # the gateway watcher flips gateway_state -> draining within ~1s. "draining": drain_requested(), "suppress_notification": payload["suppress_notification"], } def _update_refused(error: str, message: str, update_command: str) -> dict[str, Any]: _record_completed_action("hermes-update", message, exit_code=1) return { "ok": False, "pid": None, "name": "hermes-update", "error": error, "message": message, "update_command": update_command, } @router.post("/api/hermes/update") async def update_hermes(): """Kick off ``hermes update`` in the background.""" if is_commit_build(_server_path("PROJECT_ROOT")): return _update_refused("commit-build", COMMIT_BUILD_UPDATE_MESSAGE, "") if _dashboard_local_update_managed_externally(): message = _MANAGED_EXTERNALLY_MESSAGE + " The built-in local updater is disabled here." # No runnable command exists here: the outer launcher/image owns updates. return _update_refused("dashboard_update_managed_externally", message, "") # Shared admission gate: marker-first, then the docker/nix/apt heuristics — # one decision with the CLI paths. from hermes_cli.update_contract import evaluate_update_admission, record_refusal_receipt refusal = evaluate_update_admission(_server_path("PROJECT_ROOT")) if refusal is not None: response = _update_refused( _UPDATE_REFUSAL_ERROR_CODES.get(refusal.code, "update_not_in_place"), refusal.message, refusal.update_command, ) record_refusal_receipt(refusal) return response existing = _ACTION_PROCS.get("hermes-update") if existing is not None and existing.poll() is None: response = {"ok": True, "pid": existing.pid, "name": "hermes-update", "already_running": True} action_id = _ACTION_IDS.get("hermes-update") if action_id: response["action_id"] = action_id return response action_id = secrets.token_hex(16) with http_failure("Failed to spawn hermes update", 500, "Failed to start update"): proc = _spawn_hermes_action(["update"], "hermes-update", env_overrides={"HERMES_ACTION_ID": action_id}) return {"ok": True, "pid": proc.pid, "name": "hermes-update", "action_id": action_id} _NON_APPLYABLE_MESSAGES = { "docker": format_docker_update_message, "apt": lambda: "Hermes is managed by Termux APT; run `pkg upgrade hermes-agent`.", } def _source_release_label(status: dict[str, Any]) -> str: version = status.get("sourceVersion") return f"{status.get('channel') or 'stable'} release" + (f" v{version}" if version else "") @router.get("/api/hermes/update/check") async def check_hermes_update(force: bool = False, profile: Optional[str] = None): """Report whether a Hermes update is available, without applying it. Returns install_method ('apt'|'git'|'docker'|'nix'|'nixos'|'unknown'), current_version, behind (commits behind, 0 = up to date, -1 = unknown count, null = check could not run), update_available, can_apply (git only — the dashboard button can apply in place), update_command, message (guidance for non-applyable methods) and, for git installs that are behind, commits [{sha, summary, author, at}] (additive; existing consumers ignore it). """ if is_commit_build(_server_path("PROJECT_ROOT")): return { "install_method": "desktop-app", "current_version": get_version_info().derived_version, "behind": None, "update_available": False, "can_apply": False, "update_command": "", "message": COMMIT_BUILD_UPDATE_MESSAGE, } if _dashboard_local_update_managed_externally(): return { "install_method": "managed-runtime", "current_version": get_version_info().derived_version, "behind": None, "update_available": False, "can_apply": False, "update_command": "", "message": _MANAGED_EXTERNALLY_MESSAGE, } install_method = detect_install_method(_server_path("PROJECT_ROOT")) payload: dict[str, Any] = { "install_method": install_method, "current_version": get_version_info().derived_version, "behind": None, "update_available": False, "can_apply": install_method == "git", "update_command": recommended_update_command_for_method(install_method), "message": None, } non_applyable = _NON_APPLYABLE_MESSAGES.get(install_method) if non_applyable is not None: payload["message"] = non_applyable() return payload # source_check.check_for_updates() handles git / nix-revision paths through the GitHub API and # caches the result for 24h. ``force`` busts the cache so "Check now" reflects reality. status: dict[str, Any] = {} try: from hermes_cli.source_check import check_for_updates with _config_profile_scope(profile): status = await asyncio.to_thread(check_for_updates, force=force) behind = status.get("behind") except Exception: _log.exception("Update check failed") behind = None payload["behind"] = behind if behind is None: payload["message"] = "Couldn't reach the update source — try again later." elif behind == 0 and status.get("held"): version = status.get("sourceVersion") payload.update(held=status["held"], source_version=version, message=f"This build is newer than {_source_release_label(status)}, which predates the gateway " "runtime its sessions use. It stays on this build until a newer release ships.") elif behind == 0: payload["message"] = "You're on the latest version." else: payload["update_available"] = True if status.get("rollback"): # A stable release this build already contains: applying it moves back. payload.update(rollback_to=status.get("sourceVersion"), message=f"This build is newer than {_source_release_label(status)}. " "Updating moves it back to that release.") # "What's changed" for the desktop's remote update overlay; best-effort # (empty list on any failure). Channel + release name pick its notes source. payload.update(channel=status.get("channel"), branch=status.get("branch"), source_version=status.get("sourceVersion")) payload["commits"] = [{**row, "sha": row["sha"][:7], "at": row["at"] // 1000} for row in status.get("commits", [])[:20]] return payload def _completed_exit_code( result: Optional[dict[str, Any]], durable_action_id: Optional[str], receipt: Optional[dict[str, Any]], ) -> Optional[int]: """Exit code for an action with no live process: in-memory result, else durable evidence.""" if result is not None: return result.get("exit_code") if durable_action_id: return 0 if receipt is not None and receipt.get("outcome") in ("success", "partial"): # No in-memory result and no log marker (e.g. log rotated), but THIS # action's receipt proves a completed run: report its outcome rather than # a null clients time out on. ``partial`` maps to exit 1 like the CLI. return 0 if receipt["outcome"] == "success" else 1 return None @status_router.get("/api/actions/{name}/status") async def get_action_status(name: str, lines: int = 200): """Tail an action log and report whether the process is still running.""" log_file_name = _ACTION_LOG_FILES.get(name) if log_file_name is None: raise HTTPException(status_code=404, detail=f"Unknown action: {name}") log_dir = _server_path("_ACTION_LOG_DIR") requested_lines = min(max(lines, 1), 2000) tail = _tail_lines(log_dir / log_file_name, requested_lines) durable_update_action_id = None update_receipt_summary = action_receipt_summary = None if name == "hermes-update": # ``hermes update`` mirrors to the ROOT home's update.log (main_dashboard), never this # dashboard's profile home: read it where it is written. from hermes_cli.logs import log_file_path durable_update_action_id = _durable_completed_update_action_id(_tail_lines(log_file_path("update"), 2000)) spawned_action_id = (_latest_spawned_update_action_id(_tail_lines(log_dir / log_file_name, 2000)) or _persisted_action_id(log_dir, name) or _ACTION_IDS.get(name)) if durable_update_action_id != spawned_action_id: # The root log is shared by every profile: another profile's (or an older) run's # completion never certifies the action this dashboard started. durable_update_action_id = None if durable_update_action_id: marker = f"=== hermes-update completed {durable_update_action_id} ===" if marker not in tail: tail = [*tail, marker][-requested_lines:] # The update receipt is the durable, structured truth about the last # update (written by every run, incl. refused/failed; survives the # dashboard restarting itself mid-action). Surface it so clients READ # the outcome instead of inferring it from liveness probes. # See #81193, #87359, #91277. # The store is root-wide (every profile's runs): show THIS action's receipt when one exists. update_receipt_summary = _latest_update_receipt_summary(spawned_action_id) if spawned_action_id and (update_receipt_summary or {}).get("action_id") == spawned_action_id: action_receipt_summary = update_receipt_summary proc = _ACTION_PROCS.get(name) if proc is None: result = _ACTION_RESULTS.get(name) running = False pid = result.get("pid") if result else None exit_code = _completed_exit_code(result, durable_update_action_id, action_receipt_summary) else: exit_code = proc.poll() running = exit_code is None pid = proc.pid if exit_code is not None: with contextlib.suppress(Exception): proc.wait(timeout=1) _finish_action(name, exit_code, pid) response = {"name": name, "running": running, "exit_code": exit_code, "pid": pid, "lines": tail} if durable_update_action_id: response["action_id"] = durable_update_action_id if update_receipt_summary is not None: response["receipt"] = update_receipt_summary return response def _read_latest_receipt() -> Optional[dict[str, Any]]: """Latest update receipt, or None on any failure (never raises).""" try: from hermes_cli.update_receipt import read_latest_receipt return read_latest_receipt() or None except Exception: return None def _latest_update_receipt_summary(action_id: Optional[str] = None) -> Optional[dict[str, Any]]: """Compact summary of dashboard action ``action_id``'s receipt when one exists, else of the latest receipt (written by EVERY ``hermes update`` run, incl. refused/failed), or None; never raises. Steps/skips stay in the full endpoint. ``action_id`` names the writer so a client never credits another action's run. Phase-1 bullet 3 (#91277): the receipt (written by EVERY ``hermes update`` run since #91283, including refused and failed ones, with a ``latest.json`` pointer) is the durable success signal the Desktop and dashboard should read instead of inferring outcomes from liveness probes across the update's stop/start gap (#81193, #87359). """ receipt = None if action_id: from hermes_cli.update_receipt import read_receipt_for_action receipt = read_receipt_for_action(action_id) receipt = receipt or _read_latest_receipt() if not receipt: return None try: post = receipt.get("post_update") or {} return { **{k: receipt.get(k) for k in ("outcome", "started_at", "finished_at", "action_id")}, "pre_sha": (receipt.get("pre_update") or {}).get("sha"), "post_sha": post.get("sha"), "post_version": post.get("version"), "fleet_states": sorted({str(e.get("state")) for e in receipt.get("fleet") or [] if isinstance(e, dict)}), # C3: a committed run is ``success`` even while post-commit steps are owed; carry them # so a client reports the owed step and its remedy instead of plain success. "followups": [{"step": str(f.get("step")), "reason": str(f.get("reason") or "")} for f in receipt.get("followups") or [] if isinstance(f, dict)], "user_action": receipt.get("user_action") if isinstance(receipt.get("user_action"), dict) else None, } except Exception: return None @status_router.get("/api/hermes/update/receipt") async def get_update_receipt(): """The FULL latest update receipt (steps, skips, gateway restart outcome, fleet matrix) plus a compact ``summary``; 404 when no update has run since receipts landed. Clients read this instead of inferring success from backend liveness, which misread the update's own restart gap as a failed update/boot. See #81193, #87359, #91277. """ receipt = _read_latest_receipt() if not receipt: raise HTTPException(status_code=404, detail="No update receipt found (no `hermes update` run recorded).") return {"receipt": receipt, "summary": _latest_update_receipt_summary()}