| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928 |
- """Bot Desktop runtime: one headless Xfce desktop per Hermes profile, served over RFB on a private
- Unix socket, viewed and driven from Hermes Desktop.
- Layout under ``<HERMES_HOME>/bot-desktop/``: ``display`` (allocated X display number), ``rfb.sock``
- (Xvnc RFB Unix socket, 0600), ``Xauthority``, ``env`` (DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS
- published by the launcher once Xfce's bus exists), ``launcher.pid``, ``launcher.log``, ``xdg/``
- (per-profile XDG_CONFIG_HOME so two profiles never share xfconf). Everything is profile-scoped via
- ``get_hermes_home()`` so N profiles in one gateway get N desktops: one screen per bot on the shared
- machine.
- The launcher is ``launcher.sh`` next to this module; :func:`desktop_env` is what cua-driver and headed
- Chromium spawns merge in so the agent acts on this profile's screen and nowhere else.
- """
- from __future__ import annotations
- import contextlib
- import logging
- import math
- import os
- import shlex
- import shutil
- import signal
- import subprocess
- import sys
- import time
- from dataclasses import dataclass
- from pathlib import Path
- from typing import Callable, Dict, Optional
- from agent.deadline import poll_until
- from hermes_constants import get_hermes_home
- from tools.bot_desktop import placement
- logger = logging.getLogger(__name__)
- _LAUNCHER = Path(__file__).with_name("launcher.sh")
- # Display numbers below 10 collide with real seats and default Xvfb recipes (:99 is popular too); scan a
- # private band and record the choice so restarts reuse it.
- _DISPLAY_MIN, _DISPLAY_MAX = 20, 89
- # Binaries the launcher execs; the package hint is per distro family.
- REQUIRED_BINARIES = ("Xvnc", "xfwm4", "xfce4-panel", "xfdesktop", "xfsettingsd", "dbus-run-session",
- "xauth", "xdpyinfo", "setxkbmap", "xprop")
- # Which package in each distro list ships each required binary. Fedora retired the xorg-x11-utils /
- # xorg-x11-server-utils umbrellas (per-binary packages since F35) and dnf5 refuses the whole transaction on
- # one unknown name, so every binary must map to a package that still resolves; the test suite checks that
- # each mapped package is in PACKAGES for its manager.
- BINARY_PACKAGES = {
- "apt": {"Xvnc": "tigervnc-standalone-server", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel",
- "xfdesktop": "xfdesktop4", "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus-x11",
- "xauth": "xauth", "xdpyinfo": "x11-utils", "setxkbmap": "x11-xkb-utils", "xprop": "x11-utils"},
- # tigervnc-x11-server is the real package (tigervnc-server-minimal is only a Provides on it); dbus-run-session
- # is in dbus-daemon (dbus-x11 ships dbus-launch only).
- "dnf": {"Xvnc": "tigervnc-x11-server", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel",
- "xfdesktop": "xfdesktop", "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus-daemon",
- "xauth": "xorg-x11-xauth", "xdpyinfo": "xdpyinfo", "setxkbmap": "setxkbmap", "xprop": "xprop"},
- "pacman": {"Xvnc": "tigervnc", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel", "xfdesktop": "xfdesktop",
- "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus",
- "xauth": "xorg-xauth", "xdpyinfo": "xorg-xdpyinfo", "setxkbmap": "xorg-setxkbmap", "xprop": "xorg-xprop"},
- }
- PACKAGES = {
- "apt": ["tigervnc-standalone-server", "xfce4-panel", "xfwm4", "xfdesktop4", "xfce4-settings",
- "xfce4-terminal", "dbus-x11", "x11-xserver-utils", "x11-utils", "x11-xkb-utils", "xauth",
- "fonts-dejavu-core"],
- "dnf": ["tigervnc-x11-server", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings",
- "xfce4-terminal", "dbus-daemon", "xsetroot", "xset", "xdpyinfo", "xprop", "xorg-x11-xauth", "setxkbmap",
- "dejavu-sans-fonts"],
- "pacman": ["tigervnc", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", "xfce4-terminal", "dbus",
- "xorg-xsetroot", "xorg-xset", "xorg-xdpyinfo", "xorg-xprop", "xorg-xauth", "xorg-setxkbmap",
- "ttf-dejavu"],
- }
- def state_dir() -> Path:
- return get_hermes_home() / "bot-desktop"
- def is_supported_host() -> bool:
- return sys.platform.startswith("linux")
- def missing_binaries() -> list[str]:
- return [b for b in REQUIRED_BINARIES if shutil.which(b) is None]
- def package_manager() -> Optional[str]:
- for pm in ("apt-get", "dnf", "pacman"):
- if shutil.which(pm):
- return "apt" if pm == "apt-get" else pm
- return None
- def install_command() -> Optional[str]:
- """The distro command that installs the Bot Desktop packages, as the human would type it on THIS host:
- prefixed with ``sudo`` unless Hermes already runs as root, so it is both what the pane shows and what
- :mod:`tools.bot_desktop.install` runs. ``None`` when no package manager is present.
- Not a promise that it can run here: see :func:`installable`. The published Docker image supervises
- every service under ``s6-setuidgid hermes`` (UID 10000 by default) and ships no ``sudo`` binary, so an
- install on a hosted instance is impossible no matter what this returns."""
- pm = package_manager()
- if pm is None:
- return None
- pkgs = " ".join(PACKAGES[pm])
- body = {
- "apt": f"apt-get install -y --no-install-recommends {pkgs}",
- "dnf": f"dnf install -y {pkgs}",
- "pacman": f"pacman -S --needed --noconfirm {pkgs}",
- }[pm]
- return body if is_root() else f"sudo {body}"
- def is_root() -> bool:
- return hasattr(os, "geteuid") and os.geteuid() == 0
- def installable() -> bool:
- """Whether :func:`install_command` could actually succeed on this host.
- False on an unprivileged process with no ``sudo`` to reach for, which is exactly the published Docker
- image: services drop to the ``hermes`` user and no ``sudo`` binary is installed. The packages can only
- arrive in the image there, so :func:`start` says that instead of printing a sudo line the user has no
- way to run. ``status()`` still reports ``install_command`` for the pane; surfacing this there needs a
- wire-contract change and is deliberately out of scope.
- """
- if package_manager() is None:
- return False
- return is_root() or shutil.which("sudo") is not None
- @dataclass
- class DesktopStatus:
- profile: str
- supported: bool
- installed: bool
- missing: list[str]
- running: bool
- pid: Optional[int]
- display: Optional[str]
- socket: Optional[str]
- geometry: str
- install_command: Optional[str]
- browser: Optional[str] # headed Chromium the dock's Browser icon and the agent's browser share; None = no headed browser
- blocker: Optional[str] = None # why start() would refuse right now (memory); None = may start
- memory_available_mb: Optional[int] = None
- memory_limit_mb: Optional[int] = None
- placement: str = "gateway" # "gateway" | "terminal:<backend>" — where Xvnc runs
- image_switch: Optional[dict[str, object]] = None # pending default-image switch the pane can approve
- def as_dict(self) -> dict[str, object]:
- return dict(self.__dict__)
- def _read(path: Path) -> Optional[str]:
- try:
- return path.read_text(encoding="utf-8-sig").strip() or None
- except OSError:
- return None
- def _pid_alive(pid: int) -> bool:
- """A zombie is dead for our purposes: a SIGKILLed launcher stays a zombie in the gateway until the next
- Popen reaps it, and reporting it as running would hide its orphaned X server behind a live status."""
- import psutil
- try:
- return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE
- except psutil.Error:
- return False
- def _create_time(pid: int) -> Optional[float]:
- import psutil
- try:
- return psutil.Process(pid).create_time()
- except (psutil.Error, OverflowError, ValueError):
- return None
- def _launcher_identity() -> Optional[tuple[int, float]]:
- """``(pid, create_time)`` of the live launcher, or None. ``launcher.pid`` holds ``"<pid> <create_time>"``:
- a recycled pid with a different start time is somebody else's process and must never be reported as
- ours nor killed by :func:`stop`. The pre-identity single-number format is treated as not running."""
- raw = _read(state_dir() / "launcher.pid")
- pid_s, _, born_s = (raw or "").partition(" ")
- if not pid_s.isdigit() or not born_s:
- return None
- try:
- pid, born = int(pid_s), float(born_s)
- except ValueError:
- return None
- actual = _create_time(pid)
- return (pid, born) if actual is not None and abs(actual - born) < 0.01 and _pid_alive(pid) else None
- def _launcher_pid() -> Optional[int]:
- """The live launcher's pid, or None (see :func:`_launcher_identity`)."""
- identity = _launcher_identity()
- return identity[0] if identity is not None else None
- def _recorded_launcher_pid() -> Optional[int]:
- """The pid ``launcher.pid`` names, alive or not (the orphan sweep matches process groups against it)."""
- pid_s, _, born_s = (_read(state_dir() / "launcher.pid") or "").partition(" ")
- return int(pid_s) if pid_s.isdigit() and born_s else None
- _X_LOCK_DIR = Path("/tmp") # no-tmp: ok — X servers write .X<n>-lock here by protocol (tests point it at a scratch dir)
- _X_UNIX_TABLE = Path("/proc/net/unix") # the kernel's list of bound Unix sockets (tests point it at a fixture)
- def _x_lock_pid(num: int) -> Optional[int]:
- try:
- return int((_X_LOCK_DIR / f".X{num}-lock").read_text(encoding="utf-8-sig").strip())
- except (OSError, ValueError):
- return None
- def _x_socket_bound(num: int) -> bool:
- """A running X server keeps ``@/tmp/.X11-unix/X<n>`` (abstract namespace) bound for its whole life; the
- kernel drops it only when the process exits. A /tmp reaper can remove the lock file under a live Xvnc,
- and only this binding then still says the number is taken (a new server on it dies 'already running')."""
- try:
- lines = _X_UNIX_TABLE.read_text(encoding="utf-8-sig").splitlines()
- except OSError:
- return False
- # no-tmp: ok — detects the X server's display socket at the path the X11 protocol fixes
- return any(line.split()[-1].lstrip("@") == f"/tmp/.X11-unix/X{num}" for line in lines if line.strip())
- def _display_in_use(num: int) -> bool:
- """A live X server owns ``:num``: its lock file names a running pid, or its X11 socket is bound. A lock
- left by a crashed server (dead pid, no socket) does not count, so the number can be reclaimed."""
- pid = _x_lock_pid(num)
- return (pid is not None and _pid_alive(pid)) or _x_socket_bound(num)
- def _reap_orphaned_server(sd: Path) -> bool:
- """Caller holds ``start.lock`` and has established that no live launcher exists. The launcher runs Xvnc in
- its own session, so a SIGKILLed launcher leaves the X server alive, holding the display and ``rfb.sock``;
- ``status()`` keys on the launcher and says stopped, and a naive restart allocates a second server next
- to it and overwrites the socket path both now claim. The X lock of the recorded display names that
- server: it is ours when it sits in the dead launcher's process group or its command line binds OUR
- socket. When a /tmp reaper took the lock too (or the failed launch dropped ``display``), the socket path
- on the Xvnc command line is the remaining handle — without it one X server leaks per occurrence. Kill it
- (group first), drop the state it left, and report whether anything was signalled."""
- import psutil
- def _binds_our_socket(cmdline: list) -> bool:
- return "Xvnc" in Path(cmdline[0] if cmdline else "").name and str(sd / "rfb.sock") in cmdline
- recorded = _read(sd / "display")
- pid = _x_lock_pid(int(recorded)) if recorded and recorded.isdigit() else None
- if pid is None or not _pid_alive(pid):
- pid = next((p.pid for p in psutil.process_iter(["cmdline"]) if _binds_our_socket(p.info["cmdline"] or [])), None)
- if pid is None:
- return False
- launcher = _recorded_launcher_pid()
- try:
- pgid = os.getpgid(pid) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start/stop)
- cmdline = psutil.Process(pid).cmdline()
- except (ProcessLookupError, psutil.Error):
- return False
- if pgid != launcher and not _binds_our_socket(cmdline):
- return False # somebody else's server took the number after we died; never touch it
- logger.warning("Bot Desktop launcher %s is gone but its X server (pid %s) survived on :%s; reaping",
- launcher, pid, recorded or "?")
- _kill_group_then_wait(pgid if pgid == launcher else None, pid)
- if recorded:
- (_X_LOCK_DIR / f".X{recorded}-lock").unlink(missing_ok=True)
- for name in ("launcher.pid", "env", "rfb.sock"):
- (sd / name).unlink(missing_ok=True)
- return True
- def _kill_group_then_wait(pgid: Optional[int], pid: int, grace: float = 2.0) -> None:
- """SIGTERM the group (or the lone pid), SIGKILL whatever is still there after ``grace``."""
- def _signal(sig: int) -> None:
- with contextlib.suppress(ProcessLookupError, PermissionError):
- if pgid is not None:
- os.killpg(pgid, sig) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start/stop)
- else:
- os.kill(pid, sig)
- def _anything_left() -> bool:
- # The leader dying first is the common case (bash exits on TERM, Xvnc traps it); the group is
- # done only when killpg(0) finds nobody, else a TERM-ignoring descendant keeps the display.
- if pgid is None:
- return _pid_alive(pid)
- try:
- os.killpg(pgid, 0) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start/stop)
- except ProcessLookupError:
- return False
- except PermissionError:
- return True
- return True
- def _reap_if_ours() -> None:
- # The launcher was Popen'd by whichever gateway started it; a later gateway that stops it holds
- # no Popen, so the dead leader would sit as a zombie in our table (and count as "left").
- with contextlib.suppress(ChildProcessError, OSError):
- os.waitpid(pid, os.WNOHANG) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start/stop)
- def _gone() -> bool:
- _reap_if_ours()
- return not _anything_left()
- _signal(signal.SIGTERM)
- if poll_until(_gone, grace, 0.05):
- return
- _signal(signal.SIGKILL) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start/stop)
- time.sleep(0.1)
- _reap_if_ours()
- # Host-wide (every profile allocates from one band), so it lives outside any profile home. A predictable
- # name must not be squattable: XDG_RUNTIME_DIR is the boundary — 0700 from logind, or from
- # docker/stage2-hook.sh in containers, which have none.
- _ALLOC_LOCK = Path(os.environ.get("XDG_RUNTIME_DIR") or Path.home() / ".cache") / "hermes-bot-desktop-alloc.lock"
- @contextlib.contextmanager
- def _flocked(path: Path):
- import fcntl # windows-footgun: ok — Linux-only runtime (is_supported_host gates start)
- with open(path, "a+", encoding="utf-8") as fh: # windows-footgun: ok — Linux-only runtime
- fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
- try:
- yield fh
- finally:
- fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
- def _pick_display() -> int:
- """Caller holds ``_ALLOC_LOCK``. The recorded number is only reused when no OTHER server holds it now:
- after profile A stops, B may have taken A's number, and A's launcher must never unlink B's socket."""
- recorded = _read(state_dir() / "display")
- if recorded and recorded.isdigit() and not _display_in_use(int(recorded)):
- return int(recorded)
- for num in range(_DISPLAY_MIN, _DISPLAY_MAX + 1):
- if not _display_in_use(num):
- return num
- raise RuntimeError("no free X display number in the Bot Desktop band")
- def _allocate_display() -> int:
- _ALLOC_LOCK.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
- with _flocked(_ALLOC_LOCK):
- return _pick_display()
- def desktop_env(base_env: Optional[dict[str, str]] = None) -> dict[str, str]:
- """``base_env`` (default ``os.environ``) with this profile's DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS
- merged in when its desktop is running. Unchanged otherwise, so hosts with a real seat keep it.
- Pure: never starts anything (it is called from env builders, status probes and tests)."""
- env = dict(os.environ if base_env is None else base_env)
- published = published_env()
- if published:
- touch_activity() # a browser / cua-driver spawn is the agent using its screen
- env.update(published)
- env.pop("WAYLAND_DISPLAY", None) # X11 desktop; a leaked Wayland socket flips GTK/Chromium backends
- return env
- def ensure_started_for_tool() -> None:
- """Tool-boundary hook (``computer_use`` dispatch and the headed Chromium spawn sites of the browser tool).
- Under ``terminal`` placement the sandbox screen is brought up on first use unconditionally: the sandbox is
- the boundary the user chose, a screen inside it touches nothing outside it, and the tools below refuse
- rather than run on the host when it is not up. On the gateway host, ``bot_desktop.auto_start`` (opt-in,
- default off) starts a Linux host's screen when it has NO display and the packages installed, so a headless
- gateway works the first time. Host failure is not an error here (the tool's own "no display" diagnosis is
- the right message then); a sandbox start failure is logged and left for :func:`tool_placement` to raise."""
- if published_env().get("DISPLAY"):
- touch_activity()
- return
- where = placement.resolve()
- if where.where == placement.REFUSED:
- return # tool_placement() raises the reason at the spawn site
- if where.where == placement.TERMINAL:
- try:
- start()
- except Exception as exc:
- logger.info("Bot Desktop sandbox start deferred to the tool: %s", exc)
- return
- if not _should_auto_start(os.environ):
- return
- try:
- start()
- except Exception as exc:
- logger.info("Bot Desktop auto-start skipped: %s", exc)
- def tool_placement() -> str:
- """Where the browser / cua-driver a tool is about to spawn MUST run: ``placement.GATEWAY`` or
- ``placement.TERMINAL``. Policy first, liveness second: a ``terminal`` placement whose screen is down gets
- it started here (raising with the blocker when it cannot come up) and a ``refused`` placement raises its
- reason. Neither ever yields the host — a screen that happens to be down is not permission to run the
- agent's browser outside the sandbox the user chose."""
- where = placement.resolve()
- if where.where == placement.REFUSED:
- raise RuntimeError(where.reason)
- if where.where == placement.GATEWAY:
- return placement.GATEWAY
- if not sandbox_screen_running() or not published_env().get("DISPLAY"):
- start()
- return placement.TERMINAL
- def _should_auto_start(env: dict[str, str]) -> bool:
- if not is_supported_host() or env.get("DISPLAY") or env.get("WAYLAND_DISPLAY"):
- return False
- if missing_binaries():
- return False
- from hermes_cli.config import load_config_readonly
- cfg = load_config_readonly().get("bot_desktop") or {}
- return bool(cfg.get("auto_start", False))
- # ---- idle auto-stop -------------------------------------------------------------------------------
- # A desktop nobody is using still holds ~220 MiB (plus whatever browser was left open). Every use —
- # a computer_use action, a browser spawn onto the screen, a viewer attached, a human takeover — stamps
- # ``activity``; :func:`stop_if_idle` stops a screen idle past ``bot_desktop.idle_stop_minutes`` unless a
- # human is using it (see the lease lapse below). Two callers apply it: the gateway's display watcher and
- # the screen's own launcher (``idle_watch``), so a screen whose gateway never came back still stops. The
- # next use starts it again.
- DEFAULT_IDLE_STOP_MINUTES = 30
- # A human holding the screen is never timed out while there is evidence of one: an attached viewer stamps
- # ``activity`` (every RFB update it receives, at most once a minute; the panel clock repaints every minute).
- # A human lease with no stamp for this many windows, and never less than HUMAN_LEASE_ABANDONED_MIN_S, is
- # a link that dropped and never came back, or a corrupt lease file (which reads as human): the screen is
- # stopped like any idle one. The floor keeps a short window (idle_stop_minutes: 0.1) from timing out a
- # human between two viewer stamps, which can be up to about two minutes apart.
- HUMAN_LEASE_ABANDONED_WINDOWS = 4
- HUMAN_LEASE_ABANDONED_MIN_S = 600.0
- def touch_activity() -> None:
- path = state_dir() / "activity"
- try:
- path.touch()
- os.utime(path, None)
- except OSError:
- pass
- def idle_seconds() -> Optional[float]:
- """Seconds since the last stamped use; None when the screen never recorded one (falls back to the
- env file's publish time so a screen started and then forgotten still ages)."""
- for name in ("activity", "env"):
- try:
- return max(0.0, time.time() - (state_dir() / name).stat().st_mtime)
- except OSError:
- continue
- return None
- def idle_stop_seconds() -> float:
- from hermes_cli.config import load_config_readonly
- cfg = load_config_readonly().get("bot_desktop") or {}
- try:
- minutes = float(cfg.get("idle_stop_minutes", DEFAULT_IDLE_STOP_MINUTES))
- except (TypeError, ValueError):
- minutes = DEFAULT_IDLE_STOP_MINUTES
- # .inf ("never") is off, like 0, and so are .nan and a window too large to convert to seconds (1e308): none
- # can ever be due, and the launcher's next-check arithmetic would raise on them and fail start().
- seconds = minutes * 60
- return seconds if math.isfinite(seconds) and seconds > 0 else 0.0
- def _idle_due(launcher: Optional[tuple[int, float]]) -> Optional[float]:
- """Idle seconds when the screen is due for the idle stop now, else None. ``launcher`` (``(pid,
- create_time)``) limits it to that one screen: a replacement started since is never due."""
- limit = idle_stop_seconds()
- if limit <= 0 or not is_running():
- return None
- if launcher is not None:
- current = _launcher_identity()
- if current is None or current[0] != launcher[0] or abs(current[1] - launcher[1]) >= 0.01:
- return None
- idle = idle_seconds()
- if idle is None or idle < limit:
- return None
- from tools.bot_desktop import lease as _bd_lease
- abandoned_after = max(limit * HUMAN_LEASE_ABANDONED_WINDOWS, HUMAN_LEASE_ABANDONED_MIN_S)
- if _bd_lease.get().holder == _bd_lease.HUMAN and idle < abandoned_after:
- return None
- return idle
- def stop_if_idle(launcher: Optional[tuple[int, float]] = None) -> bool:
- """Stop this profile's screen when it has been idle past the limit and no human holds it (a human lease
- with no use for ``HUMAN_LEASE_ABANDONED_WINDOWS`` windows, and at least ``HUMAN_LEASE_ABANDONED_MIN_S``,
- counts as abandoned). True when it was stopped. ``launcher`` names the screen the caller judges (the
- launcher's own check passes it). Stopping a screen for an abandoned human lease also hands that lease back
- to the agent.
- The decision is taken again under ``start.lock`` before anything is killed: a check that waited on the
- lock while the screen was stopped and started again (or used) must not kill the screen it finds then."""
- idle = _idle_due(launcher)
- if idle is None:
- return False
- from tools.bot_desktop import lease as _bd_lease
- held: Optional[_bd_lease.Lease] = None # the lease the final decision saw
- def still_due() -> bool:
- nonlocal held
- # Read before _idle_due reads it: a takeover landing between the two reads then carries a newer
- # epoch than the one released below, so it is kept instead of being handed back.
- held = _bd_lease.get()
- return _idle_due(launcher) is not None
- logger.info("Bot Desktop for profile %s idle for %.0f min; stopping", _profile_name(), idle / 60)
- if not stop(still_due=still_due):
- return False
- if held is not None and held.holder == _bd_lease.HUMAN:
- # An abandoned human lease stopped the screen. Left in place it would fence the agent out of the next
- # screen ("a human has control"); only the lease generation judged abandoned goes, so any takeover
- # since (another viewer, or the same one handing back and taking over again) is kept.
- try:
- _bd_lease.release(held.viewer_id, epoch=held.epoch)
- except OSError as exc: # the screen is already stopped: a full or read-only home must not undo that
- logger.warning("Bot Desktop for profile %s stopped, but its abandoned lease was not released: %s",
- _profile_name(), exc)
- return True
- def published_env() -> dict[str, str]:
- """Variables the launcher wrote once Xfce's private bus existed; empty when the desktop is down.
- Pure file reads on the common path: this is called from every browser / cua-driver env builder, so it
- must not load config (that initializes HERMES_HOME). A sandbox-hosted screen leaves a host-side marker at
- start; only its presence routes to the sandbox probe."""
- from tools.bot_desktop import sandbox_host
- if sandbox_host._read_marker():
- return _sandbox_published_env()
- if _launcher_pid() is None:
- return {}
- raw = _read(state_dir() / "env")
- if not raw:
- return {}
- out: dict[str, str] = {}
- for line in raw.splitlines():
- key, sep, value = line.partition("=")
- if sep:
- out[key.strip()] = value.strip()
- return out
- def rfb_socket_path() -> Optional[Path]:
- """Host path of the RFB socket; None when down OR when the screen lives in a sandbox (use
- :func:`open_rfb_stream` there: the socket is not on this filesystem)."""
- from tools.bot_desktop import sandbox_host
- if sandbox_host._read_marker():
- return None
- sock = state_dir() / "rfb.sock"
- return sock if _launcher_pid() is not None and sock.exists() else None
- def in_sandbox() -> bool:
- """True when this profile's screen is placed inside the terminal backend (policy; reads config)."""
- return placement.resolve().where == placement.TERMINAL
- def sandbox_screen_running() -> bool:
- """True when a sandbox-hosted screen is UP for this profile: the start marker exists AND the sandbox it
- names is alive — registered in this process, or (after a gateway restart emptied the registry) the
- marker's recorded container still running. Only a sandbox that is provably gone (its container removed
- out from under us) drops the marker, so the next start rebuilds instead of the browser exec-ing into a
- dead container; an unregistered-but-alive one is re-attached by ``_sandbox_env(create=True)`` at the
- next spawn (the terminal planner reuses the persisted container by label)."""
- from tools.bot_desktop import sandbox_host
- marker = sandbox_host._read_marker()
- if not marker:
- return False
- if _sandbox_env(create=False) is not None:
- return True
- if sandbox_host.marker_sandbox_alive(marker):
- return True
- sandbox_host._marker().unlink(missing_ok=True)
- return False
- def _owned_sandbox_env():
- """The environment hosting the screen the marker records, re-attaching after a restart when the recorded
- sandbox is still alive; None when there is no marker or its sandbox is gone. Never builds a sandbox for
- a screen that is not there."""
- from tools.bot_desktop import sandbox_host
- marker = sandbox_host._read_marker()
- if not marker:
- return None
- env = _sandbox_env(create=False)
- if env is not None:
- return env
- if sandbox_host.marker_sandbox_alive(marker):
- return _sandbox_env(create=True)
- return None
- def is_running() -> bool:
- return bool(published_env().get("DISPLAY"))
- def open_rfb_stream() -> subprocess.Popen:
- """Popen whose stdin/stdout carry RFB bytes for a sandbox-hosted screen (``in_sandbox()`` only)."""
- from tools.bot_desktop import sandbox_host
- env = _sandbox_env(create=False)
- if env is None:
- raise RuntimeError("the sandbox hosting this screen is not running")
- return sandbox_host.open_rfb_stream(env, _profile_name())
- def geometry() -> str:
- from hermes_cli.config import load_config_readonly
- cfg = load_config_readonly().get("bot_desktop") or {}
- return str(cfg.get("geometry") or "1440x900")
- def status(profile: Optional[str] = None) -> DesktopStatus:
- from tools.bot_desktop import browser as _bd_browser
- from tools.bot_desktop import resources
- from tools.bot_desktop import sandbox_host
- where = placement.resolve()
- if where.where == placement.TERMINAL or sandbox_host._read_marker():
- # A screen already running inside a sandbox is reported (and stoppable) even after the placement
- # setting moved: the recorded owner wins over the current policy until it is stopped.
- return _sandbox_status(profile, where)
- missing: list[str] = missing_binaries() if is_supported_host() else list(REQUIRED_BINARIES)
- pid = _launcher_pid()
- env = published_env()
- running = pid is not None and bool(env.get("DISPLAY"))
- mem = resources.memory_info() if is_supported_host() else resources.MemoryInfo(None, None)
- return DesktopStatus(
- profile=profile or _profile_name(),
- supported=is_supported_host(),
- installed=not missing,
- missing=missing,
- running=running,
- pid=pid,
- display=env.get("DISPLAY"),
- socket=str(rfb_socket_path()) if rfb_socket_path() else None,
- geometry=geometry(),
- install_command=install_command() if missing else None,
- browser=_bd_browser.executable() if is_supported_host() else None,
- # A running screen is never "blocked": the check guards the allocation, not the session.
- blocker=None if running or missing or not is_supported_host() else resources.memory_blocker(mem),
- memory_available_mb=mem.available_mb,
- memory_limit_mb=mem.limit_mb,
- )
- def _sandbox_status(profile: Optional[str], where) -> DesktopStatus:
- """Status of a sandbox-placed screen. Package presence is only known once the sandbox exists; before
- that the pane shows "installed" with the image hint carried in ``install_command`` so Start can explain."""
- from tools.bot_desktop import sandbox_host
- env = _owned_sandbox_env() or _sandbox_env(create=False)
- missing = sandbox_host.missing_binaries(env) if env is not None else []
- published = sandbox_host.published_env(env, profile or _profile_name()) if env is not None else {}
- running = bool(published.get("DISPLAY"))
- # No install_command: the pane's Install button runs apt on the HOST, which is the wrong machine here.
- # A sandbox missing the stack is a blocker (shown in place of Start) naming the image that has it.
- blocker = None
- image_switch = None
- if missing:
- advice = sandbox_host.plugin_image_advice(where.backend) or (
- f"use {sandbox_host.SANDBOX_IMAGE_HINT} as terminal.{where.backend}_image (the default sandbox base "
- f"plus the desktop stack)")
- blocker = (f"The terminal backend's sandbox image lacks {', '.join(missing)}. {advice[0].upper()}{advice[1:]}, "
- f"or set bot_desktop.placement: gateway.")
- if where.backend == "docker":
- # The usual reason on an upgraded install: the persisted container predates the default
- # flip and was kept on purpose. The pane offers the switch instead of a config hint.
- from hermes_cli.sandbox_image_switch import pending
- sw = pending()
- if sw is not None:
- image_switch = {"current_image": sw.current_image, "target_image": sw.target_image,
- "containers": len(sw.containers)}
- blocker = (f"Your sandbox container still runs {sw.current_image}, which has no desktop. "
- f"Switch it to {sw.target_image}: files in /root and /workspace stay, packages "
- f"installed inside the container are reinstalled on demand.")
- return DesktopStatus(
- profile=profile or _profile_name(),
- supported=True,
- installed=True,
- missing=missing,
- running=running,
- pid=None,
- display=published.get("DISPLAY"),
- socket=None,
- geometry=geometry(),
- install_command=None,
- browser=None,
- blocker=blocker,
- memory_available_mb=None,
- memory_limit_mb=None,
- placement=f"{placement.TERMINAL}:{where.backend}",
- image_switch=image_switch,
- )
- def _profile_name() -> str:
- try:
- from hermes_cli.profiles import get_active_profile_name
- return get_active_profile_name() or "default"
- except Exception:
- return "default"
- # The gate lives in ``resources`` so start() and status() cannot disagree about it. Measured in the
- # official image: gateway idle 304 MiB, +216 for Xvnc/Xfce, 1073 MiB with one Chromium page. The OOM
- # killer picks by score, so on a small instance the casualty is the dashboard or the gateway, not the
- # desktop that caused the pressure.
- def start(*, wait_seconds: float = 15.0) -> DesktopStatus:
- """Start this profile's desktop (idempotent). Blocks until the launcher publishes its env file or
- ``wait_seconds`` pass; raises ``RuntimeError`` naming the blocker.
- ``bot_desktop.placement`` decides WHERE: inside the configured terminal backend (docker/ssh/a streaming plugin backend;
- ``sandbox_host``), or on the gateway host (the rest of this function). A sandbox backend that cannot host
- a screen refuses rather than silently falling back to the host beside it.
- Two locks: the per-profile ``start.lock``, held from the running-check to the launcher's publish so two
- start() calls for one profile spawn one launcher (the loser sees it running), and the host-wide
- display-allocation lock, held only until this Xvnc has written ``/tmp/.X<n>-lock`` (a second profile
- picking the same number before that would fail and its stale-lock cleanup could remove our socket).
- Holding it for the whole Xfce bring-up serialized every profile's start behind one desktop launch."""
- where = placement.resolve()
- if where.where == placement.REFUSED:
- raise RuntimeError(where.reason)
- if where.where == placement.TERMINAL:
- return _start_in_sandbox(wait_seconds)
- if not is_supported_host():
- raise RuntimeError("Bot Desktop runs on Linux gateway hosts only")
- missing = missing_binaries()
- if missing:
- # Three dead ends: an operator told "unprivileged, no sudo" while running as root hunts the wrong bug.
- need = f"Bot Desktop needs {', '.join(missing)} on the gateway host"
- if package_manager() is None:
- raise RuntimeError(
- f"{need}, and no supported package manager (apt/dnf/pacman) is available to install them. "
- "Install TigerVNC (Xvnc) and the Xfce core components with this distro's own tooling.")
- if not installable():
- raise RuntimeError(
- f"{need}, and this host cannot install them: the process is unprivileged and there is no "
- "sudo. On the published Docker image the packages have to be baked in, so this needs a "
- "newer image rather than an install.")
- raise RuntimeError(f"{need}. Install: {install_command()}")
- sd = state_dir()
- sd.mkdir(parents=True, exist_ok=True)
- os.chmod(sd, 0o700)
- with _flocked(sd / "start.lock"):
- if _launcher_pid() is not None and published_env().get("DISPLAY"):
- return status()
- from tools.bot_desktop import resources
- floor = resources.min_free_mb()
- mem = resources.memory_info()
- if (blocker := resources.memory_blocker(mem, need=floor)) is not None:
- raise RuntimeError(blocker)
- if mem.available_mb is not None and mem.available_mb < resources.tight_headroom_mb(floor):
- logger.warning(
- "Bot Desktop starting with %d MB available; a browser with a few pages open can use most "
- "of that.", mem.available_mb)
- if _launcher_pid() is None:
- _reap_orphaned_server(sd)
- _ALLOC_LOCK.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
- try:
- return _spawn_and_wait(sd, wait_seconds)
- except RuntimeError:
- # _pick_display reuses the recorded number first: left in place after a failed launch (e.g. Xvnc
- # 'server already running' on it), every retry would pick the same number and the profile wedges.
- (sd / "display").unlink(missing_ok=True)
- raise
- def _spawn_and_wait(sd: Path, wait_seconds: float) -> DesktopStatus:
- """Caller holds ``start.lock``. Takes ``_ALLOC_LOCK`` itself, from picking the number to Xvnc's claim."""
- with contextlib.ExitStack() as alloc:
- alloc.enter_context(_flocked(_ALLOC_LOCK))
- num = _pick_display()
- (sd / "display").write_text(str(num), encoding="utf-8")
- env_file = sd / "env"
- env_file.unlink(missing_ok=True)
- # The agent drives this desktop and its dock opens a terminal, so it starts from the
- # scrubbed child env like any other agent child, keeping the user's HOME.
- from tools.environments.local import served_profile_child_env
- child_env = served_profile_child_env(inherit_credentials=False)
- child_env["HOME"] = child_env["HERMES_REAL_HOME"]
- for key in ("DISPLAY", "XAUTHORITY", "WAYLAND_DISPLAY", "DBUS_SESSION_BUS_ADDRESS", "SESSION_MANAGER"):
- child_env.pop(key, None)
- from hermes_cli._launchers import installation_command
- from tools.bot_desktop.idle_watch import next_check_in
- child_env.update({
- "HERMES_BD_PROFILE": _profile_name(),
- "HERMES_BD_DISPLAY_NUM": str(num),
- "HERMES_BD_SOCKET": str(sd / "rfb.sock"),
- "HERMES_BD_XAUTH": str(sd / "Xauthority"),
- "HERMES_BD_ENV_FILE": str(env_file),
- "HERMES_BD_CONFIG_HOME": str(sd / "xdg"),
- "HERMES_BD_GEOMETRY": geometry(),
- # The launcher's idle self-stop runs tools.bot_desktop.idle_watch through the install's stable
- # command (the ``hermes --run-module`` launcher on a store install, which follows Python pin changes
- # and picks the current dependency generation at every run; this interpreter elsewhere), in this
- # home, first once the screen could be due (nothing is idle at start). The screen outlives updates,
- # so the inherited PYTHONPATH generation is never pinned, and on a store install neither is this
- # interpreter.
- "HERMES_BD_IDLE_CMD": shlex.join(installation_command(
- Path(__file__).resolve().parents[2], module="tools.bot_desktop.idle_watch")),
- "HERMES_BD_HOME": str(get_hermes_home()),
- "HERMES_BD_IDLE_FIRST_DELAY": str(next_check_in(idle_stop_seconds(), 0.0)),
- })
- from tools.bot_desktop.browser import dock_exec_line, dock_launch
- if (browser := dock_launch()) is not None:
- # The bare executable (the launcher checks it exists) and the ready-made, spec-quoted Exec= line.
- child_env["HERMES_BD_BROWSER_EXEC"] = browser[0]
- child_env["HERMES_BD_BROWSER_EXEC_LINE"] = dock_exec_line(*browser)
- # A new file per start: the log is a diagnostic for THIS launch, and nothing rotates it otherwise. Not
- # truncated in place: the old screen's idle check (in a session of its own) may still hold the old file
- # open, and its late write would land in this one behind a run of NUL bytes.
- (sd / "launcher.log").unlink(missing_ok=True)
- log = open(sd / "launcher.log", "wb")
- # The profile's screen outlives whichever CLI/gateway started it, on purpose: launcher.pid carries
- # (pid, create_time) so any later process reports/stops it, and the idle stop (idle_stop_minutes) ends it.
- proc = subprocess.Popen( # windows-footgun: ok — Linux-only runtime (is_supported_host)
- ["bash", str(_LAUNCHER)], env=child_env, stdin=subprocess.DEVNULL, stdout=log, stderr=log,
- start_new_session=True, close_fds=True) # detached-spawn: ok — per-profile screen, outlives its starter
- log.close()
- born = _create_time(proc.pid)
- (sd / "launcher.pid").write_text(f"{proc.pid} {born if born is not None else 0}", encoding="utf-8")
- deadline = time.monotonic() + wait_seconds
- while time.monotonic() < deadline:
- if _x_lock_pid(num) is not None:
- alloc.close() # the number is Xvnc's now; other profiles may allocate (idempotent)
- if proc.poll() is not None:
- tail = (sd / "launcher.log").read_bytes()[-2000:].decode("utf-8", "replace")
- raise RuntimeError(f"Bot Desktop launcher exited with {proc.returncode}:\n{tail}")
- if env_file.exists() and (sd / "rfb.sock").exists():
- logger.info("Bot Desktop for profile %s up on :%s", _profile_name(), num)
- touch_activity()
- return status()
- time.sleep(0.1)
- # Giving up must take the launch down: left alone, the launcher publishes DISPLAY and rfb.sock a moment
- # later and a screen whose start() reported failure stays up as "running". The launcher is its own
- # session leader, so its group is exactly this launch (Xvnc, dbus, Xfce) and nothing else.
- _kill_group_then_wait(proc.pid, proc.pid)
- proc.wait()
- for name in ("launcher.pid", "env", "rfb.sock"):
- (sd / name).unlink(missing_ok=True)
- raise RuntimeError(f"Bot Desktop did not publish its display within {wait_seconds:.0f}s (see {sd / 'launcher.log'})")
- def _sandbox_env(*, create: bool):
- """The terminal environment hosting this profile's screen, or None. ``create=False`` for status probes
- (a status call must never build a container)."""
- return placement.terminal_environment(create=create)
- def _start_in_sandbox(wait_seconds: float) -> DesktopStatus:
- from tools.bot_desktop import sandbox_host
- env = _sandbox_env(create=True)
- if env is None:
- raise RuntimeError("the terminal backend's sandbox could not be started, so there is nowhere to put the screen")
- sd = state_dir()
- sd.mkdir(parents=True, exist_ok=True)
- os.chmod(sd, 0o700)
- with _flocked(sd / "start.lock"):
- # The dock's Browser icon runs the sandbox's own Playwright Chromium on the profile the agent's browser
- # uses there: the human's browser is the bot's browser, as on the host.
- from tools.bot_desktop.browser import dock_exec_line
- exe = sandbox_host.chromium_executable(env)
- browser = (exe, dock_exec_line(exe, sandbox_host.browser_profile_dir(env), sandbox_bypass=True)) if exe else (None, None)
- published = sandbox_host.start(env, _profile_name(), geometry=geometry(), wait_seconds=max(wait_seconds, 20.0),
- browser_exec=browser[0], browser_exec_line=browser[1])
- (sd / "env").write_text("".join(f"{k}={v}\n" for k, v in published.items()), encoding="utf-8")
- touch_activity()
- return status()
- def _sandbox_published_env() -> dict[str, str]:
- """Published env of a sandbox-hosted screen (the caller saw the host-side marker written at start)."""
- from tools.bot_desktop import sandbox_host
- env = _sandbox_env(create=False)
- if env is None:
- return {}
- return sandbox_host.published_env(env, _profile_name())
- def stop(still_due: Optional[Callable[[], bool]] = None) -> bool:
- """Stop this profile's desktop; True when a running launcher (or the X server a dead one left behind)
- was signalled. ``still_due`` is asked again before anything is killed (under ``start.lock`` for a screen on
- this host); False there leaves the screen running."""
- from tools.bot_desktop import sandbox_host
- if sandbox_host._read_marker() or placement.resolve().where == placement.TERMINAL:
- if still_due is not None and not still_due():
- return False
- # The marker names the sandbox that owns the screen; stop THAT one (re-attaching after a restart),
- # whatever the placement setting says now. A marker whose sandbox is gone is simply dropped.
- env = _owned_sandbox_env()
- stopped = sandbox_host.stop(env, _profile_name()) if env is not None else False
- sandbox_host._marker().unlink(missing_ok=True)
- for name in ("env", "activity"):
- (state_dir() / name).unlink(missing_ok=True)
- return stopped
- if not is_supported_host():
- return False
- sd = state_dir()
- sd.mkdir(parents=True, exist_ok=True)
- with _flocked(sd / "start.lock"):
- if still_due is not None and not still_due():
- logger.info("Bot Desktop for profile %s changed while the idle stop waited; kept", _profile_name())
- return False
- return _stop_locked(sd)
- def _stop_locked(sd: Path) -> bool:
- pid = _launcher_pid()
- if pid is None:
- reaped = _reap_orphaned_server(sd)
- (sd / "env").unlink(missing_ok=True)
- return reaped
- # The launcher runs in its own session; killing the group takes Xvnc, dbus and Xfce with it.
- _kill_group_then_wait(pid, pid, grace=5.0)
- for name in ("launcher.pid", "env", "activity", "idle-watch.out"):
- (sd / name).unlink(missing_ok=True)
- return True
|