runtime.py 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928
  1. """Bot Desktop runtime: one headless Xfce desktop per Hermes profile, served over RFB on a private
  2. Unix socket, viewed and driven from Hermes Desktop.
  3. Layout under ``<HERMES_HOME>/bot-desktop/``: ``display`` (allocated X display number), ``rfb.sock``
  4. (Xvnc RFB Unix socket, 0600), ``Xauthority``, ``env`` (DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS
  5. published by the launcher once Xfce's bus exists), ``launcher.pid``, ``launcher.log``, ``xdg/``
  6. (per-profile XDG_CONFIG_HOME so two profiles never share xfconf). Everything is profile-scoped via
  7. ``get_hermes_home()`` so N profiles in one gateway get N desktops: one screen per bot on the shared
  8. machine.
  9. The launcher is ``launcher.sh`` next to this module; :func:`desktop_env` is what cua-driver and headed
  10. Chromium spawns merge in so the agent acts on this profile's screen and nowhere else.
  11. """
  12. from __future__ import annotations
  13. import contextlib
  14. import logging
  15. import math
  16. import os
  17. import shlex
  18. import shutil
  19. import signal
  20. import subprocess
  21. import sys
  22. import time
  23. from dataclasses import dataclass
  24. from pathlib import Path
  25. from typing import Callable, Dict, Optional
  26. from agent.deadline import poll_until
  27. from hermes_constants import get_hermes_home
  28. from tools.bot_desktop import placement
  29. logger = logging.getLogger(__name__)
  30. _LAUNCHER = Path(__file__).with_name("launcher.sh")
  31. # Display numbers below 10 collide with real seats and default Xvfb recipes (:99 is popular too); scan a
  32. # private band and record the choice so restarts reuse it.
  33. _DISPLAY_MIN, _DISPLAY_MAX = 20, 89
  34. # Binaries the launcher execs; the package hint is per distro family.
  35. REQUIRED_BINARIES = ("Xvnc", "xfwm4", "xfce4-panel", "xfdesktop", "xfsettingsd", "dbus-run-session",
  36. "xauth", "xdpyinfo", "setxkbmap", "xprop")
  37. # Which package in each distro list ships each required binary. Fedora retired the xorg-x11-utils /
  38. # xorg-x11-server-utils umbrellas (per-binary packages since F35) and dnf5 refuses the whole transaction on
  39. # one unknown name, so every binary must map to a package that still resolves; the test suite checks that
  40. # each mapped package is in PACKAGES for its manager.
  41. BINARY_PACKAGES = {
  42. "apt": {"Xvnc": "tigervnc-standalone-server", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel",
  43. "xfdesktop": "xfdesktop4", "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus-x11",
  44. "xauth": "xauth", "xdpyinfo": "x11-utils", "setxkbmap": "x11-xkb-utils", "xprop": "x11-utils"},
  45. # tigervnc-x11-server is the real package (tigervnc-server-minimal is only a Provides on it); dbus-run-session
  46. # is in dbus-daemon (dbus-x11 ships dbus-launch only).
  47. "dnf": {"Xvnc": "tigervnc-x11-server", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel",
  48. "xfdesktop": "xfdesktop", "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus-daemon",
  49. "xauth": "xorg-x11-xauth", "xdpyinfo": "xdpyinfo", "setxkbmap": "setxkbmap", "xprop": "xprop"},
  50. "pacman": {"Xvnc": "tigervnc", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel", "xfdesktop": "xfdesktop",
  51. "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus",
  52. "xauth": "xorg-xauth", "xdpyinfo": "xorg-xdpyinfo", "setxkbmap": "xorg-setxkbmap", "xprop": "xorg-xprop"},
  53. }
  54. PACKAGES = {
  55. "apt": ["tigervnc-standalone-server", "xfce4-panel", "xfwm4", "xfdesktop4", "xfce4-settings",
  56. "xfce4-terminal", "dbus-x11", "x11-xserver-utils", "x11-utils", "x11-xkb-utils", "xauth",
  57. "fonts-dejavu-core"],
  58. "dnf": ["tigervnc-x11-server", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings",
  59. "xfce4-terminal", "dbus-daemon", "xsetroot", "xset", "xdpyinfo", "xprop", "xorg-x11-xauth", "setxkbmap",
  60. "dejavu-sans-fonts"],
  61. "pacman": ["tigervnc", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", "xfce4-terminal", "dbus",
  62. "xorg-xsetroot", "xorg-xset", "xorg-xdpyinfo", "xorg-xprop", "xorg-xauth", "xorg-setxkbmap",
  63. "ttf-dejavu"],
  64. }
  65. def state_dir() -> Path:
  66. return get_hermes_home() / "bot-desktop"
  67. def is_supported_host() -> bool:
  68. return sys.platform.startswith("linux")
  69. def missing_binaries() -> list[str]:
  70. return [b for b in REQUIRED_BINARIES if shutil.which(b) is None]
  71. def package_manager() -> Optional[str]:
  72. for pm in ("apt-get", "dnf", "pacman"):
  73. if shutil.which(pm):
  74. return "apt" if pm == "apt-get" else pm
  75. return None
  76. def install_command() -> Optional[str]:
  77. """The distro command that installs the Bot Desktop packages, as the human would type it on THIS host:
  78. prefixed with ``sudo`` unless Hermes already runs as root, so it is both what the pane shows and what
  79. :mod:`tools.bot_desktop.install` runs. ``None`` when no package manager is present.
  80. Not a promise that it can run here: see :func:`installable`. The published Docker image supervises
  81. every service under ``s6-setuidgid hermes`` (UID 10000 by default) and ships no ``sudo`` binary, so an
  82. install on a hosted instance is impossible no matter what this returns."""
  83. pm = package_manager()
  84. if pm is None:
  85. return None
  86. pkgs = " ".join(PACKAGES[pm])
  87. body = {
  88. "apt": f"apt-get install -y --no-install-recommends {pkgs}",
  89. "dnf": f"dnf install -y {pkgs}",
  90. "pacman": f"pacman -S --needed --noconfirm {pkgs}",
  91. }[pm]
  92. return body if is_root() else f"sudo {body}"
  93. def is_root() -> bool:
  94. return hasattr(os, "geteuid") and os.geteuid() == 0
  95. def installable() -> bool:
  96. """Whether :func:`install_command` could actually succeed on this host.
  97. False on an unprivileged process with no ``sudo`` to reach for, which is exactly the published Docker
  98. image: services drop to the ``hermes`` user and no ``sudo`` binary is installed. The packages can only
  99. arrive in the image there, so :func:`start` says that instead of printing a sudo line the user has no
  100. way to run. ``status()`` still reports ``install_command`` for the pane; surfacing this there needs a
  101. wire-contract change and is deliberately out of scope.
  102. """
  103. if package_manager() is None:
  104. return False
  105. return is_root() or shutil.which("sudo") is not None
  106. @dataclass
  107. class DesktopStatus:
  108. profile: str
  109. supported: bool
  110. installed: bool
  111. missing: list[str]
  112. running: bool
  113. pid: Optional[int]
  114. display: Optional[str]
  115. socket: Optional[str]
  116. geometry: str
  117. install_command: Optional[str]
  118. browser: Optional[str] # headed Chromium the dock's Browser icon and the agent's browser share; None = no headed browser
  119. blocker: Optional[str] = None # why start() would refuse right now (memory); None = may start
  120. memory_available_mb: Optional[int] = None
  121. memory_limit_mb: Optional[int] = None
  122. placement: str = "gateway" # "gateway" | "terminal:<backend>" — where Xvnc runs
  123. image_switch: Optional[dict[str, object]] = None # pending default-image switch the pane can approve
  124. def as_dict(self) -> dict[str, object]:
  125. return dict(self.__dict__)
  126. def _read(path: Path) -> Optional[str]:
  127. try:
  128. return path.read_text(encoding="utf-8-sig").strip() or None
  129. except OSError:
  130. return None
  131. def _pid_alive(pid: int) -> bool:
  132. """A zombie is dead for our purposes: a SIGKILLed launcher stays a zombie in the gateway until the next
  133. Popen reaps it, and reporting it as running would hide its orphaned X server behind a live status."""
  134. import psutil
  135. try:
  136. return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE
  137. except psutil.Error:
  138. return False
  139. def _create_time(pid: int) -> Optional[float]:
  140. import psutil
  141. try:
  142. return psutil.Process(pid).create_time()
  143. except (psutil.Error, OverflowError, ValueError):
  144. return None
  145. def _launcher_identity() -> Optional[tuple[int, float]]:
  146. """``(pid, create_time)`` of the live launcher, or None. ``launcher.pid`` holds ``"<pid> <create_time>"``:
  147. a recycled pid with a different start time is somebody else's process and must never be reported as
  148. ours nor killed by :func:`stop`. The pre-identity single-number format is treated as not running."""
  149. raw = _read(state_dir() / "launcher.pid")
  150. pid_s, _, born_s = (raw or "").partition(" ")
  151. if not pid_s.isdigit() or not born_s:
  152. return None
  153. try:
  154. pid, born = int(pid_s), float(born_s)
  155. except ValueError:
  156. return None
  157. actual = _create_time(pid)
  158. return (pid, born) if actual is not None and abs(actual - born) < 0.01 and _pid_alive(pid) else None
  159. def _launcher_pid() -> Optional[int]:
  160. """The live launcher's pid, or None (see :func:`_launcher_identity`)."""
  161. identity = _launcher_identity()
  162. return identity[0] if identity is not None else None
  163. def _recorded_launcher_pid() -> Optional[int]:
  164. """The pid ``launcher.pid`` names, alive or not (the orphan sweep matches process groups against it)."""
  165. pid_s, _, born_s = (_read(state_dir() / "launcher.pid") or "").partition(" ")
  166. return int(pid_s) if pid_s.isdigit() and born_s else None
  167. _X_LOCK_DIR = Path("/tmp") # no-tmp: ok — X servers write .X<n>-lock here by protocol (tests point it at a scratch dir)
  168. _X_UNIX_TABLE = Path("/proc/net/unix") # the kernel's list of bound Unix sockets (tests point it at a fixture)
  169. def _x_lock_pid(num: int) -> Optional[int]:
  170. try:
  171. return int((_X_LOCK_DIR / f".X{num}-lock").read_text(encoding="utf-8-sig").strip())
  172. except (OSError, ValueError):
  173. return None
  174. def _x_socket_bound(num: int) -> bool:
  175. """A running X server keeps ``@/tmp/.X11-unix/X<n>`` (abstract namespace) bound for its whole life; the
  176. kernel drops it only when the process exits. A /tmp reaper can remove the lock file under a live Xvnc,
  177. and only this binding then still says the number is taken (a new server on it dies 'already running')."""
  178. try:
  179. lines = _X_UNIX_TABLE.read_text(encoding="utf-8-sig").splitlines()
  180. except OSError:
  181. return False
  182. # no-tmp: ok — detects the X server's display socket at the path the X11 protocol fixes
  183. return any(line.split()[-1].lstrip("@") == f"/tmp/.X11-unix/X{num}" for line in lines if line.strip())
  184. def _display_in_use(num: int) -> bool:
  185. """A live X server owns ``:num``: its lock file names a running pid, or its X11 socket is bound. A lock
  186. left by a crashed server (dead pid, no socket) does not count, so the number can be reclaimed."""
  187. pid = _x_lock_pid(num)
  188. return (pid is not None and _pid_alive(pid)) or _x_socket_bound(num)
  189. def _reap_orphaned_server(sd: Path) -> bool:
  190. """Caller holds ``start.lock`` and has established that no live launcher exists. The launcher runs Xvnc in
  191. its own session, so a SIGKILLed launcher leaves the X server alive, holding the display and ``rfb.sock``;
  192. ``status()`` keys on the launcher and says stopped, and a naive restart allocates a second server next
  193. to it and overwrites the socket path both now claim. The X lock of the recorded display names that
  194. server: it is ours when it sits in the dead launcher's process group or its command line binds OUR
  195. socket. When a /tmp reaper took the lock too (or the failed launch dropped ``display``), the socket path
  196. on the Xvnc command line is the remaining handle — without it one X server leaks per occurrence. Kill it
  197. (group first), drop the state it left, and report whether anything was signalled."""
  198. import psutil
  199. def _binds_our_socket(cmdline: list) -> bool:
  200. return "Xvnc" in Path(cmdline[0] if cmdline else "").name and str(sd / "rfb.sock") in cmdline
  201. recorded = _read(sd / "display")
  202. pid = _x_lock_pid(int(recorded)) if recorded and recorded.isdigit() else None
  203. if pid is None or not _pid_alive(pid):
  204. pid = next((p.pid for p in psutil.process_iter(["cmdline"]) if _binds_our_socket(p.info["cmdline"] or [])), None)
  205. if pid is None:
  206. return False
  207. launcher = _recorded_launcher_pid()
  208. try:
  209. pgid = os.getpgid(pid) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start/stop)
  210. cmdline = psutil.Process(pid).cmdline()
  211. except (ProcessLookupError, psutil.Error):
  212. return False
  213. if pgid != launcher and not _binds_our_socket(cmdline):
  214. return False # somebody else's server took the number after we died; never touch it
  215. logger.warning("Bot Desktop launcher %s is gone but its X server (pid %s) survived on :%s; reaping",
  216. launcher, pid, recorded or "?")
  217. _kill_group_then_wait(pgid if pgid == launcher else None, pid)
  218. if recorded:
  219. (_X_LOCK_DIR / f".X{recorded}-lock").unlink(missing_ok=True)
  220. for name in ("launcher.pid", "env", "rfb.sock"):
  221. (sd / name).unlink(missing_ok=True)
  222. return True
  223. def _kill_group_then_wait(pgid: Optional[int], pid: int, grace: float = 2.0) -> None:
  224. """SIGTERM the group (or the lone pid), SIGKILL whatever is still there after ``grace``."""
  225. def _signal(sig: int) -> None:
  226. with contextlib.suppress(ProcessLookupError, PermissionError):
  227. if pgid is not None:
  228. os.killpg(pgid, sig) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start/stop)
  229. else:
  230. os.kill(pid, sig)
  231. def _anything_left() -> bool:
  232. # The leader dying first is the common case (bash exits on TERM, Xvnc traps it); the group is
  233. # done only when killpg(0) finds nobody, else a TERM-ignoring descendant keeps the display.
  234. if pgid is None:
  235. return _pid_alive(pid)
  236. try:
  237. os.killpg(pgid, 0) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start/stop)
  238. except ProcessLookupError:
  239. return False
  240. except PermissionError:
  241. return True
  242. return True
  243. def _reap_if_ours() -> None:
  244. # The launcher was Popen'd by whichever gateway started it; a later gateway that stops it holds
  245. # no Popen, so the dead leader would sit as a zombie in our table (and count as "left").
  246. with contextlib.suppress(ChildProcessError, OSError):
  247. os.waitpid(pid, os.WNOHANG) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start/stop)
  248. def _gone() -> bool:
  249. _reap_if_ours()
  250. return not _anything_left()
  251. _signal(signal.SIGTERM)
  252. if poll_until(_gone, grace, 0.05):
  253. return
  254. _signal(signal.SIGKILL) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start/stop)
  255. time.sleep(0.1)
  256. _reap_if_ours()
  257. # Host-wide (every profile allocates from one band), so it lives outside any profile home. A predictable
  258. # name must not be squattable: XDG_RUNTIME_DIR is the boundary — 0700 from logind, or from
  259. # docker/stage2-hook.sh in containers, which have none.
  260. _ALLOC_LOCK = Path(os.environ.get("XDG_RUNTIME_DIR") or Path.home() / ".cache") / "hermes-bot-desktop-alloc.lock"
  261. @contextlib.contextmanager
  262. def _flocked(path: Path):
  263. import fcntl # windows-footgun: ok — Linux-only runtime (is_supported_host gates start)
  264. with open(path, "a+", encoding="utf-8") as fh: # windows-footgun: ok — Linux-only runtime
  265. fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
  266. try:
  267. yield fh
  268. finally:
  269. fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
  270. def _pick_display() -> int:
  271. """Caller holds ``_ALLOC_LOCK``. The recorded number is only reused when no OTHER server holds it now:
  272. after profile A stops, B may have taken A's number, and A's launcher must never unlink B's socket."""
  273. recorded = _read(state_dir() / "display")
  274. if recorded and recorded.isdigit() and not _display_in_use(int(recorded)):
  275. return int(recorded)
  276. for num in range(_DISPLAY_MIN, _DISPLAY_MAX + 1):
  277. if not _display_in_use(num):
  278. return num
  279. raise RuntimeError("no free X display number in the Bot Desktop band")
  280. def _allocate_display() -> int:
  281. _ALLOC_LOCK.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
  282. with _flocked(_ALLOC_LOCK):
  283. return _pick_display()
  284. def desktop_env(base_env: Optional[dict[str, str]] = None) -> dict[str, str]:
  285. """``base_env`` (default ``os.environ``) with this profile's DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS
  286. merged in when its desktop is running. Unchanged otherwise, so hosts with a real seat keep it.
  287. Pure: never starts anything (it is called from env builders, status probes and tests)."""
  288. env = dict(os.environ if base_env is None else base_env)
  289. published = published_env()
  290. if published:
  291. touch_activity() # a browser / cua-driver spawn is the agent using its screen
  292. env.update(published)
  293. env.pop("WAYLAND_DISPLAY", None) # X11 desktop; a leaked Wayland socket flips GTK/Chromium backends
  294. return env
  295. def ensure_started_for_tool() -> None:
  296. """Tool-boundary hook (``computer_use`` dispatch and the headed Chromium spawn sites of the browser tool).
  297. Under ``terminal`` placement the sandbox screen is brought up on first use unconditionally: the sandbox is
  298. the boundary the user chose, a screen inside it touches nothing outside it, and the tools below refuse
  299. rather than run on the host when it is not up. On the gateway host, ``bot_desktop.auto_start`` (opt-in,
  300. default off) starts a Linux host's screen when it has NO display and the packages installed, so a headless
  301. gateway works the first time. Host failure is not an error here (the tool's own "no display" diagnosis is
  302. the right message then); a sandbox start failure is logged and left for :func:`tool_placement` to raise."""
  303. if published_env().get("DISPLAY"):
  304. touch_activity()
  305. return
  306. where = placement.resolve()
  307. if where.where == placement.REFUSED:
  308. return # tool_placement() raises the reason at the spawn site
  309. if where.where == placement.TERMINAL:
  310. try:
  311. start()
  312. except Exception as exc:
  313. logger.info("Bot Desktop sandbox start deferred to the tool: %s", exc)
  314. return
  315. if not _should_auto_start(os.environ):
  316. return
  317. try:
  318. start()
  319. except Exception as exc:
  320. logger.info("Bot Desktop auto-start skipped: %s", exc)
  321. def tool_placement() -> str:
  322. """Where the browser / cua-driver a tool is about to spawn MUST run: ``placement.GATEWAY`` or
  323. ``placement.TERMINAL``. Policy first, liveness second: a ``terminal`` placement whose screen is down gets
  324. it started here (raising with the blocker when it cannot come up) and a ``refused`` placement raises its
  325. reason. Neither ever yields the host — a screen that happens to be down is not permission to run the
  326. agent's browser outside the sandbox the user chose."""
  327. where = placement.resolve()
  328. if where.where == placement.REFUSED:
  329. raise RuntimeError(where.reason)
  330. if where.where == placement.GATEWAY:
  331. return placement.GATEWAY
  332. if not sandbox_screen_running() or not published_env().get("DISPLAY"):
  333. start()
  334. return placement.TERMINAL
  335. def _should_auto_start(env: dict[str, str]) -> bool:
  336. if not is_supported_host() or env.get("DISPLAY") or env.get("WAYLAND_DISPLAY"):
  337. return False
  338. if missing_binaries():
  339. return False
  340. from hermes_cli.config import load_config_readonly
  341. cfg = load_config_readonly().get("bot_desktop") or {}
  342. return bool(cfg.get("auto_start", False))
  343. # ---- idle auto-stop -------------------------------------------------------------------------------
  344. # A desktop nobody is using still holds ~220 MiB (plus whatever browser was left open). Every use —
  345. # a computer_use action, a browser spawn onto the screen, a viewer attached, a human takeover — stamps
  346. # ``activity``; :func:`stop_if_idle` stops a screen idle past ``bot_desktop.idle_stop_minutes`` unless a
  347. # human is using it (see the lease lapse below). Two callers apply it: the gateway's display watcher and
  348. # the screen's own launcher (``idle_watch``), so a screen whose gateway never came back still stops. The
  349. # next use starts it again.
  350. DEFAULT_IDLE_STOP_MINUTES = 30
  351. # A human holding the screen is never timed out while there is evidence of one: an attached viewer stamps
  352. # ``activity`` (every RFB update it receives, at most once a minute; the panel clock repaints every minute).
  353. # A human lease with no stamp for this many windows, and never less than HUMAN_LEASE_ABANDONED_MIN_S, is
  354. # a link that dropped and never came back, or a corrupt lease file (which reads as human): the screen is
  355. # stopped like any idle one. The floor keeps a short window (idle_stop_minutes: 0.1) from timing out a
  356. # human between two viewer stamps, which can be up to about two minutes apart.
  357. HUMAN_LEASE_ABANDONED_WINDOWS = 4
  358. HUMAN_LEASE_ABANDONED_MIN_S = 600.0
  359. def touch_activity() -> None:
  360. path = state_dir() / "activity"
  361. try:
  362. path.touch()
  363. os.utime(path, None)
  364. except OSError:
  365. pass
  366. def idle_seconds() -> Optional[float]:
  367. """Seconds since the last stamped use; None when the screen never recorded one (falls back to the
  368. env file's publish time so a screen started and then forgotten still ages)."""
  369. for name in ("activity", "env"):
  370. try:
  371. return max(0.0, time.time() - (state_dir() / name).stat().st_mtime)
  372. except OSError:
  373. continue
  374. return None
  375. def idle_stop_seconds() -> float:
  376. from hermes_cli.config import load_config_readonly
  377. cfg = load_config_readonly().get("bot_desktop") or {}
  378. try:
  379. minutes = float(cfg.get("idle_stop_minutes", DEFAULT_IDLE_STOP_MINUTES))
  380. except (TypeError, ValueError):
  381. minutes = DEFAULT_IDLE_STOP_MINUTES
  382. # .inf ("never") is off, like 0, and so are .nan and a window too large to convert to seconds (1e308): none
  383. # can ever be due, and the launcher's next-check arithmetic would raise on them and fail start().
  384. seconds = minutes * 60
  385. return seconds if math.isfinite(seconds) and seconds > 0 else 0.0
  386. def _idle_due(launcher: Optional[tuple[int, float]]) -> Optional[float]:
  387. """Idle seconds when the screen is due for the idle stop now, else None. ``launcher`` (``(pid,
  388. create_time)``) limits it to that one screen: a replacement started since is never due."""
  389. limit = idle_stop_seconds()
  390. if limit <= 0 or not is_running():
  391. return None
  392. if launcher is not None:
  393. current = _launcher_identity()
  394. if current is None or current[0] != launcher[0] or abs(current[1] - launcher[1]) >= 0.01:
  395. return None
  396. idle = idle_seconds()
  397. if idle is None or idle < limit:
  398. return None
  399. from tools.bot_desktop import lease as _bd_lease
  400. abandoned_after = max(limit * HUMAN_LEASE_ABANDONED_WINDOWS, HUMAN_LEASE_ABANDONED_MIN_S)
  401. if _bd_lease.get().holder == _bd_lease.HUMAN and idle < abandoned_after:
  402. return None
  403. return idle
  404. def stop_if_idle(launcher: Optional[tuple[int, float]] = None) -> bool:
  405. """Stop this profile's screen when it has been idle past the limit and no human holds it (a human lease
  406. with no use for ``HUMAN_LEASE_ABANDONED_WINDOWS`` windows, and at least ``HUMAN_LEASE_ABANDONED_MIN_S``,
  407. counts as abandoned). True when it was stopped. ``launcher`` names the screen the caller judges (the
  408. launcher's own check passes it). Stopping a screen for an abandoned human lease also hands that lease back
  409. to the agent.
  410. The decision is taken again under ``start.lock`` before anything is killed: a check that waited on the
  411. lock while the screen was stopped and started again (or used) must not kill the screen it finds then."""
  412. idle = _idle_due(launcher)
  413. if idle is None:
  414. return False
  415. from tools.bot_desktop import lease as _bd_lease
  416. held: Optional[_bd_lease.Lease] = None # the lease the final decision saw
  417. def still_due() -> bool:
  418. nonlocal held
  419. # Read before _idle_due reads it: a takeover landing between the two reads then carries a newer
  420. # epoch than the one released below, so it is kept instead of being handed back.
  421. held = _bd_lease.get()
  422. return _idle_due(launcher) is not None
  423. logger.info("Bot Desktop for profile %s idle for %.0f min; stopping", _profile_name(), idle / 60)
  424. if not stop(still_due=still_due):
  425. return False
  426. if held is not None and held.holder == _bd_lease.HUMAN:
  427. # An abandoned human lease stopped the screen. Left in place it would fence the agent out of the next
  428. # screen ("a human has control"); only the lease generation judged abandoned goes, so any takeover
  429. # since (another viewer, or the same one handing back and taking over again) is kept.
  430. try:
  431. _bd_lease.release(held.viewer_id, epoch=held.epoch)
  432. except OSError as exc: # the screen is already stopped: a full or read-only home must not undo that
  433. logger.warning("Bot Desktop for profile %s stopped, but its abandoned lease was not released: %s",
  434. _profile_name(), exc)
  435. return True
  436. def published_env() -> dict[str, str]:
  437. """Variables the launcher wrote once Xfce's private bus existed; empty when the desktop is down.
  438. Pure file reads on the common path: this is called from every browser / cua-driver env builder, so it
  439. must not load config (that initializes HERMES_HOME). A sandbox-hosted screen leaves a host-side marker at
  440. start; only its presence routes to the sandbox probe."""
  441. from tools.bot_desktop import sandbox_host
  442. if sandbox_host._read_marker():
  443. return _sandbox_published_env()
  444. if _launcher_pid() is None:
  445. return {}
  446. raw = _read(state_dir() / "env")
  447. if not raw:
  448. return {}
  449. out: dict[str, str] = {}
  450. for line in raw.splitlines():
  451. key, sep, value = line.partition("=")
  452. if sep:
  453. out[key.strip()] = value.strip()
  454. return out
  455. def rfb_socket_path() -> Optional[Path]:
  456. """Host path of the RFB socket; None when down OR when the screen lives in a sandbox (use
  457. :func:`open_rfb_stream` there: the socket is not on this filesystem)."""
  458. from tools.bot_desktop import sandbox_host
  459. if sandbox_host._read_marker():
  460. return None
  461. sock = state_dir() / "rfb.sock"
  462. return sock if _launcher_pid() is not None and sock.exists() else None
  463. def in_sandbox() -> bool:
  464. """True when this profile's screen is placed inside the terminal backend (policy; reads config)."""
  465. return placement.resolve().where == placement.TERMINAL
  466. def sandbox_screen_running() -> bool:
  467. """True when a sandbox-hosted screen is UP for this profile: the start marker exists AND the sandbox it
  468. names is alive — registered in this process, or (after a gateway restart emptied the registry) the
  469. marker's recorded container still running. Only a sandbox that is provably gone (its container removed
  470. out from under us) drops the marker, so the next start rebuilds instead of the browser exec-ing into a
  471. dead container; an unregistered-but-alive one is re-attached by ``_sandbox_env(create=True)`` at the
  472. next spawn (the terminal planner reuses the persisted container by label)."""
  473. from tools.bot_desktop import sandbox_host
  474. marker = sandbox_host._read_marker()
  475. if not marker:
  476. return False
  477. if _sandbox_env(create=False) is not None:
  478. return True
  479. if sandbox_host.marker_sandbox_alive(marker):
  480. return True
  481. sandbox_host._marker().unlink(missing_ok=True)
  482. return False
  483. def _owned_sandbox_env():
  484. """The environment hosting the screen the marker records, re-attaching after a restart when the recorded
  485. sandbox is still alive; None when there is no marker or its sandbox is gone. Never builds a sandbox for
  486. a screen that is not there."""
  487. from tools.bot_desktop import sandbox_host
  488. marker = sandbox_host._read_marker()
  489. if not marker:
  490. return None
  491. env = _sandbox_env(create=False)
  492. if env is not None:
  493. return env
  494. if sandbox_host.marker_sandbox_alive(marker):
  495. return _sandbox_env(create=True)
  496. return None
  497. def is_running() -> bool:
  498. return bool(published_env().get("DISPLAY"))
  499. def open_rfb_stream() -> subprocess.Popen:
  500. """Popen whose stdin/stdout carry RFB bytes for a sandbox-hosted screen (``in_sandbox()`` only)."""
  501. from tools.bot_desktop import sandbox_host
  502. env = _sandbox_env(create=False)
  503. if env is None:
  504. raise RuntimeError("the sandbox hosting this screen is not running")
  505. return sandbox_host.open_rfb_stream(env, _profile_name())
  506. def geometry() -> str:
  507. from hermes_cli.config import load_config_readonly
  508. cfg = load_config_readonly().get("bot_desktop") or {}
  509. return str(cfg.get("geometry") or "1440x900")
  510. def status(profile: Optional[str] = None) -> DesktopStatus:
  511. from tools.bot_desktop import browser as _bd_browser
  512. from tools.bot_desktop import resources
  513. from tools.bot_desktop import sandbox_host
  514. where = placement.resolve()
  515. if where.where == placement.TERMINAL or sandbox_host._read_marker():
  516. # A screen already running inside a sandbox is reported (and stoppable) even after the placement
  517. # setting moved: the recorded owner wins over the current policy until it is stopped.
  518. return _sandbox_status(profile, where)
  519. missing: list[str] = missing_binaries() if is_supported_host() else list(REQUIRED_BINARIES)
  520. pid = _launcher_pid()
  521. env = published_env()
  522. running = pid is not None and bool(env.get("DISPLAY"))
  523. mem = resources.memory_info() if is_supported_host() else resources.MemoryInfo(None, None)
  524. return DesktopStatus(
  525. profile=profile or _profile_name(),
  526. supported=is_supported_host(),
  527. installed=not missing,
  528. missing=missing,
  529. running=running,
  530. pid=pid,
  531. display=env.get("DISPLAY"),
  532. socket=str(rfb_socket_path()) if rfb_socket_path() else None,
  533. geometry=geometry(),
  534. install_command=install_command() if missing else None,
  535. browser=_bd_browser.executable() if is_supported_host() else None,
  536. # A running screen is never "blocked": the check guards the allocation, not the session.
  537. blocker=None if running or missing or not is_supported_host() else resources.memory_blocker(mem),
  538. memory_available_mb=mem.available_mb,
  539. memory_limit_mb=mem.limit_mb,
  540. )
  541. def _sandbox_status(profile: Optional[str], where) -> DesktopStatus:
  542. """Status of a sandbox-placed screen. Package presence is only known once the sandbox exists; before
  543. that the pane shows "installed" with the image hint carried in ``install_command`` so Start can explain."""
  544. from tools.bot_desktop import sandbox_host
  545. env = _owned_sandbox_env() or _sandbox_env(create=False)
  546. missing = sandbox_host.missing_binaries(env) if env is not None else []
  547. published = sandbox_host.published_env(env, profile or _profile_name()) if env is not None else {}
  548. running = bool(published.get("DISPLAY"))
  549. # No install_command: the pane's Install button runs apt on the HOST, which is the wrong machine here.
  550. # A sandbox missing the stack is a blocker (shown in place of Start) naming the image that has it.
  551. blocker = None
  552. image_switch = None
  553. if missing:
  554. advice = sandbox_host.plugin_image_advice(where.backend) or (
  555. f"use {sandbox_host.SANDBOX_IMAGE_HINT} as terminal.{where.backend}_image (the default sandbox base "
  556. f"plus the desktop stack)")
  557. blocker = (f"The terminal backend's sandbox image lacks {', '.join(missing)}. {advice[0].upper()}{advice[1:]}, "
  558. f"or set bot_desktop.placement: gateway.")
  559. if where.backend == "docker":
  560. # The usual reason on an upgraded install: the persisted container predates the default
  561. # flip and was kept on purpose. The pane offers the switch instead of a config hint.
  562. from hermes_cli.sandbox_image_switch import pending
  563. sw = pending()
  564. if sw is not None:
  565. image_switch = {"current_image": sw.current_image, "target_image": sw.target_image,
  566. "containers": len(sw.containers)}
  567. blocker = (f"Your sandbox container still runs {sw.current_image}, which has no desktop. "
  568. f"Switch it to {sw.target_image}: files in /root and /workspace stay, packages "
  569. f"installed inside the container are reinstalled on demand.")
  570. return DesktopStatus(
  571. profile=profile or _profile_name(),
  572. supported=True,
  573. installed=True,
  574. missing=missing,
  575. running=running,
  576. pid=None,
  577. display=published.get("DISPLAY"),
  578. socket=None,
  579. geometry=geometry(),
  580. install_command=None,
  581. browser=None,
  582. blocker=blocker,
  583. memory_available_mb=None,
  584. memory_limit_mb=None,
  585. placement=f"{placement.TERMINAL}:{where.backend}",
  586. image_switch=image_switch,
  587. )
  588. def _profile_name() -> str:
  589. try:
  590. from hermes_cli.profiles import get_active_profile_name
  591. return get_active_profile_name() or "default"
  592. except Exception:
  593. return "default"
  594. # The gate lives in ``resources`` so start() and status() cannot disagree about it. Measured in the
  595. # official image: gateway idle 304 MiB, +216 for Xvnc/Xfce, 1073 MiB with one Chromium page. The OOM
  596. # killer picks by score, so on a small instance the casualty is the dashboard or the gateway, not the
  597. # desktop that caused the pressure.
  598. def start(*, wait_seconds: float = 15.0) -> DesktopStatus:
  599. """Start this profile's desktop (idempotent). Blocks until the launcher publishes its env file or
  600. ``wait_seconds`` pass; raises ``RuntimeError`` naming the blocker.
  601. ``bot_desktop.placement`` decides WHERE: inside the configured terminal backend (docker/ssh/a streaming plugin backend;
  602. ``sandbox_host``), or on the gateway host (the rest of this function). A sandbox backend that cannot host
  603. a screen refuses rather than silently falling back to the host beside it.
  604. Two locks: the per-profile ``start.lock``, held from the running-check to the launcher's publish so two
  605. start() calls for one profile spawn one launcher (the loser sees it running), and the host-wide
  606. display-allocation lock, held only until this Xvnc has written ``/tmp/.X<n>-lock`` (a second profile
  607. picking the same number before that would fail and its stale-lock cleanup could remove our socket).
  608. Holding it for the whole Xfce bring-up serialized every profile's start behind one desktop launch."""
  609. where = placement.resolve()
  610. if where.where == placement.REFUSED:
  611. raise RuntimeError(where.reason)
  612. if where.where == placement.TERMINAL:
  613. return _start_in_sandbox(wait_seconds)
  614. if not is_supported_host():
  615. raise RuntimeError("Bot Desktop runs on Linux gateway hosts only")
  616. missing = missing_binaries()
  617. if missing:
  618. # Three dead ends: an operator told "unprivileged, no sudo" while running as root hunts the wrong bug.
  619. need = f"Bot Desktop needs {', '.join(missing)} on the gateway host"
  620. if package_manager() is None:
  621. raise RuntimeError(
  622. f"{need}, and no supported package manager (apt/dnf/pacman) is available to install them. "
  623. "Install TigerVNC (Xvnc) and the Xfce core components with this distro's own tooling.")
  624. if not installable():
  625. raise RuntimeError(
  626. f"{need}, and this host cannot install them: the process is unprivileged and there is no "
  627. "sudo. On the published Docker image the packages have to be baked in, so this needs a "
  628. "newer image rather than an install.")
  629. raise RuntimeError(f"{need}. Install: {install_command()}")
  630. sd = state_dir()
  631. sd.mkdir(parents=True, exist_ok=True)
  632. os.chmod(sd, 0o700)
  633. with _flocked(sd / "start.lock"):
  634. if _launcher_pid() is not None and published_env().get("DISPLAY"):
  635. return status()
  636. from tools.bot_desktop import resources
  637. floor = resources.min_free_mb()
  638. mem = resources.memory_info()
  639. if (blocker := resources.memory_blocker(mem, need=floor)) is not None:
  640. raise RuntimeError(blocker)
  641. if mem.available_mb is not None and mem.available_mb < resources.tight_headroom_mb(floor):
  642. logger.warning(
  643. "Bot Desktop starting with %d MB available; a browser with a few pages open can use most "
  644. "of that.", mem.available_mb)
  645. if _launcher_pid() is None:
  646. _reap_orphaned_server(sd)
  647. _ALLOC_LOCK.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
  648. try:
  649. return _spawn_and_wait(sd, wait_seconds)
  650. except RuntimeError:
  651. # _pick_display reuses the recorded number first: left in place after a failed launch (e.g. Xvnc
  652. # 'server already running' on it), every retry would pick the same number and the profile wedges.
  653. (sd / "display").unlink(missing_ok=True)
  654. raise
  655. def _spawn_and_wait(sd: Path, wait_seconds: float) -> DesktopStatus:
  656. """Caller holds ``start.lock``. Takes ``_ALLOC_LOCK`` itself, from picking the number to Xvnc's claim."""
  657. with contextlib.ExitStack() as alloc:
  658. alloc.enter_context(_flocked(_ALLOC_LOCK))
  659. num = _pick_display()
  660. (sd / "display").write_text(str(num), encoding="utf-8")
  661. env_file = sd / "env"
  662. env_file.unlink(missing_ok=True)
  663. # The agent drives this desktop and its dock opens a terminal, so it starts from the
  664. # scrubbed child env like any other agent child, keeping the user's HOME.
  665. from tools.environments.local import served_profile_child_env
  666. child_env = served_profile_child_env(inherit_credentials=False)
  667. child_env["HOME"] = child_env["HERMES_REAL_HOME"]
  668. for key in ("DISPLAY", "XAUTHORITY", "WAYLAND_DISPLAY", "DBUS_SESSION_BUS_ADDRESS", "SESSION_MANAGER"):
  669. child_env.pop(key, None)
  670. from hermes_cli._launchers import installation_command
  671. from tools.bot_desktop.idle_watch import next_check_in
  672. child_env.update({
  673. "HERMES_BD_PROFILE": _profile_name(),
  674. "HERMES_BD_DISPLAY_NUM": str(num),
  675. "HERMES_BD_SOCKET": str(sd / "rfb.sock"),
  676. "HERMES_BD_XAUTH": str(sd / "Xauthority"),
  677. "HERMES_BD_ENV_FILE": str(env_file),
  678. "HERMES_BD_CONFIG_HOME": str(sd / "xdg"),
  679. "HERMES_BD_GEOMETRY": geometry(),
  680. # The launcher's idle self-stop runs tools.bot_desktop.idle_watch through the install's stable
  681. # command (the ``hermes --run-module`` launcher on a store install, which follows Python pin changes
  682. # and picks the current dependency generation at every run; this interpreter elsewhere), in this
  683. # home, first once the screen could be due (nothing is idle at start). The screen outlives updates,
  684. # so the inherited PYTHONPATH generation is never pinned, and on a store install neither is this
  685. # interpreter.
  686. "HERMES_BD_IDLE_CMD": shlex.join(installation_command(
  687. Path(__file__).resolve().parents[2], module="tools.bot_desktop.idle_watch")),
  688. "HERMES_BD_HOME": str(get_hermes_home()),
  689. "HERMES_BD_IDLE_FIRST_DELAY": str(next_check_in(idle_stop_seconds(), 0.0)),
  690. })
  691. from tools.bot_desktop.browser import dock_exec_line, dock_launch
  692. if (browser := dock_launch()) is not None:
  693. # The bare executable (the launcher checks it exists) and the ready-made, spec-quoted Exec= line.
  694. child_env["HERMES_BD_BROWSER_EXEC"] = browser[0]
  695. child_env["HERMES_BD_BROWSER_EXEC_LINE"] = dock_exec_line(*browser)
  696. # A new file per start: the log is a diagnostic for THIS launch, and nothing rotates it otherwise. Not
  697. # truncated in place: the old screen's idle check (in a session of its own) may still hold the old file
  698. # open, and its late write would land in this one behind a run of NUL bytes.
  699. (sd / "launcher.log").unlink(missing_ok=True)
  700. log = open(sd / "launcher.log", "wb")
  701. # The profile's screen outlives whichever CLI/gateway started it, on purpose: launcher.pid carries
  702. # (pid, create_time) so any later process reports/stops it, and the idle stop (idle_stop_minutes) ends it.
  703. proc = subprocess.Popen( # windows-footgun: ok — Linux-only runtime (is_supported_host)
  704. ["bash", str(_LAUNCHER)], env=child_env, stdin=subprocess.DEVNULL, stdout=log, stderr=log,
  705. start_new_session=True, close_fds=True) # detached-spawn: ok — per-profile screen, outlives its starter
  706. log.close()
  707. born = _create_time(proc.pid)
  708. (sd / "launcher.pid").write_text(f"{proc.pid} {born if born is not None else 0}", encoding="utf-8")
  709. deadline = time.monotonic() + wait_seconds
  710. while time.monotonic() < deadline:
  711. if _x_lock_pid(num) is not None:
  712. alloc.close() # the number is Xvnc's now; other profiles may allocate (idempotent)
  713. if proc.poll() is not None:
  714. tail = (sd / "launcher.log").read_bytes()[-2000:].decode("utf-8", "replace")
  715. raise RuntimeError(f"Bot Desktop launcher exited with {proc.returncode}:\n{tail}")
  716. if env_file.exists() and (sd / "rfb.sock").exists():
  717. logger.info("Bot Desktop for profile %s up on :%s", _profile_name(), num)
  718. touch_activity()
  719. return status()
  720. time.sleep(0.1)
  721. # Giving up must take the launch down: left alone, the launcher publishes DISPLAY and rfb.sock a moment
  722. # later and a screen whose start() reported failure stays up as "running". The launcher is its own
  723. # session leader, so its group is exactly this launch (Xvnc, dbus, Xfce) and nothing else.
  724. _kill_group_then_wait(proc.pid, proc.pid)
  725. proc.wait()
  726. for name in ("launcher.pid", "env", "rfb.sock"):
  727. (sd / name).unlink(missing_ok=True)
  728. raise RuntimeError(f"Bot Desktop did not publish its display within {wait_seconds:.0f}s (see {sd / 'launcher.log'})")
  729. def _sandbox_env(*, create: bool):
  730. """The terminal environment hosting this profile's screen, or None. ``create=False`` for status probes
  731. (a status call must never build a container)."""
  732. return placement.terminal_environment(create=create)
  733. def _start_in_sandbox(wait_seconds: float) -> DesktopStatus:
  734. from tools.bot_desktop import sandbox_host
  735. env = _sandbox_env(create=True)
  736. if env is None:
  737. raise RuntimeError("the terminal backend's sandbox could not be started, so there is nowhere to put the screen")
  738. sd = state_dir()
  739. sd.mkdir(parents=True, exist_ok=True)
  740. os.chmod(sd, 0o700)
  741. with _flocked(sd / "start.lock"):
  742. # The dock's Browser icon runs the sandbox's own Playwright Chromium on the profile the agent's browser
  743. # uses there: the human's browser is the bot's browser, as on the host.
  744. from tools.bot_desktop.browser import dock_exec_line
  745. exe = sandbox_host.chromium_executable(env)
  746. browser = (exe, dock_exec_line(exe, sandbox_host.browser_profile_dir(env), sandbox_bypass=True)) if exe else (None, None)
  747. published = sandbox_host.start(env, _profile_name(), geometry=geometry(), wait_seconds=max(wait_seconds, 20.0),
  748. browser_exec=browser[0], browser_exec_line=browser[1])
  749. (sd / "env").write_text("".join(f"{k}={v}\n" for k, v in published.items()), encoding="utf-8")
  750. touch_activity()
  751. return status()
  752. def _sandbox_published_env() -> dict[str, str]:
  753. """Published env of a sandbox-hosted screen (the caller saw the host-side marker written at start)."""
  754. from tools.bot_desktop import sandbox_host
  755. env = _sandbox_env(create=False)
  756. if env is None:
  757. return {}
  758. return sandbox_host.published_env(env, _profile_name())
  759. def stop(still_due: Optional[Callable[[], bool]] = None) -> bool:
  760. """Stop this profile's desktop; True when a running launcher (or the X server a dead one left behind)
  761. was signalled. ``still_due`` is asked again before anything is killed (under ``start.lock`` for a screen on
  762. this host); False there leaves the screen running."""
  763. from tools.bot_desktop import sandbox_host
  764. if sandbox_host._read_marker() or placement.resolve().where == placement.TERMINAL:
  765. if still_due is not None and not still_due():
  766. return False
  767. # The marker names the sandbox that owns the screen; stop THAT one (re-attaching after a restart),
  768. # whatever the placement setting says now. A marker whose sandbox is gone is simply dropped.
  769. env = _owned_sandbox_env()
  770. stopped = sandbox_host.stop(env, _profile_name()) if env is not None else False
  771. sandbox_host._marker().unlink(missing_ok=True)
  772. for name in ("env", "activity"):
  773. (state_dir() / name).unlink(missing_ok=True)
  774. return stopped
  775. if not is_supported_host():
  776. return False
  777. sd = state_dir()
  778. sd.mkdir(parents=True, exist_ok=True)
  779. with _flocked(sd / "start.lock"):
  780. if still_due is not None and not still_due():
  781. logger.info("Bot Desktop for profile %s changed while the idle stop waited; kept", _profile_name())
  782. return False
  783. return _stop_locked(sd)
  784. def _stop_locked(sd: Path) -> bool:
  785. pid = _launcher_pid()
  786. if pid is None:
  787. reaped = _reap_orphaned_server(sd)
  788. (sd / "env").unlink(missing_ok=True)
  789. return reaped
  790. # The launcher runs in its own session; killing the group takes Xvnc, dbus and Xfce with it.
  791. _kill_group_then_wait(pid, pid, grace=5.0)
  792. for name in ("launcher.pid", "env", "activity", "idle-watch.out"):
  793. (sd / name).unlink(missing_ok=True)
  794. return True