| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752 |
- """Process Registry -- in-memory registry for background processes spawned via
- terminal(background=true): rolling 200KB output buffer, poll/log/wait/kill, JSON
- checkpoint for crash recovery, session-scoped tracking for gateway reset protection.
- Nothing runs on the host unless TERMINAL_ENV=local; other backends run in their sandbox.
- """
- import codecs
- from contextlib import suppress
- import json
- import logging
- import os
- import platform
- import shlex
- import signal
- import stat
- import subprocess
- import tempfile
- import threading
- import time
- import uuid
- from pathlib import Path
- _IS_WINDOWS = platform.system() == "Windows"
- # systemd transient scopes exist only on Linux; gate every scope-path branch on this
- # (not merely "not Windows") so macOS and other POSIX platforms never touch systemd.
- # See #70716.
- _IS_LINUX = platform.system() == "Linux"
- from tools.environments.local import _append_dropped_path_entries, _find_shell, _resolve_safe_cwd, _sanitize_subprocess_env, _terminal_env
- from hermes_cli._subprocess_compat import windows_hide_flags
- from dataclasses import dataclass, field
- from typing import Any, Dict, List, Literal, NamedTuple, Optional
- from hermes_cli.config import get_hermes_home
- from tools.process_registry_notifications import (
- format_process_notification, heartbeat_output_tail, is_child_process_notification, should_surface_notification)
- from tools.process_registry_checkpoint import ProcessCheckpointMixin
- from tools.process_registry_output_log import ProcessOutputLogMixin, open_output_log, record_exit_command, worker_output_log
- from tools.process_registry_ownership import ProcessOwnershipMixin
- from tools.process_registry_termination import ProcessTerminationMixin
- from tools.process_registry_results import load_completed_results, save_completed_result
- from tools.process_registry_env_log import log_delta_command
- from tools.process_registry_wsl import _is_wsl_launcher_command
- logger = logging.getLogger(__name__)
- # Crash-recovery checkpoint (gateway only)
- CHECKPOINT_PATH = get_hermes_home() / "processes.json"
- _CHECKPOINT_PATH_AT_IMPORT = CHECKPOINT_PATH
- def _checkpoint_path() -> Path:
- """Active profile's checkpoint file at call time: the patched ``CHECKPOINT_PATH`` when a test
- changed it, else live profile-scoped HERMES_HOME — the multiplexed gateway serves every
- profile from one process, so the import-time constant would pin every profile's process
- checkpoint to the launch home."""
- return CHECKPOINT_PATH if CHECKPOINT_PATH != _CHECKPOINT_PATH_AT_IMPORT else get_hermes_home() / "processes.json"
- MAX_OUTPUT_CHARS = 200_000 # rolling output buffer
- # Tail of the output a completion notification carries. Right for a build log; a spawner whose
- # output IS the payload (a bot DM's reply) asks for more per process (completion_output_chars).
- COMPLETION_OUTPUT_CHARS = 2000
- FINISHED_TTL_SECONDS = 1800 # keep finished processes 30 minutes
- MAX_PROCESSES = 64 # max tracked processes (LRU pruning)
- # Watch-pattern rate limiting, PER SESSION: one watch-match notification per
- # WATCH_MIN_INTERVAL_SECONDS; a match inside the cooldown is dropped and counts as one
- # strike per window; WATCH_STRIKE_LIMIT consecutive strike windows permanently disable
- # watching and fall back to notify_on_complete semantics.
- WATCH_MIN_INTERVAL_SECONDS = 15
- WATCH_STRIKE_LIMIT = 3
- # Lifetime cap, independent of strikes: a pattern recurring just above the cooldown never
- # strikes yet forces a full-context agent turn each time; watch_patterns is "ONLY for
- # rare one-shot signals", so after this many deliveries fall back to notify_on_complete.
- # MAX_ACTIVE_PROCESS_AGE = 86400 # 24h default — see session_reset.bg_process_max_age_hours (#29177)
- # A process whose pattern recurs at a cadence just above WATCH_MIN_INTERVAL_SECONDS (e.g. a service
- # restarted repeatedly over a day) never trips the consecutive-strike limit, since each match lands in its
- # own clean cooldown window, yet still forces a full-context agent turn every single time (#93513).
- # watch_patterns is documented as "ONLY for rare one-shot mid-process signals", so once a session has
- # delivered this many matches over its whole life we disable it and fall back to notify_on_complete, same as
- # the strike-limit path.
- WATCH_LIFETIME_MAX_HITS = 8
- # Heartbeat: an opt-in periodic "still running, here is the output since last time" event for
- # long bounded jobs (merge trains, full test suites, deploys). Unlike watch patterns it is
- # time-driven, so it is bounded by construction (≤ 3600/HEARTBEAT_MIN_SECONDS events per hour
- # per process) and needs no strike/lifetime breaker. The floor exists so a model cannot turn
- # it into a 5-second poll; the output slice is capped like a completion notice.
- HEARTBEAT_MIN_SECONDS = 60
- HEARTBEAT_TICK_SECONDS = 5
- # Reader final drain after a reconcile saw the direct child exit: the exited child's unread tail is at
- # most one pipe buffer (Linux pipe-max-size default 1 MiB), and the reader publishes within one select
- # interval plus that drain.
- _FINAL_DRAIN_MAX_CHARS = 1 << 20
- _READER_FINAL_DRAIN_WAIT_SECONDS = 1.0
- # Global circuit breaker across all sessions so concurrent siblings can't collectively
- # flood the user even when each is under its own cap.
- WATCH_GLOBAL_MAX_PER_WINDOW = 15
- WATCH_GLOBAL_WINDOW_SECONDS = 10
- WATCH_GLOBAL_COOLDOWN_SECONDS = 30
- # --- systemd cgroup isolation for gateway-spawned local executors ------------------
- # Under a systemd gateway with MemoryMax, local background commands inherit the gateway's
- # cgroup, so a memory-heavy executor can get the ENTIRE gateway killed by systemd-oomd;
- # ``systemd-run --user --scope`` gives the worker its own transient cgroup. Usability is
- # probed and cached for a bounded TTL (binary present but user D-Bus absent in system services/containers).
- # A memory-heavy executor (Codex, tests, Node) can push the whole cgroup past MemoryMax and trigger
- # systemd-oomd to kill the ENTIRE gateway — taking down the messaging control plane and silently losing the
- # active turn. We probe whether ``systemd-run --user --scope`` is actually usable (the binary can
- # exist on the PATH while the user D-Bus session is unavailable — common for system services and
- # containers), and cache the verdict for a bounded TTL. See #70716.
- _SYSTEMD_SCOPE_AVAILABLE: Optional[bool] = None
- _SYSTEMD_SCOPE_PROBE_LOCK = threading.Lock()
- _SYSTEMD_SCOPE_PROBED_AT = 0.0
- # Both verdicts expire: the user bus can vanish after a True (session logout without linger,
- # #110803) and reappear after a False (linger enabled later, #104893).
- _SYSTEMD_SCOPE_PROBE_TTL_SECONDS = 60.0
- # systemd >= 254 expands ``$$``/``${X}`` in a ``--scope`` command line itself unless told not to;
- # older systemd-run rejects the option (and never expanded there), so the probe drops it on rejection.
- _SYSTEMD_RUN_NO_EXPAND = True
- _MIN_WORKER_MEMORY_MAX_BYTES = 64 * 1024 * 1024
- _DEFAULT_WORKER_MEMORY_MAX_BYTES = 1024 * 1024 * 1024
- _WORKER_MEMORY_MAX_CAP_BYTES = 4 * 1024 * 1024 * 1024
- def _worker_memory_max_bytes() -> int:
- """Finite per-worker cgroup limit that can never widen host risk.
- ``TERMINAL_LOCAL_MEMORY_MAX_MB`` is honored only when it *tightens* the safe
- bound (min of the gateway's cgroup-v2 ``memory.max`` and half of physical RAM,
- capped at 4 GiB), so an oversized override cannot exceed the enclosing slice.
- The proposed local-memory-guard environment override is honored when it tightens the safe bound, so this
- isolation composes with PR #57121 instead of inventing a second knob.
- """
- override_bound: Optional[int] = None
- override = os.getenv("TERMINAL_LOCAL_MEMORY_MAX_MB", "").strip()
- if override:
- try:
- parsed = int(override) * 1024 * 1024
- except ValueError:
- parsed = -1
- if parsed >= _MIN_WORKER_MEMORY_MAX_BYTES:
- override_bound = parsed
- else:
- logger.warning(
- "Ignoring invalid TERMINAL_LOCAL_MEMORY_MAX_MB=%r; "
- "expected an integer representing at least %d MiB",
- override, _MIN_WORKER_MEMORY_MAX_BYTES // (1024 * 1024))
- candidates: list[int] = []
- try:
- for line in Path("/proc/self/cgroup").read_text(encoding="utf-8").splitlines():
- if line.startswith("0::"):
- relative = line.partition("::")[2].lstrip("/")
- raw_limit = (
- Path("/sys/fs/cgroup") / relative / "memory.max"
- ).read_text(encoding="utf-8-sig").strip()
- if raw_limit.isdigit():
- cgroup_limit = int(raw_limit)
- if cgroup_limit >= _MIN_WORKER_MEMORY_MAX_BYTES:
- candidates.append(cgroup_limit)
- break
- except (OSError, ValueError):
- pass
- try:
- physical_bytes = int(os.sysconf("SC_PHYS_PAGES")) * int(
- os.sysconf("SC_PAGE_SIZE")
- )
- physical_bound = min(
- _WORKER_MEMORY_MAX_CAP_BYTES,
- max(_MIN_WORKER_MEMORY_MAX_BYTES, physical_bytes // 2),
- )
- candidates.append(physical_bound)
- except (OSError, ValueError, TypeError):
- pass
- safe_bound = min(candidates) if candidates else _DEFAULT_WORKER_MEMORY_MAX_BYTES
- return min(override_bound, safe_bound) if override_bound else safe_bound
- def _systemd_scope_argv(binary: str, unit_name: str, *argv: str) -> list[str]:
- """``systemd-run --user --scope`` argv shared by the probe and real spawns.
- ``--collect`` self-cleans the scope after exit; ``--unit`` names it for systemctl.
- No ``OOMPolicy=``: transient scopes reject it on systemd <253 (#102486).
- ``--expand-environment=no`` keeps the command byte-identical (#132385)."""
- no_expand = ["--expand-environment=no"] if _SYSTEMD_RUN_NO_EXPAND else []
- return [
- binary, "--user", "--scope", "--quiet", *no_expand, "--unit", unit_name, "--collect",
- "--property", "MemoryAccounting=yes",
- "--property", f"MemoryMax={_worker_memory_max_bytes()}",
- "--", *argv,
- ]
- def _default_user_runtime_dir() -> Path:
- """``/run/user/<uid>``; a function so tests can point it at a temp dir with a real socket."""
- return Path(f"/run/user/{os.getuid()}") # windows-footgun: ok — only reached behind the _IS_LINUX gate in systemd_user_bus_env
- def _secure_user_runtime_dir(path: Path) -> bool:
- """Accept only an absolute, owned, non-writable real directory."""
- try:
- metadata = path.lstat()
- return (
- path.is_absolute()
- and stat.S_ISDIR(metadata.st_mode)
- and metadata.st_uid == os.getuid() # windows-footgun: ok — only reached behind the _IS_LINUX gate in systemd_user_bus_env
- and metadata.st_mode & 0o022 == 0
- )
- except OSError:
- return False
- def systemd_user_bus_env(base_env: Optional[dict[str, str]] = None) -> dict[str, str]:
- """Build an environment that can reach this user's lingering systemd manager.
- System-level gateway units run as an unprivileged ``User=`` but normally do
- not inherit login-session variables. When the conventional runtime
- directory is owned by this uid and its bus exists, derive the two standard
- variables. Derived fresh on every call rather than adopted once at boot:
- linger may be enabled after the gateway started (existing installs), so
- the bus can appear later and the probe's failure TTL must be able to
- recover (#104893).
- The returned copy is passed explicitly to the probe and every scoped spawn;
- ``os.environ`` is left unchanged.
- """
- env = dict(os.environ if base_env is None else base_env)
- if not _IS_LINUX:
- return env
- configured = env.get("XDG_RUNTIME_DIR")
- if configured and _secure_user_runtime_dir(Path(configured)):
- runtime_dir = Path(configured)
- else:
- runtime_dir = _default_user_runtime_dir()
- if not _secure_user_runtime_dir(runtime_dir):
- return env
- bus_path = runtime_dir / "bus"
- try:
- bus_metadata = bus_path.lstat()
- except OSError:
- return env
- if not stat.S_ISSOCK(bus_metadata.st_mode) or bus_metadata.st_uid != os.getuid(): # windows-footgun: ok — behind the _IS_LINUX gate above
- return env
- env["XDG_RUNTIME_DIR"] = str(runtime_dir)
- env["DBUS_SESSION_BUS_ADDRESS"] = f"unix:path={bus_path}"
- return env
- def _systemd_scope_cached() -> Optional[bool]:
- """Cached probe verdict, or None when a (re)probe is due."""
- if _SYSTEMD_SCOPE_AVAILABLE is None:
- return None
- stale = time.monotonic() - _SYSTEMD_SCOPE_PROBED_AT >= _SYSTEMD_SCOPE_PROBE_TTL_SECONDS
- return None if stale else _SYSTEMD_SCOPE_AVAILABLE
- def _systemd_run_user_scope_available() -> bool:
- """True if ``systemd-run --user --scope`` can create a cgroup.
- ``shutil.which`` alone is insufficient: system services and containers may lack
- the user D-Bus bus even with the binary on PATH (every spawn would fail with
- ``Failed to connect to user bus``), so a cheap probe is run and cached.
- Use ``/bin/sh -c 'exit 0'``: NixOS provides ``/bin/sh`` but not ``/bin/true``
- (#105365), regardless of the gateway service's PATH."""
- global _SYSTEMD_SCOPE_AVAILABLE, _SYSTEMD_SCOPE_PROBED_AT, _SYSTEMD_RUN_NO_EXPAND
- verdict = _systemd_scope_cached()
- if verdict is not None:
- return verdict
- # Double-checked locking: a concurrent first-use spawn must not observe a temporary
- # False mid-probe, or it would launch back inside the gateway cgroup.
- with _SYSTEMD_SCOPE_PROBE_LOCK:
- verdict = _systemd_scope_cached()
- if verdict is not None:
- return verdict
- available = False
- if _IS_LINUX:
- try:
- import shutil
- binary = shutil.which("systemd-run")
- if binary:
- # Unique unit avoids collisions; the timeout bounds D-Bus.
- probe_unit = f"hermes-probe-scope-{os.getpid()}-{uuid.uuid4().hex[:8]}"
- for _attempt in range(2):
- result = subprocess.run(
- _systemd_scope_argv(binary, probe_unit, "/bin/sh", "-c", "exit 0"),
- stdin=subprocess.DEVNULL, capture_output=True,
- timeout=3,
- env=systemd_user_bus_env(),
- check=False,
- )
- if not (result.returncode and _SYSTEMD_RUN_NO_EXPAND
- and b"expand-environment" in (result.stderr or b"")):
- break
- # systemd < 254 rejects the option: drop it and probe again.
- _SYSTEMD_RUN_NO_EXPAND = False
- available = result.returncode == 0
- if not available:
- logger.debug(
- "systemd-run --user --scope probe failed (rc=%s): %s",
- result.returncode, (result.stderr or b"").decode("utf-8", "replace").strip(),
- )
- except Exception as exc:
- logger.debug("systemd-run --user --scope probe error: %s", exc)
- _SYSTEMD_SCOPE_AVAILABLE = available
- _SYSTEMD_SCOPE_PROBED_AT = time.monotonic()
- return available
- def _is_supervised_gateway_process() -> bool:
- """Whether this process is the live, supervised Hermes gateway itself.
- Supervisor markers and ``_HERMES_GATEWAY`` are inherited by every descendant (and
- importing ``gateway.run`` sets the latter), so also require ownership of the live
- gateway PID file — scopes are for the gateway, not terminal children or CLIs.
- Reads the launch marker (``HERMES_SUPERVISED_CHILD`` included), not the restart-route
- probe: a Windows Scheduled-Task gateway sets only that marker, and the self-kill guards
- gated here must protect it too (#113667)."""
- if os.environ.get("_HERMES_GATEWAY") != "1":
- return False
- try:
- from gateway.restart import is_supervised_gateway_launch
- from gateway.status import get_running_pid
- return is_supervised_gateway_launch() and get_running_pid(cleanup_stale=False) == os.getpid()
- except Exception as exc:
- logger.debug("Could not verify supervised gateway process identity: %s", exc)
- return False
- def _build_systemd_scope_argv(shell_argv: list[str], unit_suffix: str) -> list[str]:
- """Wrap *shell_argv* in a ``systemd-run --user --scope`` invocation with its own
- memory accounting, so an OOM in the worker cannot kill the gateway cgroup.
- ``--collect`` makes the transient scope self-clean after exit; ``--unit`` gives it a recognisable name
- for ``systemctl --user status`` / journalctl. See #70716.
- """
- import shutil
- binary = shutil.which("systemd-run")
- if binary is None:
- # Caller should have probed availability; never pass None into Popen anyway.
- return shell_argv
- return _systemd_scope_argv(binary, f"hermes-worker-{unit_suffix}", *shell_argv)
- _scope_degraded_warned = False
- class RestartSafeScopeUnavailable(RuntimeError):
- """A ``require_restart_safe_scope=True`` child could not get its transient scope.
- Host-level (no user D-Bus / no ``systemd-run``), never a property of the
- child being launched: callers that keep per-job retry budgets must not
- charge it (a kanban card parked ``blocked`` for an unreachable bus, #114720).
- """
- def _warn_scope_degraded_once(detail: str, *, consequence: str) -> None:
- """Warn once per process: the condition is host-level and the probe verdict
- is cached, so this would otherwise fire on every cron dispatch."""
- global _scope_degraded_warned
- if _scope_degraded_warned:
- return
- _scope_degraded_warned = True
- logger.warning("%s; %s", detail, consequence)
- _CRON_DEGRADED_CONSEQUENCE = (
- "cron children are dispatched as direct external subprocesses without restart-safe "
- "cgroup isolation (killed if the gateway restarts mid-job). Set "
- "cron.require_restart_safe_scope=true in config.yaml to fail closed instead."
- )
- _UNIT_DEGRADED_CONSEQUENCE = (
- "workers are spawned unmanaged inside this systemd unit's cgroup and will be KILLED when the "
- "unit exits (Type=oneshot dispatch timers lose every worker within a second). Give the unit's "
- "user a session bus (`loginctl enable-linger <user>`) so workers get their own scope, or set "
- "KillMode=process on the unit."
- )
- class GatewayChildDispatch(NamedTuple):
- """How a managed-gateway child is launched.
- ``in_process``: not a managed systemd gateway, ``argv is command``, the caller
- keeps its in-process path. ``scoped``: ``argv`` is the systemd-run wrapper.
- ``degraded``: no user scope could be created; ``argv`` is the direct command but
- the caller MUST still launch it as an external subprocess — the distinct mode
- exists so this case can never collapse into ``in_process`` and recreate the
- restart interruption #101940 closed.
- """
- mode: Literal["in_process", "scoped", "degraded"]
- argv: list[str]
- def scoped_spawn_lost_user_bus(spawn_env: dict[str, str]) -> bool:
- """After a ``systemd-run --user --scope`` wrapper exits before its child could start: True
- when the user bus is gone (:func:`systemd_user_bus_env` derives nothing), in which case the
- cached True verdict is replaced so the next dispatch re-probes and degrades instead of
- consuming another occurrence on the same dead wrapper (#110803).
- *spawn_env* is the environment the wrapper was launched with: re-deriving from it (minus the
- bus address it carried) honours a configured ``XDG_RUNTIME_DIR`` exactly as the spawn did, so
- an unrelated wrapper exit on a host whose bus lives outside ``/run/user/<uid>`` is not
- misread as a lost bus."""
- global _SYSTEMD_SCOPE_AVAILABLE, _SYSTEMD_SCOPE_PROBED_AT
- base_env = dict(spawn_env)
- base_env.pop("DBUS_SESSION_BUS_ADDRESS", None)
- if "DBUS_SESSION_BUS_ADDRESS" in systemd_user_bus_env(base_env):
- return False
- with _SYSTEMD_SCOPE_PROBE_LOCK:
- _SYSTEMD_SCOPE_AVAILABLE = False
- _SYSTEMD_SCOPE_PROBED_AT = time.monotonic()
- return True
- def restart_safe_gateway_child_argv(
- command: list[str], *, unit_suffix: str, require_restart_safe_scope: bool,
- outlives_parent: bool = False,
- ) -> GatewayChildDispatch:
- """Place a managed-systemd gateway child outside the gateway cgroup.
- A systemd-supervised gateway restart kills every process in the service
- cgroup, so children that must survive it run in a transient user scope.
- Hosts with no user systemd session (containers, LXCs without linger) cannot
- create one; hard-failing there is a silent cron outage, so callers state the
- policy: ``require_restart_safe_scope=True`` raises
- :class:`RestartSafeScopeUnavailable` (kanban's long-lived workers), ``False``
- degrades to a direct external subprocess with a once-per-process warning
- (cron, behind ``cron.require_restart_safe_scope``).
- ``outlives_parent=True`` (fire-and-forget kanban workers): any *other*
- systemd unit — a ``Type=oneshot`` dispatch timer, an operator's sequencer
- service — tears its cgroup down when it exits, so the child is scope-wrapped
- there too (#113612). Whether that unit actually kills its children
- (``KillMode``, lifetime) is not knowable here, so without a user bus it
- degrades with a loud warning instead of refusing: a long-lived
- ``Type=simple`` sequencer without linger keeps working. A cron job blocks its
- caller until it finishes and never needs this.
- """
- if not _IS_LINUX:
- return GatewayChildDispatch("in_process", command)
- if not os.environ.get("INVOCATION_ID"):
- return GatewayChildDispatch("in_process", command)
- supervised_gateway = _is_supervised_gateway_process()
- if not supervised_gateway and not outlives_parent:
- return GatewayChildDispatch("in_process", command)
- def _degrade(detail: str) -> GatewayChildDispatch:
- if supervised_gateway:
- if require_restart_safe_scope:
- # Stored as the cron execution's error and shown on the job row: name the remedy.
- raise RestartSafeScopeUnavailable(
- f"cannot create restart-safe systemd scope for gateway child: {detail}"
- )
- _warn_scope_degraded_once(f"managed gateway: {detail}", consequence=_CRON_DEGRADED_CONSEQUENCE)
- else:
- _warn_scope_degraded_once(f"systemd unit dispatch: {detail}", consequence=_UNIT_DEGRADED_CONSEQUENCE)
- return GatewayChildDispatch("degraded", command)
- if not _systemd_run_user_scope_available():
- return _degrade(
- "systemd-run --user --scope is unavailable (usually no reachable user D-Bus session at "
- f"/run/user/{os.getuid()}/bus). On a system-level service install, run " # windows-footgun: ok — behind the _IS_LINUX return above
- "`sudo loginctl enable-linger <gateway-user>` and restart the gateway."
- )
- scoped = _build_systemd_scope_argv(command, unit_suffix=unit_suffix)
- if scoped == command:
- return _degrade("systemd-run disappeared after the availability probe")
- return GatewayChildDispatch("scoped", scoped)
- def _stop_systemd_unit(unit_name: str) -> bool:
- """Stop a transient systemd user scope by unit name.
- Reaps the *entire* cgroup — catching double-forked descendants reparented to init
- inside the scope that survive a plain PID signal (SIGTERM all, SIGKILL after
- ``TimeoutStopSec``). True if stopped or already gone; False if ``systemctl`` is
- unavailable or the stop failed.
- See #70716.
- """
- import shutil
- binary = shutil.which("systemctl")
- if binary is None:
- return False
- try:
- result = subprocess.run(
- [binary, "--user", "stop", unit_name],
- capture_output=True,
- timeout=15,
- stdin=subprocess.DEVNULL,
- env=systemd_user_bus_env(),
- check=False,
- )
- if result.returncode != 0:
- stderr = (result.stderr or b"").decode(errors="replace").strip()
- if any(marker in stderr.lower() for marker in ("not loaded", "not found", "does not exist")):
- return True
- logger.debug("systemctl --user stop %s exited %d: %s", unit_name, result.returncode, stderr)
- return False
- return True
- except Exception as exc:
- logger.debug("systemctl --user stop %s failed: %s", unit_name, exc)
- return False
- def format_uptime_short(seconds: int) -> str:
- s = max(0, int(seconds))
- if s < 60:
- return f"{s}s"
- mins, secs = divmod(s, 60)
- if mins < 60:
- return f"{mins}m {secs}s"
- hours, mins = divmod(mins, 60)
- return f"{hours}h {mins}m"
- def _not_found(session_id: str) -> dict:
- return {"status": "not_found", "error": f"No process with ID {session_id}"}
- def _output_tail(session: "ProcessSession", n: int) -> str:
- """Last *n* chars of the session output with ANSI sequences stripped."""
- from tools.ansi_strip import strip_ansi
- return strip_ansi(session.output_buffer[-n:])
- def _completion_output(session: "ProcessSession") -> dict:
- """``output`` sized by the session's ``completion_output_chars`` plus ``output_cut`` when trimmed.
- Shared by the completion notification AND the wait/poll/kill snapshots: a bot in an api_server
- or one-shot session cannot receive notifications and polls instead, so the polled result must
- carry the same whole reply and the same cut marker (#115334)."""
- limit = session.completion_output_chars or COMPLETION_OUTPUT_CHARS
- cut = len(session.output_buffer) - limit
- return {"output": _output_tail(session, limit), **({"output_cut": cut} if cut > 0 else {})}
- @dataclass
- class ProcessSession:
- """A tracked background process with output buffering."""
- id: str # "proc_xxxxxxxxxxxx"
- command: str
- task_id: str = "" # Task/sandbox isolation key (CONTAINER key,
- # may be collapsed by _resolve_container_task_id)
- owner_task_id: str = "" # RAW spawning task id ("sa-..."); ownership
- # checks must use this, not task_id
- session_key: str = "" # Gateway session key (reset protection)
- pid: Optional[int] = None
- process: Optional[subprocess.Popen] = None # Popen handle (local only)
- env_ref: Any = None # Environment object (sandbox spawns)
- cwd: Optional[str] = None
- started_at: float = 0.0 # time.time() of spawn
- host_start_time: Optional[int] = None # kernel start ticks (/proc/<pid>/stat f22) — PID-reuse guard
- exited: bool = False
- exited_at: float = 0.0 # time.time() of the FIRST move to finished (0 = unknown)
- exit_code: Optional[int] = None # None while running
- completion_reason: str = "exited" # exited|killed|lost|failed_start|already_exited
- termination_source: str = "" # process.kill|kill_all|backend_lost|failed_start
- output_buffer: str = "" # Rolling tail (last max_output_chars)
- max_output_chars: int = MAX_OUTPUT_CHARS
- detached: bool = False # Recovered from checkpoint (no pipe)
- pid_scope: str = "host" # "host" for local/PTY PIDs, "sandbox" for env-local PIDs
- systemd_unit: str = "" # transient scope unit name when spawned under systemd-run
- wsl_chain: bool = False # spawned via wsl[.exe]: the host PID is the short-lived
- # launcher; Linux-side workers outlive it (#120546)
- handoff_note: str = "" # why a subagent handed this process to its parent (rides the notice)
- persist_on_release: bool = False # opt out of agent-lifecycle cleanup (release()/turn-abandon kill
- # sweeps), per terminal(background=true, persist_on_release=true) (#41225)
- output_log: str = "" # file the process writes to (worker spawns; no pipe to lose)
- pty_fallback: str = "" # why a pty=true local spawn runs in pipe mode instead
- # Watcher/notification routing (persisted for crash recovery)
- watcher_platform: str = ""
- watcher_chat_id: str = ""
- watcher_user_id: str = ""
- watcher_user_name: str = ""
- watcher_thread_id: str = ""
- watcher_message_id: str = "" # Triggering message id — reply anchor for topic routing
- watcher_interval: int = 0 # 0 = no watcher configured
- # Session-db id of the spawning conversation; lets the gateway drop completions whose
- # session was closed at a user boundary (/new) instead of injecting into the NEW one.
- parent_session_id: str = ""
- notify_on_complete: bool = False # Queue agent notification on exit
- completion_output_chars: int = 0 # Output chars the completion carries; 0 = COMPLETION_OUTPUT_CHARS
- watch_patterns: list[str] = field(default_factory=list)
- heartbeat_seconds: int = 0 # 0 = off; else a "heartbeat" event every N s while running
- total_output_chars: int = 0 # Chars ever ingested (the buffer is a rolling tail)
- _heartbeat_last: float = field(default=0.0, repr=False) # time of the last heartbeat (or spawn)
- _heartbeat_total_at_last: int = field(default=0, repr=False) # total_output_chars at that moment
- _heartbeat_seq: int = field(default=0, repr=False)
- _watch_hits: int = field(default=0, repr=False) # total matches delivered
- _watch_suppressed: int = field(default=0, repr=False) # matches dropped by rate limit
- _watch_disabled: bool = field(default=False, repr=False) # permanently killed after strike limit
- # Rate-limit window state (see WATCH_*). A strike is a WINDOW with drops, not a drop.
- _watch_cooldown_until: float = field(default=0.0, repr=False)
- _watch_strike_candidate: bool = field(default=False, repr=False)
- _watch_consecutive_strikes: int = field(default=0, repr=False)
- _completion_event: threading.Event = field(default_factory=threading.Event, repr=False)
- # Set by the _move_to_finished call that moved the session out of _running; only that call
- # publishes the completion, so a duplicate finisher must not release waiters early.
- _finish_claimed: bool = field(default=False, repr=False)
- # The last stop of ``systemd_unit`` failed, so its descendants may still be alive; bulk
- # cleanup retries the stop even after the session finished.
- _scope_stop_pending: bool = field(default=False, repr=False)
- _lock: threading.Lock = field(default_factory=threading.Lock)
- _reader_thread: Optional[threading.Thread] = field(default=None, repr=False)
- _reader_finish_requested: threading.Event = field(default_factory=threading.Event, repr=False)
- _reader_selectable: bool = field(default=False, repr=False)
- _pty: Any = field(default=None, repr=False) # ptyprocess handle (use_pty=True)
- _owned_pgid: Optional[int] = field(default=None, repr=False) # see ProcessOwnershipMixin
- def __post_init__(self):
- # A session built without an explicit owner is owned by its own task, so ownership checks compare
- # ``owner_task_id`` alone instead of repeating an ``or task_id`` fallback at every call site.
- if not self.owner_task_id:
- self.owner_task_id = self.task_id
- def append_output(self, text: str) -> None:
- """Append to the rolling output buffer under the session lock, keeping the tail."""
- with self._lock:
- self._append_locked(text)
- def append_output_if_running(self, text: str) -> bool:
- """Append unless the session has exited. Decided under the lock a kill holds while it
- snapshots the output and sets ``exited``, so a chunk is either in the kill's receipt or
- dropped, never added after it."""
- with self._lock:
- if self.exited:
- return False
- self._append_locked(text)
- return True
- def _append_locked(self, text: str) -> None:
- self.output_buffer += text
- self.total_output_chars += len(text)
- if len(self.output_buffer) > self.max_output_chars:
- self.output_buffer = self.output_buffer[-self.max_output_chars:]
- def mark_exited(self, exit_code, reason: str = "exited", source: str = "") -> None:
- """Record an exit. A kill that raced the observer already recorded its own
- exit_code/reason; never overwrite it."""
- self.exited = True
- if self.completion_reason != "killed":
- self.exit_code = exit_code
- self.completion_reason = reason
- if source:
- self.termination_source = source
- # Watcher routing fields, in event-dict key order (``watcher_<key>`` on the session).
- _WATCHER_ROUTE_KEYS = ("platform", "chat_id", "user_id", "user_name", "thread_id", "message_id")
- # Session fields persisted verbatim in the crash-recovery checkpoint (plus
- # ``session_id``; ``command`` is redacted and ``owner_task_id`` defaulted on write).
- _CHECKPOINT_FIELDS = (
- "command", "pid", "pid_scope", "host_start_time", "systemd_unit", "wsl_chain", "cwd",
- "started_at", "task_id", "owner_task_id", "session_key",
- *(f"watcher_{k}" for k in _WATCHER_ROUTE_KEYS), "watcher_interval",
- "parent_session_id", "notify_on_complete", "completion_output_chars", "watch_patterns",
- "heartbeat_seconds", "persist_on_release", "output_log")
- _CHECKPOINT_DEFAULTS = {
- f.name: ([] if f.name == "watch_patterns" else f.default)
- for f in ProcessSession.__dataclass_fields__.values()
- if f.name in _CHECKPOINT_FIELDS
- }
- _WSL_CHAIN_NOTE = (
- "Spawned via a wsl[.exe] launcher: the recorded host PID is the short-lived "
- "launcher, not the Linux-side workers. Inspect them with `wsl -e ps` / "
- "`wsl --list --running` from the host."
- )
- class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOutputLogMixin, ProcessOwnershipMixin):
- """In-memory registry of running and finished background processes.
- Thread-safe: accessed from executor threads (terminal_tool, process handlers),
- the gateway asyncio loop (watchers, reset checks) and the cleanup thread."""
- _SHELL_NOISE_SUBSTRINGS = (
- "no job control in this shell", "cannot set terminal process group",
- "tcsetattr: Inappropriate ioctl for device")
- # Class default so registries built via __new__ (tests) still restore on first drain.
- _completions_restored = False
- def __init__(self):
- self._running: dict[str, ProcessSession] = {}
- self._finished: dict[str, ProcessSession] = {}
- self._lock = threading.Lock()
- # Side-channel for check_interval watchers (gateway reads after agent run)
- self.pending_watchers: list[dict[str, Any]] = []
- # Unified queue for all background events (distinguished by "type"); the CLI
- # process_loop and the gateway drain it after each agent turn to trigger new turns.
- import queue as _queue_mod
- self.completion_queue: _queue_mod.Queue = _queue_mod.Queue()
- # Durable delegation completions are NOT rehydrated here: this constructor runs at
- # import time in every process that touches the tools graph (cron clients, `hermes
- # sessions list`, workers), and restoring means writing the canonical ledger. The
- # owner replays them explicitly (GatewayRunner._start_recover_previous_run) and the
- # first consumer through restore_completions() (#123265). Importing the module is
- # side-effect free and keeps its import-order contract for later completion writers.
- import tools.async_delegation
- # Completions the agent already consumed via wait()/read_log() (output in
- # hand): drain loops AND gateway/tui watchers skip them.
- self._completion_consumed: set = set()
- # Sessions merely *observed* exited via poll(). poll() is read-only and must NOT
- # mark consumed (a status check would suppress the watcher's autonomous delivery
- # turn), but the CLI has the poll result inline in the same turn, so
- # drain_notifications() skips these to avoid a duplicate [SYSTEM: ...];
- # gateway/tui watchers deliberately ignore this set.
- # See #8228.
- self._poll_observed: set = set()
- # Global watch-match circuit breaker across all sessions.
- self._global_watch_lock = threading.Lock()
- self._global_watch_window_start = self._global_watch_tripped_until = 0.0
- self._global_watch_window_hits = self._global_watch_suppressed_during_trip = 0
- # Driver-installed sinks (desktop gateway): on_output(session, chunk) streams
- # live output from reader threads; on_close(session_or_none, process_id) drops
- # a read-only terminal tab without killing the process.
- self.on_output = None
- self.on_close = None
- self._heartbeat_thread: Optional[threading.Thread] = None
- # ── heartbeat ───────────────────────────────────────────────────────────
- def arm_heartbeat(self, session: ProcessSession, seconds: int) -> int:
- """Enable periodic heartbeat events for ``session``; returns the effective interval."""
- seconds = max(int(seconds), HEARTBEAT_MIN_SECONDS)
- session.heartbeat_seconds = seconds
- session._heartbeat_last = time.time()
- # The output baseline stays at spawn (field default 0), never here: the spawn call
- # arms the heartbeat only after its bookkeeping, and a fast-starting process has
- # already written its first lines by then. Those lines belong to the first heartbeat.
- self._ensure_heartbeat_thread()
- return seconds
- def _ensure_heartbeat_thread(self) -> None:
- with self._lock:
- if self._heartbeat_thread is not None and self._heartbeat_thread.is_alive():
- return
- self._heartbeat_thread = threading.Thread(
- target=self._heartbeat_loop, name="process-heartbeat", daemon=True)
- self._heartbeat_thread.start()
- def _heartbeat_loop(self) -> None:
- """One daemon thread for every heartbeat session: reader threads block on the pipe and
- cannot keep time, and a per-process timer would leak one thread per job."""
- while True:
- time.sleep(HEARTBEAT_TICK_SECONDS)
- now = time.time()
- with self._lock:
- due = [s for s in self._running.values()
- if s.heartbeat_seconds > 0 and not s.exited
- and now - s._heartbeat_last >= s.heartbeat_seconds]
- for session in due:
- self._emit_heartbeat(session, now)
- def _emit_heartbeat(self, session: ProcessSession, now: float) -> None:
- """Queue a heartbeat carrying the output since the last one. A tick with nothing new is
- skipped outright: every queued event costs the owner a full model turn, and "still running,
- no output" is already visible on the process surfaces (status stack, /agents dock)."""
- session._heartbeat_last = now
- with session._lock:
- delta = session.total_output_chars - session._heartbeat_total_at_last
- output = session.output_buffer[-delta:] if delta > 0 else ""
- session._heartbeat_total_at_last = session.total_output_chars
- if not output:
- return
- output, omitted = heartbeat_output_tail(output)
- session._heartbeat_seq += 1
- notification = {
- **self._watch_event_base(session),
- "type": "heartbeat",
- "seq": session._heartbeat_seq,
- "interval": session.heartbeat_seconds,
- "elapsed": int(now - session.started_at) if session.started_at else 0,
- "output": output,
- "omitted": omitted,
- "started_at": session.started_at,
- }
- _redact_process_result(notification)
- self.completion_queue.put(notification)
- @staticmethod
- def _clean_shell_noise(text: str) -> str:
- """Strip shell startup warnings from the beginning of output."""
- lines = text.split("\n")
- while lines and any(noise in lines[0] for noise in ProcessRegistry._SHELL_NOISE_SUBSTRINGS):
- lines.pop(0)
- return "\n".join(lines)
- def _emit_output(self, session: ProcessSession, chunk: str) -> None:
- """Forward a chunk to the live-output sink; called from reader threads, never raises."""
- sink = self.on_output
- if sink is None or not chunk:
- return
- with suppress(Exception):
- sink(session, chunk)
- def _check_watch_patterns(self, session: ProcessSession, new_text: str) -> None:
- """Scan a freshly-read chunk for watch patterns and queue notifications.
- Per-session rate limiting (see WATCH_* constants): one match per cooldown
- window, a match inside the window is one strike, WATCH_STRIKE_LIMIT consecutive
- strikes or WATCH_LIFETIME_MAX_HITS total deliveries disable watching and
- promote the session to notify_on_complete."""
- if not session.watch_patterns or session._watch_disabled:
- return
- # Late chunks after the reader declared exit are post-exit noise; dropping them
- # avoids stale notifications minutes after the process ended.
- if session.exited:
- return
- hits = [ # (first matching pattern, line) — one match per line
- (next(p for p in session.watch_patterns if p in line), line.rstrip())
- for line in new_text.splitlines() if any(p in line for p in session.watch_patterns)]
- if not hits:
- return
- matched_pattern = hits[0][0]
- matched_lines = [line for _, line in hits]
- now = time.time()
- with session._lock:
- if session._watch_cooldown_until and now < session._watch_cooldown_until:
- # Inside the cooldown: drop, count one strike per window, disable +
- # promote once the strike limit is hit.
- session._watch_suppressed += len(matched_lines)
- if session._watch_strike_candidate:
- return
- session._watch_strike_candidate = True
- session._watch_consecutive_strikes += 1
- if session._watch_consecutive_strikes < WATCH_STRIKE_LIMIT:
- return
- session._watch_disabled = True
- # Promote so the agent still gets exactly one notification on exit,
- # plus exactly one summary so it sees why things went quiet.
- session.notify_on_complete = True
- self._emit_watch_disabled(
- session, session._watch_suppressed,
- f"{WATCH_STRIKE_LIMIT} consecutive rate-limit windows triggered "
- f"(min spacing {WATCH_MIN_INTERVAL_SECONDS}s). ")
- return
- # Cooldown expired. A prior window with no drops resets the
- # consecutive-strike counter (healthy cadence again).
- if session._watch_cooldown_until and not session._watch_strike_candidate:
- session._watch_consecutive_strikes = 0
- session._watch_strike_candidate = False
- # Emit and start a new cooldown window.
- session._watch_cooldown_until = now + WATCH_MIN_INTERVAL_SECONDS
- session._watch_hits += 1
- suppressed = session._watch_suppressed
- session._watch_suppressed = 0
- # Lifetime cap: this match is still delivered, but no further ones.
- lifetime_exhausted = session._watch_hits >= WATCH_LIFETIME_MAX_HITS
- if lifetime_exhausted:
- session._watch_disabled = True
- session.notify_on_complete = True
- output = "\n".join(matched_lines[:20])
- if len(output) > 2000:
- output = output[:2000] + "\n...(truncated)"
- if self._global_watch_admit(now):
- notification = {
- **self._watch_event_base(session),
- "type": "watch_match",
- "pattern": matched_pattern,
- "output": output,
- "suppressed": suppressed,
- }
- _redact_process_result(notification)
- self.completion_queue.put(notification)
- # Even when the breaker drops the final match, still explain the silence.
- if lifetime_exhausted:
- self._emit_watch_disabled(
- session, 0, f"reached the lifetime cap of {WATCH_LIFETIME_MAX_HITS} delivered matches. ",
- )
- def _emit_watch_disabled(self, session: ProcessSession, suppressed: int, why: str) -> None:
- """Queue the one-shot watch_disabled summary (strike-limit or lifetime-cap path)."""
- self.completion_queue.put({
- **self._watch_event_base(session),
- "type": "watch_disabled",
- "suppressed": suppressed,
- "message": (
- f"Watch patterns disabled for process {session.id} — {why}"
- f"Falling back to notify_on_complete semantics; you'll get "
- f"exactly one notification when the process exits."),
- })
- @staticmethod
- def _watch_event_base(session: ProcessSession) -> dict:
- """Session identity + watcher routing fields shared by every watch event."""
- return {
- "event_id": uuid.uuid4().hex,
- "started_at": session.started_at,
- "parent_session_id": session.parent_session_id,
- "session_id": session.id,
- "session_key": session.session_key,
- "task_id": session.task_id,
- "owner_task_id": session.owner_task_id,
- "command": session.command,
- **{key: getattr(session, f"watcher_{key}") for key in _WATCHER_ROUTE_KEYS},
- }
- @staticmethod
- def _global_watch_event(type_: str, message: str, **extra) -> dict:
- """Unaddressed (all-sessions) watch breaker event."""
- return {
- "session_id": "", "session_key": "", "command": "", "type": type_, **extra,
- "message": message,
- "platform": "", "chat_id": "", "user_id": "", "user_name": "", "thread_id": "",
- }
- def _global_watch_admit(self, now: float) -> bool:
- """True if this watch_match may pass the global breaker.
- In cooldown: drop and count. Otherwise slide the rolling window; exceeding
- the cap trips the breaker for WATCH_GLOBAL_COOLDOWN_SECONDS with ONE
- "tripped" summary, and the cooldown's end emits ONE "released" summary."""
- events = [] # summary events, queued outside the lock
- with self._global_watch_lock:
- # Handle cooldown expiry first so we can emit the release summary.
- if self._global_watch_tripped_until and now >= self._global_watch_tripped_until:
- suppressed = self._global_watch_suppressed_during_trip
- self._global_watch_tripped_until = 0.0
- self._global_watch_suppressed_during_trip = 0
- self._global_watch_window_start, self._global_watch_window_hits = now, 0
- if suppressed > 0:
- events.append(self._global_watch_event(
- "watch_overflow_released",
- f"Watch-pattern notifications resumed. "
- f"{suppressed} match event(s) were suppressed during the flood.",
- suppressed=suppressed))
- if self._global_watch_tripped_until and now < self._global_watch_tripped_until:
- # Still in cooldown — drop and count.
- self._global_watch_suppressed_during_trip += 1
- admit = False
- else:
- if now - self._global_watch_window_start >= WATCH_GLOBAL_WINDOW_SECONDS:
- self._global_watch_window_start, self._global_watch_window_hits = now, 0
- admit = self._global_watch_window_hits < WATCH_GLOBAL_MAX_PER_WINDOW
- if admit:
- self._global_watch_window_hits += 1
- else:
- self._global_watch_tripped_until = now + WATCH_GLOBAL_COOLDOWN_SECONDS
- self._global_watch_suppressed_during_trip += 1
- events.append(self._global_watch_event(
- "watch_overflow_tripped",
- f"Watch-pattern overflow: >{WATCH_GLOBAL_MAX_PER_WINDOW} "
- f"notifications in {WATCH_GLOBAL_WINDOW_SECONDS}s across all processes. "
- f"Suppressing further watch_match events for "
- f"{WATCH_GLOBAL_COOLDOWN_SECONDS}s."))
- for msg in events:
- self.completion_queue.put(msg)
- return admit
- @staticmethod
- def _is_host_pid_alive(pid: Optional[int]) -> bool:
- """Best-effort liveness check for host-visible PIDs."""
- if not pid:
- return False
- # ``os.kill(pid, 0)`` is NOT a no-op on Windows (bpo-14484) — use the
- # cross-platform existence check.
- from gateway.status import _pid_exists
- return _pid_exists(pid)
- @staticmethod
- def _safe_host_start_time(pid: Optional[int]) -> Optional[int]:
- """Kernel start ticks for a host PID, or None when unavailable."""
- try:
- from gateway.status import get_process_start_time
- return get_process_start_time(pid) if pid else None
- except Exception:
- return None
- @classmethod
- def _host_pid_is_ours(cls, pid: Optional[int], expected_start: Optional[int]) -> bool:
- """True only if ``pid`` is alive AND still the process we spawned.
- The kernel recycles PIDs, so a stored number can later name an unrelated
- process (seen in the wild: a browser's session leader tree-killed). The kernel
- start time captured at spawn must match the live one; with no baseline
- (legacy checkpoints, no ``/proc``) degrade to a bare liveness check."""
- return cls._is_host_pid_alive(pid) and (
- expected_start is None or cls._safe_host_start_time(pid) == expected_start)
- def _detached_host_fate(self, pid: Optional[int], expected_start: Optional[int]) -> str:
- """How a recovered host PID should be supervised.
- ``running`` — alive and still ours (start time matches, no baseline, or
- the start-time probe could not be read). Re-attach; do not invent an exit.
- ``reused`` — alive, and the start time positively differs. The number
- belongs to someone else; close our entry and never signal that PID.
- ``gone`` — the PID is not alive. Prune it; no exit status was collected.
- """
- if self._host_pid_is_ours(pid, expected_start):
- return "running"
- if not pid or not self._is_host_pid_alive(pid):
- return "gone"
- if expected_start is None:
- return "running"
- current = self._safe_host_start_time(pid)
- if current is None or current == expected_start:
- return "running"
- return "reused"
- def _refresh_detached_session(self, session: Optional[ProcessSession]) -> Optional[ProcessSession]:
- """Re-attach, close, or prune a recovered host-PID session.
- A completion is not queued here: recovery has no waitable handle, so it
- never collected an exit status.
- """
- if session is None or session.exited or not session.detached or session.pid_scope != "host" or session.output_log:
- return session # a log-backed adoption's reader collects the recorded exit itself
- fate = self._detached_host_fate(session.pid, session.host_start_time)
- if fate == "running":
- return session
- if fate == "gone":
- return self._prune_uncollected_detached(session)
- self._close_reused_detached(session)
- return session
- def _close_reused_detached(self, session: ProcessSession) -> None:
- """Close an entry whose PID was recycled onto another process.
- The stranger is not signalled, and no completion is queued: we never
- collected an exit status for the process we spawned.
- """
- with session._lock:
- if session.exited:
- return
- session.exited = True
- with self._lock:
- if session.id in self._running:
- session.exited_at = time.time()
- self._running.pop(session.id, None)
- self._finished[session.id] = session
- self._write_checkpoint()
- session._completion_event.set()
- def _prune_uncollected_detached(self, session: ProcessSession) -> Optional[ProcessSession]:
- """Drop a recovered entry whose PID is gone, without inventing an exit.
- An owned systemd scope stays reachable so kill can still reap it. A
- scope-less entry is removed: poll/list must not report a collected exit.
- """
- with session._lock:
- session.exited = True
- with self._lock:
- self._running.pop(session.id, None)
- if session.systemd_unit:
- self._finished[session.id] = session
- else:
- self._finished.pop(session.id, None)
- self._write_checkpoint()
- session._completion_event.set()
- return session if session.systemd_unit else None
- def _uncollected_gone(self, session: Optional[ProcessSession]) -> bool:
- """True when a detached entry was closed without a collected exit status
- and the PID is no longer alive. List/poll must not report that as exited."""
- return bool(
- session is not None and session.exited and session.exit_code is None
- and session.detached and session.pid_scope == "host"
- and not self._is_host_pid_alive(session.pid))
- @staticmethod
- def _config_value(section: str, key: str, fallback):
- """``config.yaml`` value for ``section.key``, else the DEFAULT_CONFIG value.
- Raises if config is unreadable; callers wrap with their own hard fallback so
- registry code paths never crash on a broken config file."""
- from hermes_cli.config import DEFAULT_CONFIG, cfg_get, read_raw_config
- val = cfg_get(read_raw_config(), section, key)
- return DEFAULT_CONFIG[section][key] if val is None else val
- @staticmethod
- def _config_seconds(key: str, fallback: float) -> float:
- """``terminal.<key>`` as a non-negative float (0 disables); *fallback* if unreadable."""
- try:
- return max(float(ProcessRegistry._config_value("terminal", key, fallback)), 0.0)
- except Exception:
- return fallback
- @staticmethod
- def _daemon_term_grace_seconds() -> float:
- """Grace (s) between SIGTERM and escalated SIGKILL; 0 disables escalation."""
- return ProcessRegistry._config_seconds("daemon_term_grace_seconds", 2.0)
- # ----- Spawn -----
- @staticmethod
- def _new_session(command, task_id, owner_task_id, session_key, cwd, **extra) -> ProcessSession:
- from gateway.session_context import get_session_env
- return ProcessSession(
- id=f"proc_{uuid.uuid4().hex[:12]}", command=command, task_id=task_id,
- owner_task_id=owner_task_id, session_key=session_key, cwd=cwd,
- parent_session_id=get_session_env("HERMES_SESSION_ID", ""),
- wsl_chain=_is_wsl_launcher_command(command),
- started_at=time.time(), **extra)
- @staticmethod
- def _env_temp_dir(env: Any) -> str:
- """Return the writable sandbox temp dir for env-backed background tasks."""
- get_temp_dir = getattr(env, "get_temp_dir", None)
- if callable(get_temp_dir):
- try:
- temp_dir = get_temp_dir()
- if isinstance(temp_dir, str) and temp_dir.startswith("/"):
- return temp_dir.rstrip("/") or "/"
- except Exception as exc:
- logger.debug("Could not resolve environment temp dir: %s", exc)
- return tempfile.gettempdir()
- def _scope_argv(self, session: ProcessSession, safe_command: str, unit_suffix: str, label: str) -> list[str]:
- """Login-shell argv for *safe_command* (parity with LocalEnvironment: rc files
- sourced, user tools on PATH), wrapped in a transient systemd scope when we are
- the supervised gateway (own cgroup: an OOM kills only the worker, not the
- gateway and its messaging control plane)."""
- argv = [_find_shell(), "-lic", f"set +m; {safe_command}"]
- # This applies to both pipe mode and the PTY path above. See #70716.
- in_supervised_gateway = _IS_LINUX and _is_supervised_gateway_process()
- if in_supervised_gateway and _systemd_run_user_scope_available():
- session.systemd_unit = f"hermes-worker-{unit_suffix}.scope"
- return _build_systemd_scope_argv(argv, unit_suffix=unit_suffix)
- if in_supervised_gateway:
- # Under a supervisor but no private cgroup: a worker OOM can still take
- # the whole gateway down.
- logger.debug(
- "%s background executor not isolated in a systemd scope "
- "(systemd-run --user unavailable); worker shares the gateway cgroup.", label)
- return argv
- @staticmethod
- def _spawn_env(env_vars: dict) -> dict:
- """Sanitized child env from the session's client toolchain; PYTHONUNBUFFERED so poll sees progress."""
- from agent.session_client_env import toolchain_environ
- env = _terminal_env(_sanitize_subprocess_env(toolchain_environ(), env_vars))
- env["PYTHONUNBUFFERED"] = "1"
- return env
- def _track_started(self, session: ProcessSession, reader_target, reader_name: str, extra_args=()) -> None:
- """Register before the reader can publish completion, even for an exited child."""
- from contextvars import copy_context
- # Reader completion must retain the producer's multiplex profile scope.
- reader = threading.Thread(target=copy_context().run, args=(reader_target, session, *extra_args),
- daemon=True, name=reader_name)
- session._reader_thread = reader
- with self._lock:
- self._prune_if_needed()
- # Completion takes this lock too. Starting here also leaves no
- # ghost entry if the interpreter cannot start another thread.
- reader.start()
- self._running[session.id] = session
- self._write_checkpoint()
- def _spawn_local_pty(self, session: ProcessSession, safe_command: str, env_vars: dict) -> ProcessSession:
- """PTY spawn for interactive CLI tools (Codex, Claude Code, REPLs).
- Raises ImportError when no PTY backend is installed and re-raises any spawn
- failure; ``spawn_local`` falls back to pipe mode in both cases."""
- if _IS_WINDOWS:
- from winpty import PtyProcess as _PtyProcessCls
- else:
- from ptyprocess import PtyProcess as _PtyProcessCls
- pty_argv = self._scope_argv(session, safe_command, session.id, "PTY")
- pty_env = self._spawn_env(env_vars)
- if session.systemd_unit:
- pty_env = systemd_user_bus_env(pty_env)
- # A PTY is a real TTY, so pager-happy tools (git log/diff, man) WILL page and
- # hang waiting for `q` — default them to cat, honoring any pager the user set.
- pty_env.setdefault("GIT_PAGER", "cat")
- pty_env.setdefault("PAGER", "cat")
- pty_proc = _PtyProcessCls.spawn(pty_argv, cwd=session.cwd, env=pty_env, dimensions=(30, 120))
- self._own_pty_job(session, pty_proc)
- session.pid = pty_proc.pid
- session.host_start_time = self._safe_host_start_time(session.pid)
- session._pty = pty_proc
- self._track_started(session, self._pty_reader_loop, f"proc-pty-reader-{session.id}")
- return session
- def spawn_local(
- self, command: str, cwd: str | None = None, task_id: str = "", session_key: str = "",
- env_vars: dict | None = None, use_pty: bool = False, owner_task_id: str = "",
- persist_on_release: bool = False, restore_login_path: bool = False) -> ProcessSession:
- """Spawn a background process locally (TERMINAL_ENV=local; other backends: spawn_via_env()).
- ``use_pty`` asks ptyprocess/pywinpty for a pseudo-terminal; on failure: pipe + ``pty_fallback``.
- ``persist_on_release``: no agent kill sweeps (#41225); ``restore_login_path``: re-add reset PATH dirs (#56634)."""
- # Bash runs ``A && B &`` as a subshell holding our stdout pipe open while a server B lives;
- # the rewriter makes it ``A && { B & }`` (#68915). Lazy: terminal_tool imports this module.
- from tools.terminal_tool_sudo import _rewrite_compound_background as _rewrite_bg
- safe_command = _append_dropped_path_entries(_rewrite_bg(command), self._spawn_env(env_vars).get("PATH", "") if restore_login_path else "")
- session = self._new_session(command, task_id, owner_task_id, session_key, _resolve_safe_cwd(cwd or os.getcwd()),
- persist_on_release=persist_on_release)
- pty_scope_attempted = False
- # A worker's pipe or PTY dies with the worker (the child's next write is SIGPIPE): write to a log.
- log = worker_output_log(session.id)
- if use_pty and log is None:
- try:
- return self._spawn_local_pty(session, safe_command, env_vars)
- except ImportError:
- session.pty_fallback = "the PTY library (ptyprocess/pywinpty) is not installed"
- logger.warning("ptyprocess not installed, falling back to pipe mode")
- except Exception as e:
- session.pty_fallback = f"the PTY spawn failed ({e})"
- logger.warning("PTY spawn failed (%s), falling back to pipe mode", e)
- if session.systemd_unit:
- pty_scope_attempted = True
- if not _stop_systemd_unit(session.systemd_unit):
- raise RuntimeError(
- "PTY scope could not be reaped; refusing pipe fallback "
- "to avoid duplicate command execution"
- ) from e
- session.systemd_unit = ""
- # Pipe path (non-PTY or PTY fallback).
- _popen_kwargs = {"creationflags": windows_hide_flags()} if _IS_WINDOWS else {}
- unit_suffix = f"{session.id}-pipe-fallback" if pty_scope_attempted else session.id
- spawn_argv = self._scope_argv(session, record_exit_command(log, safe_command) if log else safe_command,
- unit_suffix, "Local")
- spawn_env = self._spawn_env(env_vars)
- if session.systemd_unit:
- spawn_env = systemd_user_bus_env(spawn_env)
- # start_new_session is REQUIRED with systemd-run --scope too: the scope does not
- # give the worker a new session, so from an interactive TUI the worker would
- # share the foreground process group and background spawns would stop the whole
- # session (observed as dead TUIs in state T). Cgroup isolation is unaffected —
- # the scope attaches to the invoked process, not the spawning session.
- output = open_output_log(log) if log else subprocess.PIPE
- try:
- proc = self._spawn_job(
- session, spawn_argv, persist_on_release, text=True, cwd=session.cwd, env=spawn_env,
- encoding="utf-8", errors="replace", stdout=output, stderr=subprocess.STDOUT,
- stdin=subprocess.DEVNULL, **_popen_kwargs)
- finally:
- if log:
- os.close(output)
- session.process, session.output_log = proc, str(log or "")
- session.pid = proc.pid
- session.host_start_time = self._safe_host_start_time(session.pid)
- try:
- self._track_started(session, self._log_reader_loop if log else self._reader_loop, f"proc-reader-{session.id}")
- except Exception:
- self._reap_untracked(session, proc)
- raise
- return session
- def adopt_local(
- self, proc: subprocess.Popen, *, command: str, cwd: Optional[str], task_id: str = "",
- session_key: str = "", owner_task_id: str = "", output_so_far: str = "",
- notify_on_complete: bool = True) -> ProcessSession:
- """Take over a still-running foreground Popen as a tracked background session
- (yield-to-background: the user sent a message while the command was running).
- The caller has stopped its own drain thread; the registry's reader continues from
- the pipe's current position and ``output_so_far`` seeds the buffer so nothing
- already captured is lost."""
- session = self._new_session(command, task_id, owner_task_id, session_key, cwd)
- session.process = proc
- session.pid = proc.pid
- session.host_start_time = self._safe_host_start_time(session.pid)
- session.notify_on_complete = notify_on_complete
- handed_pgid = getattr(proc, "_hermes_owned_pgid", None)
- self._take_handoff_group(session, proc)
- if output_so_far:
- session.append_output(output_so_far)
- try:
- self._track_started(session, self._reader_loop, f"proc-reader-{session.id}")
- except Exception:
- # The caller resumes its foreground wait: give it back its group, still covered.
- self._return_handoff_group(session, proc, handed_pgid)
- raise
- return session
- def spawn_via_env(
- self, env: Any, command: str, cwd: str | None = None, task_id: str = "", session_key: str = "",
- timeout: int = 10, owner_task_id: str = "", persist_on_release: bool = False) -> ProcessSession:
- """Spawn a background process inside a non-local backend's sandbox.
- The command is wrapped to capture its in-sandbox PID and redirect output to a
- log file that later execute() calls poll. No live pipe or stdin, but it runs in
- the correct sandbox context. ``persist_on_release`` keeps the process out of
- agent-lifecycle kill sweeps (#41225)."""
- session = self._new_session(command, task_id, owner_task_id, session_key, cwd, env_ref=env, pid_scope="sandbox",
- persist_on_release=persist_on_release)
- temp_dir = self._env_temp_dir(env)
- log_path, pid_path, exit_path = (f"{temp_dir}/hermes_bg_{session.id}.{ext}" for ext in ("log", "pid", "exit"))
- q = shlex.quote
- bg_command = (
- f"mkdir -p {q(temp_dir)} && "
- f"( nohup bash -lc {q(command)} > {q(log_path)} 2>&1; "
- f"rc=$?; printf '%s\\n' \"$rc\" > {q(exit_path)} ) & "
- f"echo $! > {q(pid_path)} && cat {q(pid_path)}")
- result = env.execute(bg_command, cwd=cwd or "", timeout=timeout, rewrite_compound_background=False)
- output = result.get("output", "").strip()
- session.pid = next((int(ln) for ln in map(str.strip, output.splitlines()) if ln.isdigit()), None)
- # No PID from the wrapper (no bash in the image, sandbox gone, broken redirect): the
- # command never started, so raise like a failed Popen instead of returning a session.
- if session.pid is None:
- raise RuntimeError(f"launch wrapper exited {result.get('returncode')} without a PID: "
- f"{output or '(no output)'}")
- self._track_started(
- session, self._env_poller_loop, f"proc-poller-{session.id}", (env, log_path, pid_path, exit_path))
- return session
- # ----- Reader / Poller Threads -----
- def _reader_loop(self, session: ProcessSession):
- """Background thread: read stdout from a local Popen process.
- ``buffer.read1(4096)`` not ``TextIOWrapper.read(4096)``: on pipes the latter
- blocks until EOF, landing "live" output in one burst at exit. Orphaned-pipe
- guard: a backgrounded grandchild (``node server.js &``) inherits our pipe's write
- end so EOF never arrives while it lives, which would park this thread and never
- fire ``notify_on_complete``; on POSIX we wait on an ``fd_wait`` selector (no FD_SETSIZE
- cap, #94928) and stop draining shortly after the direct child exits.
- Windows pipes lack select(), so the lazy ``_reconcile_local_exit`` is the net (#68915, #8340).
- """
- # ``bash -lic`` without a tty writes its startup warnings one write() per line, so the
- # reader can wake between them; strip leading noise from every chunk until the
- # process has produced real output, not just from the first read.
- head_noise = True
- # A split multibyte UTF-8 char would become U+FFFD with stateless decoding; the
- # incremental decoder holds the partial sequence until the rest arrives.
- decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") # (openclaw/openclaw#112325)
- def _append_chunk(chunk: str):
- nonlocal head_noise
- if head_noise:
- chunk = self._clean_shell_noise(chunk)
- head_noise = not chunk.strip()
- self._ingest_output(session, chunk)
- _waiter = None
- try:
- proc = session.process
- if proc is None or proc.stdout is None:
- return
- stdout = proc.stdout
- raw_read = getattr(getattr(stdout, "buffer", None), "read1", None)
- def _read_once():
- """One 4 KiB read: decoded text ('' for a partial multibyte tail), None at EOF."""
- if raw_read is None: # mocked/alternate streams without a raw buffer: less "live"
- return stdout.read(4096) or None
- raw = raw_read(4096)
- return decoder.decode(raw) if raw else None
- # select() needs a real OS fd; mocked streams (tests, adapters) may lack
- # fileno() and use the blocking read instead.
- try:
- fd = stdout.fileno() if raw_read is not None and not _IS_WINDOWS else None
- except Exception:
- fd = None
- if not (isinstance(fd, int) and fd >= 0):
- fd = None
- from tools.environments.fd_wait import open_fd_waiter
- _waiter = open_fd_waiter(fd) if fd is not None else None
- session._reader_selectable = _waiter is not None
- idle_after_exit = 0
- drained_after_request = 0
- while True:
- if _waiter is not None:
- # Once a finish is requested, read only what is already buffered: the exited
- # child's tail, never an orphaned grandchild's future writes.
- wait_s = 0 if session._reader_finish_requested.is_set() else 0.2
- ready = _waiter.wait(wait_s)
- if not ready:
- if session._reader_finish_requested.is_set():
- break
- # Direct child gone and pipe idle ~200ms: a few more cycles for a
- # buffered tail, then stop rather than wait forever on an orphaned
- # grandchild's pipe.
- if proc.poll() is not None:
- # See #68915.
- idle_after_exit += 1
- if idle_after_exit >= 3:
- break
- continue
- chunk = _read_once()
- if chunk is None:
- break # true EOF — all writers closed
- if chunk:
- _append_chunk(chunk)
- if session._reader_finish_requested.is_set():
- # A single chunk is not the tail: the child can exit with a whole pipe buffer
- # unread. The cap only stops a grandchild that never lets the pipe go empty.
- drained_after_request += len(chunk)
- if drained_after_request >= _FINAL_DRAIN_MAX_CHARS:
- break
- idle_after_exit = 0
- except Exception as e:
- logger.debug("Process stdout reader ended: %s", e)
- finally:
- if _waiter is not None:
- _waiter.close()
- self._finish_reader(
- session, decoder, _append_chunk, "Process",
- session.process.wait, lambda: session.process.returncode)
- def _finish_reader(self, session, decoder, append, label, wait, exit_code) -> None:
- """Reader-thread teardown: flush the decoder (a truncated multibyte tail becomes
- one U+FFFD instead of vanishing), reap the child (no zombies), record the exit.
- A process may close stdout long before it exits. The reader owns a dedicated
- daemon thread, so it must keep waiting rather than publish a false completion
- and discard the only ``Popen`` handle that can reap the child.
- """
- with suppress(Exception):
- tail = decoder.decode(b"", final=True)
- if tail:
- append(tail)
- try:
- wait()
- except Exception as e:
- # A PTY child reaped by isalive() already has its exitstatus; only an
- # unknown status must stay tracked for later reconciliation.
- if exit_code() is None:
- logger.warning("%s wait failed; leaving process tracked: %s", label, e)
- return
- logger.warning("%s wait failed; recording known exit status: %s", label, e)
- self._finish_exited(session, exit_code())
- _log_delta_command = staticmethod(log_delta_command)
- def _env_poller_loop(self, session: ProcessSession, env: Any, log_path: str, pid_path: str, exit_path: str):
- """Background thread: poll a sandbox log file for non-local backends."""
- q = shlex.quote
- # Byte offset already read from the log (bytes, not chars: the shell counts bytes).
- prev_output_bytes = 0
- while not session.exited:
- time.sleep(2)
- try:
- # Read only the bytes written since the last poll.
- raw = env.execute(self._log_delta_command(q(log_path), prev_output_bytes),
- timeout=10).get("output", "")
- header, _, delta = raw.partition("\n")
- try:
- size_str, offset_str = header.split()
- new_size = int(size_str)
- used_offset = int(offset_str)
- except ValueError:
- # No usable header (command failed, shell missing a tool): skip this
- # poll rather than act on a half-read value.
- new_size = None
- used_offset = None
- delta = ""
- if new_size is not None:
- if used_offset < prev_output_bytes:
- # Log rotated/truncated: what we hold no longer lines up. Restart.
- with session._lock:
- if session.exited:
- return
- session.output_buffer = ""
- prev_output_bytes = new_size
- if delta:
- # A kill that landed after this fetch already published its output
- # snapshot: leave the output as the kill reported it (as the PTY reader does).
- with session._lock:
- if session.exited:
- return
- session.output_buffer += delta
- if len(session.output_buffer) > session.max_output_chars:
- session.output_buffer = session.output_buffer[-session.max_output_chars:]
- self._check_watch_patterns(session, delta)
- self._emit_output(session, delta)
- check = env.execute(
- f"kill -0 \"$(cat {q(pid_path)} 2>/dev/null)\" 2>/dev/null; echo $?", timeout=5)
- check_output = check.get("output", "").strip()
- if check_output and check_output.splitlines()[-1].strip() != "0":
- # Exited -- read the exit code captured by the wrapper shell.
- exit_str = env.execute(f"cat {q(exit_path)} 2>/dev/null", timeout=5).get("output", "").strip()
- try:
- exit_code = int(exit_str.splitlines()[-1].strip())
- except (ValueError, IndexError):
- exit_code = -1
- self._finish_exited(session, exit_code)
- return
- except Exception:
- # Environment might be gone (sandbox reaped, etc.). A kill that already
- # settled the session keeps its result.
- self._finish_exited(session, -1, reason="lost", source="backend_lost")
- return
- def _pty_reader_loop(self, session: ProcessSession):
- """Background thread: read output from a PTY process."""
- pty = session._pty
- # Same split-multibyte handling as _reader_loop.
- # PTY reads can split a multibyte UTF-8 character across chunks just like pipe reads — hold partial
- # sequences until the rest arrives. (Ported from openclaw/openclaw#112325.)
- decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
- # Programs in a PTY can block waiting for replies to device-status / window-size /
- # cursor-position / DEC private-mode queries. Answer the bounded set and strip the
- # queries from captured output. POSIX only: Windows ConPTY is a real console host that
- # answers itself (and pywinpty yields str chunks, not bytes).
- responder = None
- if not _IS_WINDOWS:
- from tools.pty_query_responder import PtyQueryResponder
- responder = PtyQueryResponder(rows=30, cols=120)
- def ingest(text: str) -> None:
- # A kill can leave this reader running while a detached descendant holds the
- # slave open (_release_finished_handles defers the close to it). Keep draining,
- # but leave the killed session's output as the kill reported it.
- self._ingest_output(session, text, unless_exited=True)
- try:
- while pty.isalive():
- try:
- chunk = pty.read(4096)
- if chunk:
- # ptyprocess returns bytes; pywinpty returns str
- if responder is not None and isinstance(chunk, bytes):
- chunk, replies = responder.process(chunk)
- if replies:
- try:
- pty.write(replies)
- except Exception:
- logger.debug(
- "PTY query response write failed",
- exc_info=True,
- )
- text = chunk if isinstance(chunk, str) else decoder.decode(chunk)
- if text:
- ingest(text)
- except Exception: # EOFError included
- break
- except Exception as e:
- logger.debug("PTY stdout reader ended: %s", e)
- if responder is not None:
- # A query prefix split across the final reads is plain output after all.
- tail = decoder.decode(responder.flush())
- if tail:
- ingest(tail)
- self._finish_reader(
- session, decoder, ingest, "PTY",
- pty.wait, lambda: pty.exitstatus if hasattr(pty, 'exitstatus') else -1)
- def _ingest_output(self, session: ProcessSession, text: str, *, unless_exited: bool = False) -> None:
- """Buffer a freshly-read chunk, then scan watch patterns and stream it live.
- ``unless_exited`` drops the chunk once the session has exited (atomically with a kill)."""
- if not unless_exited:
- session.append_output(text)
- elif not session.append_output_if_running(text):
- return
- self._check_watch_patterns(session, text)
- self._emit_output(session, text)
- def _finish_exited(self, session: ProcessSession, exit_code, reason: str = "exited", source: str = "") -> None:
- """Mark a reader-observed exit (a raced kill keeps its own code/reason) and finish.
- Under the session lock, like kill_process's commit: otherwise the kill can land between
- mark_exited's ``killed`` check and its writes, and the reader overwrites the kill."""
- with session._lock:
- session.mark_exited(exit_code, reason, source)
- self._move_to_finished(session)
- def _move_to_finished(self, session: ProcessSession) -> bool:
- """Move a session from running to finished.
- Idempotent: kill_process() and the reader thread can both call this; only
- the FIRST move enqueues the completion notification, so no duplicates.
- Returns True when this call is the one that persisted the session."""
- with self._lock:
- was_running = session.id in self._running
- if was_running:
- session._finish_claimed = True
- session.exited_at = time.time()
- # Keep the session tracked until its result is durable. A finite
- # parent must not observe completion and exit during this write.
- save_completed_result(session)
- self._running.pop(session.id)
- self._finished[session.id] = session
- self._release_job_group(session)
- # Release the retained Popen/PTY handles now: otherwise every
- # finished-but-unpruned session keeps its stdout pipe (or PTY master)
- # FD open until FINISHED_TTL_SECONDS elapses, and heavy background
- # churn can exhaust the gateway's FD limit. On the reader-thread path
- # the pipe is already at EOF; on the kill/reconcile paths the reader
- # may still be draining — its next read raises on the closed stream
- # and the loop exits, dropping at most the unread tail of a process
- # that was just killed. poll()/wait()/read_log() serve from the
- # buffered ``output_buffer``, never from the pipe.
- try:
- self._publish_finished(session, was_running)
- finally:
- # The owner releases waiters only after its notice is queued (or its publish
- # failed, so a waiter never parks forever). A duplicate leaves the event to it.
- if was_running or not session._finish_claimed:
- session._completion_event.set()
- return was_running
- def _publish_finished(self, session: ProcessSession, was_running: bool) -> None:
- self._release_finished_handles(session)
- self._write_checkpoint()
- if was_running and session.notify_on_complete:
- notification = {
- "type": "completion",
- "session_id": session.id,
- "session_key": session.session_key,
- "task_id": session.task_id,
- "owner_task_id": session.owner_task_id,
- "command": session.command,
- **({"handoff_note": session.handoff_note} if session.handoff_note else {}),
- **self._exit_fields(session),
- # A consumer that relays the output (a bot DM's reply) must know it is not whole.
- **_completion_output(session),
- # Stable producer identity across checkpoint recovery (unlike a
- # consumer-observed completion timestamp).
- "started_at": session.started_at,
- }
- _redact_process_result(notification)
- self.completion_queue.put(notification)
- @staticmethod
- def _exit_fields(session: ProcessSession) -> dict:
- return {
- "exit_code": session.exit_code,
- "completion_reason": session.completion_reason,
- "termination_source": session.termination_source,
- }
- def _release_finished_handles(self, session: ProcessSession):
- """Close a finished session's OS handles (Popen pipes / PTY master).
- Best-effort and idempotent: the session may have no local Popen (env
- backends, detached recovery), or the handles may already be closed by
- the reader loop / kill path. Closing a Popen's stream objects does not
- kill anything — the child has already exited — it only releases the
- parent's pipe FDs, which is exactly the retained-resource leak.
- """
- proc = session.process
- if proc is not None:
- for stream in (proc.stdout, proc.stderr, proc.stdin):
- if stream is not None:
- with suppress(OSError, ValueError): # a stdin flush can hit EPIPE
- stream.close()
- if session._pty is not None:
- # A live ptyprocess reader sits in a blocking read holding the PTY file
- # object's buffer lock, and that read only ends once every holder of
- # the slave side is gone. A descendant that setsid()s past the kill
- # keeps it open, so close() here would block forever (under _lock on
- # the prune path). The reader closes the PTY itself via
- # _finish_reader once its read ends. pywinpty reads don't block, so
- # Windows closes here as before.
- reader = session._reader_thread
- if (not _IS_WINDOWS and reader is not None and reader.is_alive()
- and reader is not threading.current_thread()):
- return
- # ptyprocess/pywinpty close() is idempotent (``closed`` flag) and
- # closes the master fd exactly once; it raises only if the child
- # ignores SIGKILL, which we don't want to surface on the finish path.
- with suppress(Exception):
- session._pty.close()
- # ----- Query Methods -----
- @staticmethod
- def _reader_finalizing(session: ProcessSession) -> bool:
- """A reconcile saw the direct child exit and set ``exited``, but the reader that owns the
- output is still on its final drain: the buffer is not the whole output yet, so nothing
- may consume it as the completion."""
- reader = session._reader_thread
- return (session._reader_finish_requested.is_set() and not session._completion_event.is_set()
- and reader is not None and reader.is_alive())
- def is_completion_consumed(self, session_id: str) -> bool:
- """Check if a completion notification was already consumed via wait/log."""
- return session_id in self._completion_consumed
- def is_session_waiting(self, session_id: str) -> bool:
- """Whether a goal loop (``hermes_cli.goals`` wait barrier) should stay parked on
- this session: still running AND, with ``watch_patterns``, none matched yet (a
- long-lived watcher unblocks on its trigger, not on exit). Unknown/exited/
- already-fired sessions return False so a stale barrier can never wedge the loop."""
- with self._lock:
- session = (self._running.get(session_id) or self._finished.get(session_id)) if session_id else None
- if session is None:
- return False
- with suppress(Exception):
- refreshed = self._refresh_detached_session(session)
- if refreshed is None:
- return False
- session = refreshed
- return not session.exited and not (
- session.watch_patterns and not session._watch_disabled and session._watch_hits > 0)
- def wait_for_pending_completions(
- self, task_id: Optional[str] = None, *, timeout: float | None = None, poll_interval: float = 1.0,
- ) -> dict:
- """Bounded linger for ``notify_on_complete`` background processes at one-shot exit.
- A one-shot CLI run (``hermes -q/-Q/-z``) exits when its turn ends; a background
- process it spawned still holds a stdout pipe owned by the dying parent and dies of
- SIGPIPE seconds later (Bot Mode handoff replies were the visible casualty). Only
- ``notify_on_complete`` processes carry a completion contract — servers/daemons/
- watchers aren't the parent's to wait for. ``task_id=None`` waits on every tracked
- process; ``timeout=None`` reads ``terminal.oneshot_completion_wait_seconds`` (``<= 0``
- disables). Each pass re-reconciles child state so an orphaned-pipe exit can't wedge
- the linger. Returns ``{"waited", "completed", "timed_out"}`` id lists.
- Bot Mode handoff REPLIES are the visible casualty (#90879): a recipient invoked as ``hermes -p <bot>
- chat -Q --query-file ...`` dispatches its reply via ``message_agent`` / ``bot_relay`` exactly this
- way, then exits, and the reply process is destroyed ~3s later. The sender waits forever for a reply
- that was already killed.
- See #17327.
- """
- if timeout is None:
- timeout = self._oneshot_completion_wait_seconds()
- result: dict = {"waited": [], "completed": [], "timed_out": []}
- with self._lock:
- # `_finished` too: `_move_to_finished` pops a session from `_running` and enqueues its completion
- # only after releasing handles and writing the checkpoint. A parent whose turn ends inside that
- # window would otherwise see nothing pending, drain nothing and exit without the follow-up turn.
- pending = [
- s for store in (self._running, self._finished) for s in store.values()
- if s.notify_on_complete and not s._completion_event.is_set()
- and (task_id is None or s.owner_task_id == task_id)
- ]
- if not pending or timeout <= 0:
- return result
- result["waited"] = [s.id for s in pending]
- logger.info(
- "One-shot exit lingering (bounded %ss) for %d notify_on_complete "
- "background process(es): %s",
- timeout, len(pending), ", ".join(s.id for s in pending))
- deadline = time.monotonic() + max(float(timeout), 0.0)
- interval = max(float(poll_interval), 0.05)
- try:
- from tools.interrupt import is_interrupted as _is_interrupted
- except Exception:
- _is_interrupted = lambda: False
- interrupted = False
- for session in pending:
- try:
- while not session._completion_event.is_set():
- if interrupted or _is_interrupted():
- interrupted = True
- break
- remaining = deadline - time.monotonic()
- if remaining <= 0:
- break
- # Reconcile first so orphaned-pipe and detached exits fire the event.
- with suppress(Exception):
- # Reconcile first: catches direct-child exits whose reader is blocked on a pipe held
- # open by a descendant (#17327) and detached/env sessions, so the event actually
- # fires.
- # Reconcile against real child state before reading session.exited. Guards against
- # orphaned-pipe reader hangs (issue #17327).
- # Reconcile against real child state — guards against orphaned- pipe reader hangs
- # where the reader is blocked but the direct child has already exited (issue
- # #17327).
- self._reconcile_local_exit(session)
- if self._refresh_detached_session(session) is None:
- break
- if session._completion_event.is_set():
- break
- session._completion_event.wait(min(remaining, interval))
- except KeyboardInterrupt:
- # Stop waiting, but never let the interrupt skip the caller's durable
- # teardown (session flush, end_session) that follows.
- interrupted = True
- result["completed" if session._completion_event.is_set() else "timed_out"].append(session.id)
- if result["timed_out"]:
- logger.warning(
- "One-shot exit linger timed out after %ss with %d background "
- "process(es) still running: %s — they may be killed when this "
- "process exits.",
- timeout, len(result["timed_out"]), ", ".join(result["timed_out"]))
- return result
- @staticmethod
- def _oneshot_completion_wait_seconds() -> float:
- """Linger (s) for one-shot exits with pending notify_on_complete processes; 0 disables."""
- return ProcessRegistry._config_seconds("oneshot_completion_wait_seconds", 600.0)
- def _drain_should_skip(self, session_id: str, *, skip_poll_observed: bool = True) -> bool:
- """Skip a completion the CLI agent already has this turn — consumed via wait/log
- or observed inline via poll(). Gateway/tui watchers check only
- ``is_completion_consumed`` so a read-only poll never suppresses their turn.
- Skips when the agent has either truly consumed the output (wait/log → ``_completion_consumed``) or
- observed the exit inline via poll() (``_poll_observed``). In both cases the CLI agent already has
- the result this turn, so injecting a [SYSTEM: ...] completion would be a duplicate (#8228).
- """
- return session_id in self._completion_consumed or (skip_poll_observed and session_id in self._poll_observed)
- @staticmethod
- def _surface_child_process_notifications() -> bool:
- """``delegation.surface_child_process_notifications``; False on any config
- error — never crash the drain loop."""
- try:
- return bool(ProcessRegistry._config_value("delegation", "surface_child_process_notifications", False))
- except Exception:
- return False
- @staticmethod
- def _owns_event(evt: dict, session_key: str, owns_event, is_async_delegation: bool) -> bool:
- """Routing verdict for one drained event (see drain_notifications); False = requeue."""
- evt_session_key = str(evt.get("session_key") or "")
- requires_positive_proof = is_async_delegation or bool(evt_session_key or evt.get("origin_ui_session_id"))
- if owns_event is not None and requires_positive_proof:
- try:
- return bool(owns_event(evt))
- except Exception:
- return False # fail closed — never leak on a broken check
- if session_key and requires_positive_proof:
- return evt_session_key == session_key
- # Restored payloads from a previous process: an unfiltered drain cannot prove
- # ownership, so leave them for the owner.
- return not (is_async_delegation and evt.get("restored"))
- def restore_completions(self) -> int:
- """Rehydrate durable delegation completions from the LAUNCH profile's ledger, once per
- process. Called by the first consumer that drains the queue (CLI/TUI drain, gateway boot,
- TUI poller) so a mere ``import model_tools`` never touches state.db (#123265). The replay
- always runs in the launch scope: the TUI poller / prompt_turn drain call this under the
- session's profile binding, and a once-per-process replay taken under a secondary's scope
- would leave the launch ledger unreplayed for the life of the process. Secondaries are
- replayed by the gateway's ``_restore_secondary_completion_ledgers``."""
- if self._completions_restored:
- return 0
- self._completions_restored = True
- from hermes_constants import reset_hermes_home_override, set_hermes_home_override
- token = set_hermes_home_override(None)
- try:
- from tools.async_delegation import restore_undelivered_completions
- return restore_undelivered_completions(self.completion_queue)
- except Exception as exc:
- logger.warning("Could not restore async delegation completions: %s", exc)
- return 0
- finally:
- reset_hermes_home_override(token)
- def drain_notifications(
- self, session_key: str = "", owns_event=None, *, skip_poll_observed: bool = True,
- ) -> list[tuple[dict, str]]:
- """Pop all pending events and return ``(raw_event, formatted_text)`` pairs.
- Skips completions per ``_drain_should_skip`` (gateway/TUI pass
- ``skip_poll_observed=False``). Routing (``_owns_event``): async-delegation events
- always need ownership proof, ordinary events once they carry ``session_key`` or
- ``origin_ui_session_id``; ``owns_event(evt)`` (strongest; the TUI passes a
- compression-chain-aware check) consumes ONLY on True, ``session_key`` uses plain
- equality; non-owned events are re-queued for their owner. No filter consumes
- everything (legacy single-session) except restored delegation payloads (fail-closed)."""
- self.restore_completions()
- results: list[tuple[dict, str]] = []
- requeue: list[dict] = []
- # delegation.surface_child_process_notifications, read at most once per drain
- # and only when an sa- event shows up.
- surface_child: bool | None = None
- while not self.completion_queue.empty():
- try:
- evt = self.completion_queue.get_nowait()
- except Exception:
- break
- is_async_delegation = evt.get("type") == "async_delegation"
- if not self._owns_event(evt, session_key, owns_event, is_async_delegation):
- requeue.append(evt)
- continue
- # Routing happened first so a foreign session cannot drop the owner's
- # event via its own consumed/observed state.
- _evt_sid = evt.get("session_id", "")
- if evt.get("type") == "completion" and self._drain_should_skip(
- _evt_sid, skip_poll_observed=skip_poll_observed):
- continue
- # Subagent-owned process notifications are suppressed by default — the
- # child's delegation result is the deliverable. Judge ownership on
- # owner_task_id (RAW spawning id; task_id is the container key, collapsed
- # by _resolve_container_task_id). Dropped, NOT requeued: children never
- # drain, so a requeue would pin the event forever. 'async_delegation'
- # is the result itself and is NEVER suppressed.
- _evt_task_id = str(evt.get("owner_task_id") or evt.get("task_id") or "")
- if is_child_process_notification(evt):
- if surface_child is None:
- surface_child = self._surface_child_process_notifications()
- if not should_surface_notification(evt, surface_child=surface_child):
- logger.debug(
- "Suppressed subagent-owned process notification "
- "(delegation.surface_child_process_notifications=false): "
- "type=%s session_id=%s task_id=%s",
- evt.get("type", "completion"), _evt_sid, _evt_task_id)
- continue
- if text := format_process_notification(evt):
- results.append((evt, text))
- for evt in requeue:
- self.completion_queue.put(evt)
- return results
- # Minimum suffix chars for prefix resolution; "p"/"proc_1" are too collision-prone.
- _MIN_PREFIX_CHARS = 4
- def get(self, session_id: str) -> Optional[ProcessSession]:
- """Session by full ID or unique prefix (``proc_4dae`` / bare ``4dae``, like git
- short hashes); ambiguous or too-short prefixes resolve to None, never a guess."""
- if not isinstance(session_id, str) or not session_id:
- return None
- with self._lock:
- session = self._running.get(session_id) or self._finished.get(session_id)
- if session is None:
- session = load_completed_results(session_id).get(session_id)
- return self._refresh_detached_session(session if session is not None else self._resolve_prefix(session_id))
- def _resolve_prefix(self, session_id: str) -> Optional[ProcessSession]:
- """Resolve a unique session-ID prefix (a bare hex tail is normalized to
- ``proc_<tail>``); :meth:`get` tries exact first."""
- query = session_id.strip() if isinstance(session_id, str) else ""
- if not query:
- return None
- if not query.startswith("proc_"):
- query = f"proc_{query}"
- if len(query) - len("proc_") < self._MIN_PREFIX_CHARS:
- return None
- matches = load_completed_results(query)
- with self._lock:
- matches.update({
- sid: s for store in (self._running, self._finished)
- for sid, s in store.items() if sid.startswith(query)
- })
- return next(iter(matches.values())) if len(matches) == 1 else None
- def _reconcile_local_exit(self, session: ProcessSession) -> None:
- """Reconcile ``session.exited`` against the real child state.
- The reader flips ``exited`` only at EOF; when the direct child has exited but a
- descendant (e.g. a daemon from ``hermes update``) holds the pipe open, poll()
- would report "running" forever. If ``Popen.poll()`` has an exit code, drain
- readable bytes non-blocking and flip ``exited``. No-op for env/PTY, exited and
- detached sessions.
- The reader thread (`_reader_loop`) sets `session.exited = True` only in its `finally` block, which
- runs when `stdout.read()` returns EOF. If the direct `Popen` child has exited but a descendant
- process (e.g. a daemon spawned by `hermes update` restarting the gateway) is still holding the
- stdout pipe open, the reader blocks forever and poll() keeps returning "running" indefinitely (issue
- #17327 — 74 polls over 7 minutes on Feishu).
- """
- if session is None or session.exited:
- return
- proc = getattr(session, "process", None)
- if proc is None:
- return
- try:
- rc = proc.poll()
- except Exception:
- return
- if rc is None:
- return # Direct child still running — reader block is legitimate.
- reader = session._reader_thread
- if (
- not _IS_WINDOWS
- and session._reader_selectable
- and reader is not None
- and reader.is_alive()
- ):
- # The reader owns the pipe and completion payload. Asking it to
- # finish avoids a competing TextIOWrapper read here racing the
- # reader, publishing an empty owner-stamped result, then closing
- # the pipe before the buffered tail is ingested. It wakes within
- # the reader's bounded select interval and drains what is buffered.
- session._reader_finish_requested.set()
- # Flipping ``exited`` before that drain lands would hand poll()/wait() a truncated
- # snapshot (and wait() marks it consumed), so give the reader a moment to publish.
- if session._completion_event.wait(_READER_FINAL_DRAIN_WAIT_SECONDS):
- return
- with session._lock:
- session.mark_exited(rc)
- logger.info(
- "Reconciled session %s: direct child exited with code %s; "
- "reader will publish the owned completion after its final drain.",
- session.id, rc)
- return
- # Best-effort non-blocking drain of whatever the reader hasn't consumed.
- stdout = getattr(proc, "stdout", None)
- if stdout is not None and not _IS_WINDOWS:
- try:
- import fcntl
- fd = stdout.fileno()
- flags = fcntl.fcntl(fd, fcntl.F_GETFL)
- fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
- try:
- with suppress(BlockingIOError, OSError, ValueError):
- chunk = stdout.read()
- if chunk:
- session.append_output(chunk if isinstance(chunk, str) else chunk.decode("utf-8", errors="replace"))
- finally:
- with suppress(Exception):
- fcntl.fcntl(fd, fcntl.F_SETFL, flags)
- except Exception as e:
- logger.debug("Non-blocking drain failed for %s: %s", session.id, e)
- with session._lock:
- session.mark_exited(rc)
- logger.info(
- "Reconciled session %s: direct child exited with code %s but reader "
- "was still blocked (orphaned pipe). Flipped to exited.",
- session.id, rc)
- self._move_to_finished(session)
- @staticmethod
- def _status_head(session: ProcessSession) -> dict:
- return {"session_id": session.id, "command": session.command, "status": "exited" if session.exited else "running"}
- def poll(self, session_id: str) -> dict:
- """Check status and get new output for a background process."""
- session = self.get(session_id)
- if session is None or self._uncollected_gone(session):
- return _not_found(session_id)
- self._reconcile_local_exit(session) # orphaned-pipe reader guard
- # Read with the preview under the lock the reader appends and exits under, so the
- # finalization check describes the output actually returned.
- with session._lock:
- output_preview = _output_tail(session, 1000)
- exited, finalizing = session.exited, self._reader_finalizing(session)
- result = {
- **self._status_head(session), "pid": session.pid,
- "uptime_seconds": int(time.time() - session.started_at), "output_preview": output_preview}
- if exited:
- result.update(self._exit_fields(session))
- # Read-only: record in _poll_observed (CLI inline dedup) but NOT in
- # _completion_consumed, or a status check would suppress the watcher's
- # autonomous delivery turn. See __init__. A preview taken before the reader's
- # final drain is partial and must not suppress the full completion.
- if not finalizing:
- self._poll_observed.add(session_id)
- if session.detached and not session.output_log:
- result.update(detached=True, note="Process recovered after restart -- output history unavailable")
- return result
- def read_log(self, session_id: str, offset: int | None = None, limit: int = 200) -> dict:
- """Read the full output log with optional pagination by lines."""
- from tools.ansi_strip import strip_ansi
- session = self.get(session_id)
- if session is None:
- return _not_found(session_id)
- with session._lock:
- full_output = strip_ansi(session.output_buffer)
- exited, finalizing = session.exited, self._reader_finalizing(session)
- lines = full_output.splitlines()
- total_lines = len(lines)
- # An explicit offset=0 means "start from the first line" — previously it was conflated with the
- # default and silently returned the TAIL instead of the head (same falsy-coercion class as the
- # wait() timeout guard; salvaged from PR #60004, credit @isheng-eqi).
- if offset is None and limit > 0:
- selected = lines[-limit:]
- observed_completion_output = bool(selected) or total_lines == 0
- else:
- offset = offset or 0
- selected = lines[offset:offset + limit]
- stop = slice(offset, offset + limit).indices(total_lines)[1]
- observed_completion_output = total_lines == 0 or (bool(selected) and stop == total_lines)
- result = {
- **self._status_head(session), "output": "\n".join(selected),
- "total_lines": total_lines, "showing": f"{len(selected)} lines"}
- if exited and observed_completion_output and not finalizing:
- self._completion_consumed.add(session_id)
- return result
- def wait(self, session_id: str, timeout: int | None = None) -> dict:
- """Block until the process exits, the timeout elapses, the user interrupts, or a
- mid-turn user message (steer/redirect → ``request_yield``) releases the wait.
- ``timeout`` defaults to (and is clamped by) TERMINAL_TIMEOUT. Returns a dict
- with status exited|timeout|interrupted|not_found|error and an output snapshot."""
- from tools.interrupt import consume_yield as _consume_yield, is_interrupted as _is_interrupted
- try:
- max_timeout = int(os.getenv("TERMINAL_TIMEOUT", "180"))
- except (ValueError, TypeError):
- max_timeout = 180
- # The schema says minimum=1 but not every caller enforces it; timeout=0 is
- # falsy and would silently fall through to the default wait.
- if timeout is not None and timeout <= 0:
- return {"status": "error", "error": f"timeout must be positive (got {timeout})"}
- timeout_note = None
- effective_timeout = timeout or max_timeout
- if timeout and timeout > max_timeout:
- effective_timeout = max_timeout
- timeout_note = f"Requested wait of {timeout}s was clamped to configured limit of {max_timeout}s"
- session = self.get(session_id)
- if session is None:
- return _not_found(session_id)
- deadline = time.monotonic() + effective_timeout
- while time.monotonic() < deadline:
- session = self._refresh_detached_session(session)
- if session is None:
- return _not_found(session_id)
- self._reconcile_local_exit(session) # orphaned-pipe reader guard
- result = None
- if session.exited and not self._reader_finalizing(session):
- self._completion_consumed.add(session_id)
- result = self._exit_snapshot(session, "exited")
- elif _is_interrupted():
- result = {
- "status": "interrupted", "command": session.command, "output": _output_tail(session, 1000),
- "note": "User sent a new message -- wait interrupted"}
- elif _consume_yield(threading.current_thread().ident):
- # A steer/redirect landed mid-turn: redirect() asks tool workers to YIELD so
- # the user's message is delivered instead of parked behind this wait. The
- # process is untouched and still notify-tracked; the model should read the
- # steer text and respond, not re-issue the wait (kimi-code#3697 class).
- result = {
- "status": "interrupted", "command": session.command, "output": _output_tail(session, 1000),
- "process_running": True,
- "note": ("User sent a new message -- wait released; the process is still "
- "running and you will be notified on exit. Respond to the user now.")}
- if result is not None:
- if timeout_note:
- result["timeout_note"] = timeout_note
- return result
- remaining = deadline - time.monotonic()
- if remaining <= 0:
- break
- session._completion_event.wait(timeout=min(1.0, remaining))
- result = {
- "status": "timeout", "command": session.command, "output": _output_tail(session, 1000),
- # Not a failure — models re-issued identical waits after misreading this as an error.
- "process_running": True}
- base_note = (
- f"Wait window of {effective_timeout}s elapsed — the process is still running. This is not an error.")
- if session.started_at:
- base_note += f" Uptime: {int(time.time() - session.started_at)}s."
- base_note += (
- " notify_on_complete is set: you will be notified on exit — do more work instead of waiting again."
- if session.notify_on_complete else
- " Poll again later or use terminal(background=true, "
- "notify_on_complete=true) next time for automatic notification.")
- result["timeout_note"] = f"{timeout_note}. {base_note}" if timeout_note else base_note
- return result
- @staticmethod
- def _exit_snapshot(session: ProcessSession, status: str) -> dict:
- """Result dict for an exited session: exit metadata + the completion-sized output tail."""
- return {
- "status": status, "command": session.command,
- **ProcessRegistry._exit_fields(session), **_completion_output(session)}
- def kill_process(
- self, session_id: str, *, source: str = "process.kill", consume_output: bool = True,
- ) -> dict:
- """Kill a background process and return its output snapshot.
- ``consume_output`` is true for explicit tool/RPC kills (the caller sees the
- output). Bulk cleanup passes false so it doesn't suppress an autonomous
- completion notification — except abandoned-turn reaping (``kill_started_since``),
- which passes true so a killed abandoned process can't revive stopped work."""
- session = self.get(session_id)
- if session is None:
- return _not_found(session_id)
- if session.exited:
- # A double-forked descendant may still be alive in the systemd scope even
- # though the main process exited — stop the scope to reap survivors.
- # See #70716.
- # If the worker was spawned in its own systemd scope (#70716), stop the entire unit to reap any
- # double-forked descendants that were reparented inside the scope and survived the PID signal
- # above (reviewer gap #2). ``systemctl --user stop`` sends SIGTERM to every process in the
- # cgroup and escalates to SIGKILL after TimeoutStopSec. This is additive — the PID-based kill
- # above already handled the main process; this catches stragglers.
- scope_stopped = self._stop_scope(session)
- with session._lock:
- result = self._exit_snapshot(session, "already_exited")
- finalizing = self._reader_finalizing(session)
- if not scope_stopped:
- result["scope_stop_failed"] = session.systemd_unit
- # Only suppress the autonomous turn after its output is present in
- # the explicit kill result, matching wait/log consumption.
- if consume_output and not finalizing:
- self._completion_consumed.add(session_id)
- return result
- try:
- early = self._signal_kill(session, session_id, consume_output)
- if early is not None:
- return early
- # Additive to the PID kill: stopping the scope reaps double-forked
- # descendants reparented inside the cgroup.
- scope_stopped = self._stop_scope(session)
- # Post-kill verification (#115490): the signals above can leave
- # survivors (SIGTERM-ignoring daemons, scope escapees). A kill that
- # leaves a live tree must not write a killed receipt or prune the
- # session — the survivors would become unmanageable. Keep the
- # session running so it stays listed and killable.
- with session._lock:
- signal_race_exited = session.exited
- # A reader that finalised the session mid-signal already proved
- # real exit; only a still-running session needs tree-death proof.
- survivors = [] if signal_race_exited else self._post_kill_survivors(session)
- if survivors:
- alive = ", ".join(map(str, survivors))
- logger.warning(
- "Kill incomplete for %s: %d process(es) still alive (%s) — session kept running",
- session.id, len(survivors), alive)
- return {
- "status": "error",
- "error": (
- f"Kill incomplete: {len(survivors)} process(es) still alive "
- f"({alive}); session running"),
- "session_id": session.id, "survivors": survivors,
- "process_running": True}
- # Descendants outside the root's tree are visible only to the scope; a scope that did
- # not stop leaves nothing proving they are gone, even when the root exited meanwhile.
- # The session keeps its unit, so a later kill retries the stop.
- if not scope_stopped:
- with session._lock:
- root_exited = session.exited
- logger.warning("Kill incomplete for %s: systemd scope %s did not stop",
- session.id, session.systemd_unit)
- return {
- "status": "error",
- "error": f"Kill incomplete: systemd scope {session.systemd_unit} did not stop",
- "session_id": session.id, "scope_stop_failed": session.systemd_unit,
- "process_running": not root_exited}
- # Capture output, mark consumed, THEN expose ``exited`` to watcher tasks —
- # closes the delayed-notification race without losing the transcript.
- with session._lock:
- output = _completion_output(session)
- if consume_output:
- self._completion_consumed.add(session_id)
- session.exited = True
- session.exit_code = -15 # SIGTERM
- session.completion_reason = "killed"
- session.termination_source = source
- # The reader thread can finalise the session while the signal path
- # blocks in the SIGKILL grace window: its ``save_completed_result``
- # then persists this kill as a plain ``exited``. Re-write the receipt
- # so the durable record matches what the caller was told.
- if not self._move_to_finished(session):
- save_completed_result(session)
- self._write_checkpoint()
- return {
- "status": "killed", "session_id": session.id, "completion_reason": session.completion_reason,
- "termination_source": session.termination_source, **output}
- except Exception as e:
- return {"status": "error", "error": str(e)}
- @staticmethod
- def _stop_scope(session: ProcessSession) -> bool:
- """Stop the session's systemd scope: True when it has none or the stop worked. Every kill
- path stops the scope through here, so a failed stop always leaves it marked for retry."""
- if not session.systemd_unit:
- return True
- stopped = _stop_systemd_unit(session.systemd_unit)
- session._scope_stop_pending = not stopped
- return stopped
- def _signal_kill(self, session: ProcessSession, session_id: str, consume_output: bool) -> Optional[dict]:
- """Deliver the kill via PTY, local Popen tree, sandbox exec or recovered host
- PID. Returns a final result dict when the kill cannot proceed (recycled/dead
- recovered PID, or no runtime handle), else None."""
- if session._pty:
- try:
- session._pty.terminate(force=True)
- except Exception:
- if session.pid:
- os.kill(session.pid, signal.SIGTERM)
- elif session.process:
- # Tree kill: on Windows Popen.terminate() only kills the shell wrapper and
- # leaves Git Bash descendants behind.
- self._terminate_host_pid(session.process.pid, session.host_start_time)
- elif session.env_ref and session.pid:
- session.env_ref.execute(f"kill {session.pid} 2>/dev/null", timeout=5)
- elif session.detached and session.pid_scope == "host" and session.pid:
- # Same fate as poll/list: a gone or reused PID means our process is
- # gone — never tree-kill the stranger — but a live PID with an
- # unreadable start time is still ours and must really be killed.
- # If this recovered session also carries an owned systemd scope, stop that scope before
- # returning: a daemonized descendant may still be alive there even though the wrapper PID exited
- # or was recycled across the gateway restart (#70716, teknium1 review).
- if self._detached_host_fate(session.pid, session.host_start_time) != "running":
- scope_stopped = self._stop_scope(session)
- with session._lock:
- output = _completion_output(session)
- if consume_output:
- self._completion_consumed.add(session_id)
- # No waitable handle, so this is not a collected exit. Close the
- # entry without queueing a completion, and do not signal a PID
- # whose start time does not match.
- self._close_reused_detached(session)
- result = {"status": "already_exited", "exit_code": session.exit_code, **output}
- if not scope_stopped:
- result["scope_stop_failed"] = session.systemd_unit
- return result
- # Identity was just proven above. Re-passing the start time would make
- # an unreadable probe refuse the kill and leave the re-adopted child running.
- self._terminate_host_pid(session.pid)
- else:
- return {
- # Reject non-positive timeouts — the schema declares minimum=1, but not every caller
- # enforces schemas before dispatch. timeout=0 is falsy, so without this guard it silently
- # fell through (`0 or max_timeout`) to the DEFAULT wait instead of erroring. Salvaged from
- # PR #60004 (credit @isheng-eqi).
- "status": "error",
- "error": "Recovered process cannot be killed after restart because "
- "its original runtime handle is no longer available",
- }
- return None
- def _stdin_op(self, session_id: str, pty_op, pipe_op, ok: dict) -> dict:
- """``pty_op(pty)`` under PTY mode, else ``pipe_op(stdin)`` on the Popen pipe; *ok* on success."""
- from tools.process_registry_stdin import already_exited_error, stdin_unavailable_error
- session = self.get(session_id)
- if session is None:
- return _not_found(session_id)
- if session.exited:
- return {"status": "already_exited", "error": already_exited_error(session)}
- try:
- if session._pty:
- pty_op(session._pty)
- elif not session.process or not session.process.stdin:
- return {"status": "error", "error": stdin_unavailable_error(session)}
- else:
- pipe_op(session.process.stdin)
- return ok
- except Exception as e:
- return {"status": "error", "error": str(e)}
- def write_stdin(self, session_id: str, data: str) -> dict:
- """Send raw data to a running process's stdin (no newline appended)."""
- def via_pty(pty):
- # pywinpty expects str on Windows; ptyprocess expects bytes on POSIX.
- if _IS_WINDOWS:
- pty.write(data.decode("utf-8") if isinstance(data, bytes) else str(data))
- else:
- # surrogateescape: a PTY is a byte stream — round-trip the original
- # bytes instead of crashing on surrogate content.
- pty.write(data.encode("utf-8", "surrogateescape") if isinstance(data, str) else data)
- def via_pipe(stdin):
- stdin.write(data)
- stdin.flush()
- return self._stdin_op(session_id, via_pty, via_pipe, {"status": "ok", "bytes_written": len(data)})
- def submit_stdin(self, session_id: str, data: str = "") -> dict:
- """Send data + Enter. On a PTY Enter is the CR a keyboard sends: raw-mode TUIs (prompt_toolkit,
- Ink) read ``\\n`` as Ctrl+J and canonical children still get ``\\n`` via ICRNL; a Windows PTY
- gets ``\\r\\n`` (a bare ``\\n`` through pywinpty never ends a ConPTY line read). Pipes keep ``\\n``."""
- session = self.get(session_id)
- pty_enter = "\r\n" if _IS_WINDOWS else "\r"
- return self.write_stdin(session_id, data + (pty_enter if session is not None and session._pty else "\n"))
- def request_close_terminal(self, session_id: str) -> dict:
- """Ask the desktop GUI to close this process's read-only terminal tab. Does NOT
- kill the process — output keeps buffering and the tab can be reopened from the
- status stack. Errors when no UI close sink is wired."""
- if self.on_close is None:
- return {"status": "error", "error": "close_terminal is only available in the Hermes desktop app."}
- # The session may already be finished (or pruned) — the tab can still
- # linger and be closed, so a missing session is not an error here.
- try:
- self.on_close(self.get(session_id), session_id)
- except Exception as e:
- return {"status": "error", "error": str(e)}
- return {
- "status": "ok", "closed": session_id,
- "note": "Closed the read-only terminal tab. The process was not killed; "
- "its output remains available and the user can reopen the tab "
- "from the status stack."}
- def close_stdin(self, session_id: str) -> dict:
- """Close a running process's stdin / send EOF without killing the process."""
- session = self.get(session_id)
- msg = "EOF sent" if session is not None and session._pty else "stdin closed"
- return self._stdin_op(
- session_id, lambda pty: pty.sendeof(), lambda stdin: stdin.close(), {"status": "ok", "message": msg})
- def count_running(self) -> int:
- """O(1) running count for status-bar polling; dict ``len()`` is atomic, no lock."""
- return len(self._running)
- def list_sessions(self, task_id: str | None = None, session_key: str | None = None, *, include_retained: bool = False) -> list:
- """Running and recently-finished processes for ``task_id`` and/or ``session_key``;
- cross-task entries sharing the gateway session (a forgotten preview server
- blocking session reset) are flagged ``"session_scoped": true``.
- When ``task_id`` is given, processes that task spawned (its ``owner_task_id``) are included. When
- ``session_key`` is also given, session-scoped background processes (``background: true``) registered
- under that gateway session are surfaced too, even if they belong to a different task — so the agent
- can discover a forgotten preview server that is blocking session reset (#29177).
- """
- # Only an explicit tool query reads historical receipts. Status bars and
- # gateway liveness scans call this frequently and need the live registry.
- sessions = load_completed_results() if include_retained else {}
- with self._lock:
- sessions.update(self._finished)
- sessions.update(self._running)
- all_sessions = [
- refreshed for refreshed in (
- self._refresh_detached_session(s) for s in sessions.values()
- ) if refreshed is not None and not self._uncollected_gone(refreshed)
- ]
- if task_id or session_key:
- all_sessions = [
- s for s in all_sessions
- if (task_id and s.owner_task_id == task_id)
- or (session_key and s.session_key == session_key)
- ]
- result = []
- for s in all_sessions:
- # List-only refreshes must observe child exit even while descendants
- # keep the capture pipe open; retain the existing completion owner.
- self._reconcile_local_exit(s)
- entry = {
- "session_id": s.id,
- "command": s.command[:200],
- "cwd": s.cwd,
- "pid": s.pid,
- "owner_task_id": s.owner_task_id,
- "started_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(s.started_at)),
- "uptime_seconds": int(time.time() - s.started_at),
- "status": "exited" if s.exited else "running",
- "output_preview": s.output_buffer[-200:] if s.output_buffer else "",
- }
- # Flag processes surfaced only because they share the gateway session (not the current task) —
- # these are the long-lived background processes a user may have forgotten about (#29177).
- if task_id and session_key and s.owner_task_id != task_id and s.session_key == session_key:
- entry["session_scoped"] = True
- if s.wsl_chain:
- entry["wsl_chain"] = True
- entry["wsl_note"] = _WSL_CHAIN_NOTE
- # Trigger metadata for goal-loop judges (a watcher may never exit).
- if s.watch_patterns and not s._watch_disabled:
- entry.update(watch_patterns=list(s.watch_patterns), watch_hit=s._watch_hits > 0)
- if s.notify_on_complete:
- entry["notify_on_complete"] = True
- if s.persist_on_release:
- entry["persist_on_release"] = True
- if s.exited:
- entry["exit_code"] = s.exit_code
- entry["exited_at"] = s.exited_at
- entry["completion_reason"] = s.completion_reason
- if s.detached:
- entry["detached"] = True
- result.append(entry)
- return result
- # ----- Session/Task Queries (for gateway integration) -----
- def _any_running(self, predicate) -> bool:
- """True if any still-running session satisfies *predicate*, after refreshing
- detached sessions so a finished-but-unreaped process reads as inactive."""
- with self._lock:
- sessions = list(self._running.values())
- for session in sessions:
- self._refresh_detached_session(session)
- with self._lock:
- return any(not s.exited and predicate(s) for s in self._running.values())
- def has_active_processes(self, task_id: str) -> bool:
- """Whether any process for ``task_id`` is still running. Ownership is
- ``owner_task_id`` (the raw spawning id) like the other task-scoped queries:
- ``task_id`` on a session is the collapsed container key, shared across
- turns and delegate children, so a container-key match alone would miss a
- delegate child's own background work (#120546)."""
- return self._any_running(lambda s: s.owner_task_id == task_id)
- def running_owned_by(self, owner_task_id: str) -> list[ProcessSession]:
- """Running processes whose RAW spawning owner is ``owner_task_id``."""
- with self._lock:
- return [s for s in self._running.values() if s.owner_task_id == owner_task_id and not s.exited]
- def unread_completions_owned_by(self, owner_task_id: str) -> list[ProcessSession]:
- """Exited ``notify_on_complete`` processes of ``owner_task_id`` whose result nobody read (no wait/log/poll).
- A child's completion notice is suppressed in the parent, so an unread exit is otherwise lost silently."""
- with self._lock:
- return [s for s in self._finished.values()
- if s.owner_task_id == owner_task_id and s.notify_on_complete
- and s.exit_code is not None
- and s.id not in self._completion_consumed and s.id not in self._poll_observed]
- def transfer_ownership(self, session_id: str, *, from_owner: str, to_owner: str, to_task_id: str,
- to_session_key: str, note: str = "") -> Optional[ProcessSession]:
- """Move a RUNNING process from one owner to another under the registry lock. Ownership is the ``owner_task_id``
- field: completion notices are stamped from it at exit time and teardown kills by it, so flipping it here is the
- whole transfer. Returns the session, or None when it is unknown, already exited, or not owned by ``from_owner``
- (the caller must not report a transfer that did not happen)."""
- session = self.get(session_id)
- with self._lock:
- if session is None or session.exited or session.owner_task_id != from_owner:
- return None
- session.owner_task_id = to_owner
- session.task_id = to_task_id
- session.session_key = to_session_key
- session.handoff_note = note
- return session
- def has_active_for_session(self, session_key: str, max_active_age: Optional[float] = None) -> bool:
- """Active processes for a gateway session key. Processes older than
- ``max_active_age`` seconds are ignored as stale so a forgotten ``http.server``
- can't freeze session idle/daily reset forever; ``None`` keeps legacy behaviour
- (any running process blocks)."""
- now = time.time()
- return self._any_running(
- lambda s: s.session_key == session_key
- and (max_active_age is None or (now - s.started_at) < max_active_age))
- def has_any_active(self) -> bool:
- """Whether ANY background process is running — scale-to-zero must not
- suspend a gateway with live background work or the process is lost."""
- return self._any_running(lambda s: True)
- def snapshot_running_ids(self, task_id: str) -> frozenset[str]:
- """Running IDs owned by ``task_id`` — a turn-boundary marker: on timeout
- only processes absent from the starting snapshot belong to the abandoned
- turn; older ones intentionally span turns and must survive. Ownership is
- ``owner_task_id``: ``task_id`` is the container key (``session:<key>``,
- ``default``), shared across turns and sessions, not the turn's id."""
- return frozenset(s.id for s in self.running_owned_by(task_id))
- # Kill sources that are agent-lifecycle cleanup (session end / compression / error
- # recovery / turn abandon), not a deliberate stop: those must skip persist_on_release
- # sessions. An explicit stop (process_manage kill, CLI /stop) passes its own source
- # string and still reaches them.
- _LIFECYCLE_KILL_SOURCES = frozenset({"kill_all", "gateway_turn_timeout", "agent_close"})
- def kill_started_since(self, task_id: str, baseline_ids, *, source: str) -> int:
- """Kill ``task_id`` processes created after ``baseline_ids``. Output is
- consumed so an abandoned turn can't enqueue a follow-up reviving work the
- timeout deliberately stopped."""
- return self.kill_all(task_id, exclude_ids=frozenset(baseline_ids or ()), source=source, consume_output=True)
- def kill_all(
- self, task_id: Optional[str] = None, *, exclude_ids: frozenset = frozenset(),
- source: str = "kill_all", consume_output: bool = False) -> int:
- """Kill all running processes, optionally only those ``task_id`` spawned (its ``owner_task_id``).
- Returns how many it stopped. A session whose systemd scope did not stop is not counted
- (its descendants may live on), and a later sweep retries it even after it finished.
- Sessions with ``persist_on_release=True`` are skipped when ``source`` is an
- agent-lifecycle sweep (the default ``kill_all`` release path, a gateway turn
- timeout, or ``agent_close``): an explicitly persisted background job must survive
- session end / compression / error recovery (#41225). An explicit operator stop
- (``process.kill`` via process_manage, CLI /stop) passes its own source and still
- reaches them, so the user can always stop a persisted process on purpose."""
- lifecycle = source in self._LIFECYCLE_KILL_SOURCES
- with self._lock:
- targets = [
- s for s in (*self._running.values(), *self._finished.values())
- if (task_id is None or s.owner_task_id == task_id)
- and s.id not in exclude_ids and (not s.exited or s._scope_stop_pending)
- and not (lifecycle and s.persist_on_release)
- ]
- results = [self.kill_process(s.id, source=source, consume_output=consume_output) for s in targets]
- return sum(
- r.get("status") in {"killed", "already_exited"} and "scope_stop_failed" not in r
- for r in results)
- # ----- Cleanup / Pruning -----
- def _prune_if_needed(self):
- """Drop expired, then oldest-over-cap finished sessions (hold _lock); keep pending scope stops and unpublished notices."""
- from tools.process_registry_awaiting import retained_for_delivery
- now = time.time()
- prunable = {sid: s for sid, s in self._finished.items()
- if not getattr(s, "_scope_stop_pending", False) and not retained_for_delivery(s)}
- expired = [sid for sid, s in prunable.items() if (now - s.started_at) > FINISHED_TTL_SECONDS]
- over_cap = len(self._running) + len(self._finished) - len(expired) >= MAX_PROCESSES
- if over_cap and (survivors := [sid for sid in prunable if sid not in expired]):
- expired.append(min(survivors, key=lambda sid: self._finished[sid].started_at))
- for sid in expired:
- # Belt-and-suspenders handle release: sessions normally arrive in
- # _finished via _move_to_finished(), which already released their
- # Popen/PTY handles — but any session inserted into _finished
- # directly (defensive paths, historical checkpoints) would
- # otherwise carry its OS handles to the grave unreleased. The
- # release is idempotent, so double-closing is safe.
- self._release_finished_handles(self._finished[sid])
- del self._finished[sid]
- # Belt-and-suspenders against module-lifetime growth: forget consumed /
- # poll-observed marks for any session no longer tracked at all.
- tracked = self._running.keys() | self._finished.keys()
- self._completion_consumed &= tracked
- self._poll_observed &= tracked
- process_registry = ProcessRegistry()
- # --- the "process_manage" tool schema + handler -----------------------------------
- from tools.registry import registry, tool_error
- PROCESS_SCHEMA = {
- "name": "process_manage",
- # The enum names the verbs; the description keeps only non-obvious semantics
- # (write-vs-submit is the one real trap: a lone \n on a Windows PTY is not Enter).
- # See #95681.
- "description": (
- "Poll, wait on, send input to, or kill terminal(background=true) processes. "
- "Completed results remain retrievable by session_id when resuming their owning conversation "
- "(up to 7 days, newest 64 results per profile; rolling output tail). "
- "poll: status + new output. log: full output, paged. wait: block "
- "until exit or timeout (partial output on timeout). write vs "
- "submit: submit appends Enter — use it to answer prompts; write "
- "sends raw bytes, no newline. close: EOF stdin. kill: terminate. "
- "handoff (subagents only): transfer a running process you started to your parent agent, which then "
- "receives its completion; `data` = one sentence on its purpose. Subagent-owned processes are otherwise "
- "killed when the subagent finishes and their notifications never reach the parent."
- ),
- "parameters": {
- "type": "object",
- "properties": {
- "action": {
- "type": "string",
- "enum": ["list", "poll", "log", "wait", "kill", "write", "submit", "close", "handoff"]
- },
- "session_id": {
- "type": "string",
- "description": "From terminal background output; any unique prefix works ('4dae' for proc_4dae56ca81f6). Required except for 'list'."
- },
- "data": {
- "type": "string",
- "description": "Stdin text for write/submit; purpose sentence for handoff."
- },
- "timeout": {
- "type": "integer",
- "description": "Max seconds for 'wait'.",
- "minimum": 1
- },
- "offset": {
- "type": "integer",
- "description": "Log line offset (default: last 200)."
- },
- "limit": {
- "type": "integer",
- "description": "Max log lines.",
- "minimum": 1
- }
- },
- "required": ["action"]
- }
- }
- def transform_process_output(output: str, *, command: str, returncode: Optional[int], task_id: str = "") -> str:
- """``transform_terminal_output`` seam for background-process output — the poll/wait/log/kill
- results and the completion/heartbeat/watch notifications — so a plugin that rewrites terminal
- output sees the same command output whether it ran in the foreground or not (#70760).
- Same helper as the foreground path; callers redact AFTER it, never before, so a replacement
- the plugin returns is still masked. ``returncode`` is None while the process is running and
- ``env_type`` is not recorded per process, so it is passed empty."""
- from tools.terminal_tool_result import _apply_output_transform_hook
- return _apply_output_transform_hook(command, output, returncode, task_id or "", "")
- def _redact_process_result(result: dict) -> dict:
- """Transform, then redact secrets from background-process output before it reaches the
- model, session.db and CLI, mirroring the foreground ``terminal`` pipeline (hook first,
- redaction after) so the two surfaces can't diverge. Respects ``security.redact_secrets``;
- ``redact_terminal_output`` picks ``code_file`` from the recorded command.
- The command string itself is also redacted in case it carried an inline credential. See #43025.
- """
- if not isinstance(result, dict):
- return result
- from agent.redact import redact_sensitive_text, redact_terminal_output
- command = result.get("command") or ""
- # The hook's task_id is the process OWNER's (poll/log/wait results carry only session_id).
- task_id = str(result.get("task_id") or "")
- if not task_id and (session := process_registry.get(str(result.get("session_id") or ""))) is not None:
- task_id = str(getattr(session, "task_id", "") or "")
- for key in ("output", "output_preview"):
- if isinstance(value := result.get(key), str) and value:
- value = transform_process_output(value, command=command, returncode=result.get("exit_code"), task_id=task_id)
- result[key] = redact_terminal_output(value, command)
- if isinstance(command, str) and command:
- result["command"] = redact_sensitive_text(command, code_file=True)
- return result
- def _list_processes(task_id) -> dict:
- # Also surface session-scoped background processes (e.g. a forgotten preview
- # server): they share the gateway session_key and can block session reset.
- session_key = ""
- with suppress(Exception):
- # See #29177.
- from tools.approval_context import get_current_session_key
- session_key = get_current_session_key(default="") or ""
- return {"processes": [
- _redact_process_result(p)
- for p in process_registry.list_sessions(
- task_id=task_id, session_key=session_key or None, include_retained=True)]}
- # action -> (handler(session_id, args) -> dict, redact output?). Output-bearing
- # actions are redacted; stdin actions return only status.
- _SESSION_ACTIONS = {
- "poll": (lambda sid, a: process_registry.poll(sid), True),
- "log": (lambda sid, a: process_registry.read_log(sid, offset=a.get("offset"), limit=a.get("limit", 200)), True),
- "wait": (lambda sid, a: process_registry.wait(sid, timeout=a.get("timeout")), True),
- "kill": (lambda sid, a: process_registry.kill_process(sid), True),
- "write": (lambda sid, a: process_registry.write_stdin(sid, str(a.get("data", ""))), False),
- "submit": (lambda sid, a: process_registry.submit_stdin(sid, str(a.get("data", ""))), False),
- "close": (lambda sid, a: process_registry.close_stdin(sid), False),
- }
- def _handoff_process(session_id: str, args: dict, task_id: Optional[str]) -> dict:
- """Subagent-only: transfer a running background process to the parent agent so its completion is delivered THERE
- (child-owned process notices are suppressed and child teardown kills what it owns). Validated against the live spawn
- tree: the caller must be a registered child and must own the process; anything else is an error, never a silent
- no-op, so a PID mentioned in prose can't masquerade as a transfer."""
- from tools.delegate_tool_registry import _active_subagents, _active_subagents_lock
- from tools.terminal_tool import _resolve_container_task_id
- with _active_subagents_lock:
- record = _active_subagents.get(str(task_id or ""))
- child = record.get("agent") if record else None
- parent_ref = getattr(child, "_delegate_parent_ref", None)
- parent = parent_ref() if callable(parent_ref) else None
- if parent is None:
- return {"error": "handoff is only available to a running subagent with a live parent; you are not one."}
- parent_owner = str(getattr(parent, "_current_task_id", "") or getattr(parent, "session_id", "") or "")
- if not parent_owner:
- return {"error": "parent has no process owner id yet; retry after the parent's turn has started."}
- handed = getattr(child, "_handed_off_processes", None)
- if handed is None:
- handed = child._handed_off_processes = []
- if len(handed) >= _MAX_HANDOFFS_PER_CHILD:
- return {"error": f"handoff cap reached ({_MAX_HANDOFFS_PER_CHILD} per subagent); wait on or kill the rest yourself."}
- note = str(args.get("data") or "").strip()
- if not note:
- return {"error": "handoff requires `data`: one sentence saying what the process is for and what the parent should do with its result."}
- session = process_registry.transfer_ownership(
- session_id, from_owner=str(task_id or ""), to_owner=parent_owner,
- to_task_id=_resolve_container_task_id(parent_owner),
- to_session_key=str(getattr(parent, "session_id", "") or ""), note=note)
- if session is None:
- return {"error": f"cannot hand off {session_id}: not a running process you own (already exited? read its result "
- "with poll/log and report it instead)."}
- handed.append({"session_id": session.id, "command": session.command, "note": note})
- return {"status": "handed_off", "session_id": session.id, "command": session.command,
- "note": "Your parent now owns this process and will receive its completion; you will not. Mention the handoff "
- "in your final answer."}
- _MAX_HANDOFFS_PER_CHILD = 3
- def _handle_process(args, **kw):
- action = args.get("action", "")
- # Coerce to string — some models send session_id as an integer
- session_id = str(args.get("session_id", "")) if args.get("session_id") is not None else ""
- if action == "list":
- return json.dumps(_list_processes(kw.get("task_id")), ensure_ascii=False)
- if action == "handoff":
- if not session_id:
- return tool_error("session_id is required for handoff")
- return json.dumps(_handoff_process(session_id, args, kw.get("task_id")), ensure_ascii=False)
- if action in _SESSION_ACTIONS:
- if not session_id:
- return tool_error(f"session_id is required for {action}")
- handler, redact = _SESSION_ACTIONS[action]
- from tools.process_registry_wait_policy import apply_wait_poll_policy
- result = apply_wait_poll_policy(action, session_id, handler(session_id, args))
- return json.dumps(_redact_process_result(result) if redact else result, ensure_ascii=False)
- return tool_error(f"Unknown process action: {action}. Use: list, poll, log, wait, kill, write, submit, close, handoff")
- registry.register(
- name="process_manage",
- toolset="terminal",
- schema=PROCESS_SCHEMA,
- handler=_handle_process,
- emoji="⚙️",
- )
|