process_registry.py 146 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752
  1. """Process Registry -- in-memory registry for background processes spawned via
  2. terminal(background=true): rolling 200KB output buffer, poll/log/wait/kill, JSON
  3. checkpoint for crash recovery, session-scoped tracking for gateway reset protection.
  4. Nothing runs on the host unless TERMINAL_ENV=local; other backends run in their sandbox.
  5. """
  6. import codecs
  7. from contextlib import suppress
  8. import json
  9. import logging
  10. import os
  11. import platform
  12. import shlex
  13. import signal
  14. import stat
  15. import subprocess
  16. import tempfile
  17. import threading
  18. import time
  19. import uuid
  20. from pathlib import Path
  21. _IS_WINDOWS = platform.system() == "Windows"
  22. # systemd transient scopes exist only on Linux; gate every scope-path branch on this
  23. # (not merely "not Windows") so macOS and other POSIX platforms never touch systemd.
  24. # See #70716.
  25. _IS_LINUX = platform.system() == "Linux"
  26. from tools.environments.local import _append_dropped_path_entries, _find_shell, _resolve_safe_cwd, _sanitize_subprocess_env, _terminal_env
  27. from hermes_cli._subprocess_compat import windows_hide_flags
  28. from dataclasses import dataclass, field
  29. from typing import Any, Dict, List, Literal, NamedTuple, Optional
  30. from hermes_cli.config import get_hermes_home
  31. from tools.process_registry_notifications import (
  32. format_process_notification, heartbeat_output_tail, is_child_process_notification, should_surface_notification)
  33. from tools.process_registry_checkpoint import ProcessCheckpointMixin
  34. from tools.process_registry_output_log import ProcessOutputLogMixin, open_output_log, record_exit_command, worker_output_log
  35. from tools.process_registry_ownership import ProcessOwnershipMixin
  36. from tools.process_registry_termination import ProcessTerminationMixin
  37. from tools.process_registry_results import load_completed_results, save_completed_result
  38. from tools.process_registry_env_log import log_delta_command
  39. from tools.process_registry_wsl import _is_wsl_launcher_command
  40. logger = logging.getLogger(__name__)
  41. # Crash-recovery checkpoint (gateway only)
  42. CHECKPOINT_PATH = get_hermes_home() / "processes.json"
  43. _CHECKPOINT_PATH_AT_IMPORT = CHECKPOINT_PATH
  44. def _checkpoint_path() -> Path:
  45. """Active profile's checkpoint file at call time: the patched ``CHECKPOINT_PATH`` when a test
  46. changed it, else live profile-scoped HERMES_HOME — the multiplexed gateway serves every
  47. profile from one process, so the import-time constant would pin every profile's process
  48. checkpoint to the launch home."""
  49. return CHECKPOINT_PATH if CHECKPOINT_PATH != _CHECKPOINT_PATH_AT_IMPORT else get_hermes_home() / "processes.json"
  50. MAX_OUTPUT_CHARS = 200_000 # rolling output buffer
  51. # Tail of the output a completion notification carries. Right for a build log; a spawner whose
  52. # output IS the payload (a bot DM's reply) asks for more per process (completion_output_chars).
  53. COMPLETION_OUTPUT_CHARS = 2000
  54. FINISHED_TTL_SECONDS = 1800 # keep finished processes 30 minutes
  55. MAX_PROCESSES = 64 # max tracked processes (LRU pruning)
  56. # Watch-pattern rate limiting, PER SESSION: one watch-match notification per
  57. # WATCH_MIN_INTERVAL_SECONDS; a match inside the cooldown is dropped and counts as one
  58. # strike per window; WATCH_STRIKE_LIMIT consecutive strike windows permanently disable
  59. # watching and fall back to notify_on_complete semantics.
  60. WATCH_MIN_INTERVAL_SECONDS = 15
  61. WATCH_STRIKE_LIMIT = 3
  62. # Lifetime cap, independent of strikes: a pattern recurring just above the cooldown never
  63. # strikes yet forces a full-context agent turn each time; watch_patterns is "ONLY for
  64. # rare one-shot signals", so after this many deliveries fall back to notify_on_complete.
  65. # MAX_ACTIVE_PROCESS_AGE = 86400 # 24h default — see session_reset.bg_process_max_age_hours (#29177)
  66. # A process whose pattern recurs at a cadence just above WATCH_MIN_INTERVAL_SECONDS (e.g. a service
  67. # restarted repeatedly over a day) never trips the consecutive-strike limit, since each match lands in its
  68. # own clean cooldown window, yet still forces a full-context agent turn every single time (#93513).
  69. # watch_patterns is documented as "ONLY for rare one-shot mid-process signals", so once a session has
  70. # delivered this many matches over its whole life we disable it and fall back to notify_on_complete, same as
  71. # the strike-limit path.
  72. WATCH_LIFETIME_MAX_HITS = 8
  73. # Heartbeat: an opt-in periodic "still running, here is the output since last time" event for
  74. # long bounded jobs (merge trains, full test suites, deploys). Unlike watch patterns it is
  75. # time-driven, so it is bounded by construction (≤ 3600/HEARTBEAT_MIN_SECONDS events per hour
  76. # per process) and needs no strike/lifetime breaker. The floor exists so a model cannot turn
  77. # it into a 5-second poll; the output slice is capped like a completion notice.
  78. HEARTBEAT_MIN_SECONDS = 60
  79. HEARTBEAT_TICK_SECONDS = 5
  80. # Reader final drain after a reconcile saw the direct child exit: the exited child's unread tail is at
  81. # most one pipe buffer (Linux pipe-max-size default 1 MiB), and the reader publishes within one select
  82. # interval plus that drain.
  83. _FINAL_DRAIN_MAX_CHARS = 1 << 20
  84. _READER_FINAL_DRAIN_WAIT_SECONDS = 1.0
  85. # Global circuit breaker across all sessions so concurrent siblings can't collectively
  86. # flood the user even when each is under its own cap.
  87. WATCH_GLOBAL_MAX_PER_WINDOW = 15
  88. WATCH_GLOBAL_WINDOW_SECONDS = 10
  89. WATCH_GLOBAL_COOLDOWN_SECONDS = 30
  90. # --- systemd cgroup isolation for gateway-spawned local executors ------------------
  91. # Under a systemd gateway with MemoryMax, local background commands inherit the gateway's
  92. # cgroup, so a memory-heavy executor can get the ENTIRE gateway killed by systemd-oomd;
  93. # ``systemd-run --user --scope`` gives the worker its own transient cgroup. Usability is
  94. # probed and cached for a bounded TTL (binary present but user D-Bus absent in system services/containers).
  95. # A memory-heavy executor (Codex, tests, Node) can push the whole cgroup past MemoryMax and trigger
  96. # systemd-oomd to kill the ENTIRE gateway — taking down the messaging control plane and silently losing the
  97. # active turn. We probe whether ``systemd-run --user --scope`` is actually usable (the binary can
  98. # exist on the PATH while the user D-Bus session is unavailable — common for system services and
  99. # containers), and cache the verdict for a bounded TTL. See #70716.
  100. _SYSTEMD_SCOPE_AVAILABLE: Optional[bool] = None
  101. _SYSTEMD_SCOPE_PROBE_LOCK = threading.Lock()
  102. _SYSTEMD_SCOPE_PROBED_AT = 0.0
  103. # Both verdicts expire: the user bus can vanish after a True (session logout without linger,
  104. # #110803) and reappear after a False (linger enabled later, #104893).
  105. _SYSTEMD_SCOPE_PROBE_TTL_SECONDS = 60.0
  106. # systemd >= 254 expands ``$$``/``${X}`` in a ``--scope`` command line itself unless told not to;
  107. # older systemd-run rejects the option (and never expanded there), so the probe drops it on rejection.
  108. _SYSTEMD_RUN_NO_EXPAND = True
  109. _MIN_WORKER_MEMORY_MAX_BYTES = 64 * 1024 * 1024
  110. _DEFAULT_WORKER_MEMORY_MAX_BYTES = 1024 * 1024 * 1024
  111. _WORKER_MEMORY_MAX_CAP_BYTES = 4 * 1024 * 1024 * 1024
  112. def _worker_memory_max_bytes() -> int:
  113. """Finite per-worker cgroup limit that can never widen host risk.
  114. ``TERMINAL_LOCAL_MEMORY_MAX_MB`` is honored only when it *tightens* the safe
  115. bound (min of the gateway's cgroup-v2 ``memory.max`` and half of physical RAM,
  116. capped at 4 GiB), so an oversized override cannot exceed the enclosing slice.
  117. The proposed local-memory-guard environment override is honored when it tightens the safe bound, so this
  118. isolation composes with PR #57121 instead of inventing a second knob.
  119. """
  120. override_bound: Optional[int] = None
  121. override = os.getenv("TERMINAL_LOCAL_MEMORY_MAX_MB", "").strip()
  122. if override:
  123. try:
  124. parsed = int(override) * 1024 * 1024
  125. except ValueError:
  126. parsed = -1
  127. if parsed >= _MIN_WORKER_MEMORY_MAX_BYTES:
  128. override_bound = parsed
  129. else:
  130. logger.warning(
  131. "Ignoring invalid TERMINAL_LOCAL_MEMORY_MAX_MB=%r; "
  132. "expected an integer representing at least %d MiB",
  133. override, _MIN_WORKER_MEMORY_MAX_BYTES // (1024 * 1024))
  134. candidates: list[int] = []
  135. try:
  136. for line in Path("/proc/self/cgroup").read_text(encoding="utf-8").splitlines():
  137. if line.startswith("0::"):
  138. relative = line.partition("::")[2].lstrip("/")
  139. raw_limit = (
  140. Path("/sys/fs/cgroup") / relative / "memory.max"
  141. ).read_text(encoding="utf-8-sig").strip()
  142. if raw_limit.isdigit():
  143. cgroup_limit = int(raw_limit)
  144. if cgroup_limit >= _MIN_WORKER_MEMORY_MAX_BYTES:
  145. candidates.append(cgroup_limit)
  146. break
  147. except (OSError, ValueError):
  148. pass
  149. try:
  150. physical_bytes = int(os.sysconf("SC_PHYS_PAGES")) * int(
  151. os.sysconf("SC_PAGE_SIZE")
  152. )
  153. physical_bound = min(
  154. _WORKER_MEMORY_MAX_CAP_BYTES,
  155. max(_MIN_WORKER_MEMORY_MAX_BYTES, physical_bytes // 2),
  156. )
  157. candidates.append(physical_bound)
  158. except (OSError, ValueError, TypeError):
  159. pass
  160. safe_bound = min(candidates) if candidates else _DEFAULT_WORKER_MEMORY_MAX_BYTES
  161. return min(override_bound, safe_bound) if override_bound else safe_bound
  162. def _systemd_scope_argv(binary: str, unit_name: str, *argv: str) -> list[str]:
  163. """``systemd-run --user --scope`` argv shared by the probe and real spawns.
  164. ``--collect`` self-cleans the scope after exit; ``--unit`` names it for systemctl.
  165. No ``OOMPolicy=``: transient scopes reject it on systemd <253 (#102486).
  166. ``--expand-environment=no`` keeps the command byte-identical (#132385)."""
  167. no_expand = ["--expand-environment=no"] if _SYSTEMD_RUN_NO_EXPAND else []
  168. return [
  169. binary, "--user", "--scope", "--quiet", *no_expand, "--unit", unit_name, "--collect",
  170. "--property", "MemoryAccounting=yes",
  171. "--property", f"MemoryMax={_worker_memory_max_bytes()}",
  172. "--", *argv,
  173. ]
  174. def _default_user_runtime_dir() -> Path:
  175. """``/run/user/<uid>``; a function so tests can point it at a temp dir with a real socket."""
  176. return Path(f"/run/user/{os.getuid()}") # windows-footgun: ok — only reached behind the _IS_LINUX gate in systemd_user_bus_env
  177. def _secure_user_runtime_dir(path: Path) -> bool:
  178. """Accept only an absolute, owned, non-writable real directory."""
  179. try:
  180. metadata = path.lstat()
  181. return (
  182. path.is_absolute()
  183. and stat.S_ISDIR(metadata.st_mode)
  184. and metadata.st_uid == os.getuid() # windows-footgun: ok — only reached behind the _IS_LINUX gate in systemd_user_bus_env
  185. and metadata.st_mode & 0o022 == 0
  186. )
  187. except OSError:
  188. return False
  189. def systemd_user_bus_env(base_env: Optional[dict[str, str]] = None) -> dict[str, str]:
  190. """Build an environment that can reach this user's lingering systemd manager.
  191. System-level gateway units run as an unprivileged ``User=`` but normally do
  192. not inherit login-session variables. When the conventional runtime
  193. directory is owned by this uid and its bus exists, derive the two standard
  194. variables. Derived fresh on every call rather than adopted once at boot:
  195. linger may be enabled after the gateway started (existing installs), so
  196. the bus can appear later and the probe's failure TTL must be able to
  197. recover (#104893).
  198. The returned copy is passed explicitly to the probe and every scoped spawn;
  199. ``os.environ`` is left unchanged.
  200. """
  201. env = dict(os.environ if base_env is None else base_env)
  202. if not _IS_LINUX:
  203. return env
  204. configured = env.get("XDG_RUNTIME_DIR")
  205. if configured and _secure_user_runtime_dir(Path(configured)):
  206. runtime_dir = Path(configured)
  207. else:
  208. runtime_dir = _default_user_runtime_dir()
  209. if not _secure_user_runtime_dir(runtime_dir):
  210. return env
  211. bus_path = runtime_dir / "bus"
  212. try:
  213. bus_metadata = bus_path.lstat()
  214. except OSError:
  215. return env
  216. if not stat.S_ISSOCK(bus_metadata.st_mode) or bus_metadata.st_uid != os.getuid(): # windows-footgun: ok — behind the _IS_LINUX gate above
  217. return env
  218. env["XDG_RUNTIME_DIR"] = str(runtime_dir)
  219. env["DBUS_SESSION_BUS_ADDRESS"] = f"unix:path={bus_path}"
  220. return env
  221. def _systemd_scope_cached() -> Optional[bool]:
  222. """Cached probe verdict, or None when a (re)probe is due."""
  223. if _SYSTEMD_SCOPE_AVAILABLE is None:
  224. return None
  225. stale = time.monotonic() - _SYSTEMD_SCOPE_PROBED_AT >= _SYSTEMD_SCOPE_PROBE_TTL_SECONDS
  226. return None if stale else _SYSTEMD_SCOPE_AVAILABLE
  227. def _systemd_run_user_scope_available() -> bool:
  228. """True if ``systemd-run --user --scope`` can create a cgroup.
  229. ``shutil.which`` alone is insufficient: system services and containers may lack
  230. the user D-Bus bus even with the binary on PATH (every spawn would fail with
  231. ``Failed to connect to user bus``), so a cheap probe is run and cached.
  232. Use ``/bin/sh -c 'exit 0'``: NixOS provides ``/bin/sh`` but not ``/bin/true``
  233. (#105365), regardless of the gateway service's PATH."""
  234. global _SYSTEMD_SCOPE_AVAILABLE, _SYSTEMD_SCOPE_PROBED_AT, _SYSTEMD_RUN_NO_EXPAND
  235. verdict = _systemd_scope_cached()
  236. if verdict is not None:
  237. return verdict
  238. # Double-checked locking: a concurrent first-use spawn must not observe a temporary
  239. # False mid-probe, or it would launch back inside the gateway cgroup.
  240. with _SYSTEMD_SCOPE_PROBE_LOCK:
  241. verdict = _systemd_scope_cached()
  242. if verdict is not None:
  243. return verdict
  244. available = False
  245. if _IS_LINUX:
  246. try:
  247. import shutil
  248. binary = shutil.which("systemd-run")
  249. if binary:
  250. # Unique unit avoids collisions; the timeout bounds D-Bus.
  251. probe_unit = f"hermes-probe-scope-{os.getpid()}-{uuid.uuid4().hex[:8]}"
  252. for _attempt in range(2):
  253. result = subprocess.run(
  254. _systemd_scope_argv(binary, probe_unit, "/bin/sh", "-c", "exit 0"),
  255. stdin=subprocess.DEVNULL, capture_output=True,
  256. timeout=3,
  257. env=systemd_user_bus_env(),
  258. check=False,
  259. )
  260. if not (result.returncode and _SYSTEMD_RUN_NO_EXPAND
  261. and b"expand-environment" in (result.stderr or b"")):
  262. break
  263. # systemd < 254 rejects the option: drop it and probe again.
  264. _SYSTEMD_RUN_NO_EXPAND = False
  265. available = result.returncode == 0
  266. if not available:
  267. logger.debug(
  268. "systemd-run --user --scope probe failed (rc=%s): %s",
  269. result.returncode, (result.stderr or b"").decode("utf-8", "replace").strip(),
  270. )
  271. except Exception as exc:
  272. logger.debug("systemd-run --user --scope probe error: %s", exc)
  273. _SYSTEMD_SCOPE_AVAILABLE = available
  274. _SYSTEMD_SCOPE_PROBED_AT = time.monotonic()
  275. return available
  276. def _is_supervised_gateway_process() -> bool:
  277. """Whether this process is the live, supervised Hermes gateway itself.
  278. Supervisor markers and ``_HERMES_GATEWAY`` are inherited by every descendant (and
  279. importing ``gateway.run`` sets the latter), so also require ownership of the live
  280. gateway PID file — scopes are for the gateway, not terminal children or CLIs.
  281. Reads the launch marker (``HERMES_SUPERVISED_CHILD`` included), not the restart-route
  282. probe: a Windows Scheduled-Task gateway sets only that marker, and the self-kill guards
  283. gated here must protect it too (#113667)."""
  284. if os.environ.get("_HERMES_GATEWAY") != "1":
  285. return False
  286. try:
  287. from gateway.restart import is_supervised_gateway_launch
  288. from gateway.status import get_running_pid
  289. return is_supervised_gateway_launch() and get_running_pid(cleanup_stale=False) == os.getpid()
  290. except Exception as exc:
  291. logger.debug("Could not verify supervised gateway process identity: %s", exc)
  292. return False
  293. def _build_systemd_scope_argv(shell_argv: list[str], unit_suffix: str) -> list[str]:
  294. """Wrap *shell_argv* in a ``systemd-run --user --scope`` invocation with its own
  295. memory accounting, so an OOM in the worker cannot kill the gateway cgroup.
  296. ``--collect`` makes the transient scope self-clean after exit; ``--unit`` gives it a recognisable name
  297. for ``systemctl --user status`` / journalctl. See #70716.
  298. """
  299. import shutil
  300. binary = shutil.which("systemd-run")
  301. if binary is None:
  302. # Caller should have probed availability; never pass None into Popen anyway.
  303. return shell_argv
  304. return _systemd_scope_argv(binary, f"hermes-worker-{unit_suffix}", *shell_argv)
  305. _scope_degraded_warned = False
  306. class RestartSafeScopeUnavailable(RuntimeError):
  307. """A ``require_restart_safe_scope=True`` child could not get its transient scope.
  308. Host-level (no user D-Bus / no ``systemd-run``), never a property of the
  309. child being launched: callers that keep per-job retry budgets must not
  310. charge it (a kanban card parked ``blocked`` for an unreachable bus, #114720).
  311. """
  312. def _warn_scope_degraded_once(detail: str, *, consequence: str) -> None:
  313. """Warn once per process: the condition is host-level and the probe verdict
  314. is cached, so this would otherwise fire on every cron dispatch."""
  315. global _scope_degraded_warned
  316. if _scope_degraded_warned:
  317. return
  318. _scope_degraded_warned = True
  319. logger.warning("%s; %s", detail, consequence)
  320. _CRON_DEGRADED_CONSEQUENCE = (
  321. "cron children are dispatched as direct external subprocesses without restart-safe "
  322. "cgroup isolation (killed if the gateway restarts mid-job). Set "
  323. "cron.require_restart_safe_scope=true in config.yaml to fail closed instead."
  324. )
  325. _UNIT_DEGRADED_CONSEQUENCE = (
  326. "workers are spawned unmanaged inside this systemd unit's cgroup and will be KILLED when the "
  327. "unit exits (Type=oneshot dispatch timers lose every worker within a second). Give the unit's "
  328. "user a session bus (`loginctl enable-linger <user>`) so workers get their own scope, or set "
  329. "KillMode=process on the unit."
  330. )
  331. class GatewayChildDispatch(NamedTuple):
  332. """How a managed-gateway child is launched.
  333. ``in_process``: not a managed systemd gateway, ``argv is command``, the caller
  334. keeps its in-process path. ``scoped``: ``argv`` is the systemd-run wrapper.
  335. ``degraded``: no user scope could be created; ``argv`` is the direct command but
  336. the caller MUST still launch it as an external subprocess — the distinct mode
  337. exists so this case can never collapse into ``in_process`` and recreate the
  338. restart interruption #101940 closed.
  339. """
  340. mode: Literal["in_process", "scoped", "degraded"]
  341. argv: list[str]
  342. def scoped_spawn_lost_user_bus(spawn_env: dict[str, str]) -> bool:
  343. """After a ``systemd-run --user --scope`` wrapper exits before its child could start: True
  344. when the user bus is gone (:func:`systemd_user_bus_env` derives nothing), in which case the
  345. cached True verdict is replaced so the next dispatch re-probes and degrades instead of
  346. consuming another occurrence on the same dead wrapper (#110803).
  347. *spawn_env* is the environment the wrapper was launched with: re-deriving from it (minus the
  348. bus address it carried) honours a configured ``XDG_RUNTIME_DIR`` exactly as the spawn did, so
  349. an unrelated wrapper exit on a host whose bus lives outside ``/run/user/<uid>`` is not
  350. misread as a lost bus."""
  351. global _SYSTEMD_SCOPE_AVAILABLE, _SYSTEMD_SCOPE_PROBED_AT
  352. base_env = dict(spawn_env)
  353. base_env.pop("DBUS_SESSION_BUS_ADDRESS", None)
  354. if "DBUS_SESSION_BUS_ADDRESS" in systemd_user_bus_env(base_env):
  355. return False
  356. with _SYSTEMD_SCOPE_PROBE_LOCK:
  357. _SYSTEMD_SCOPE_AVAILABLE = False
  358. _SYSTEMD_SCOPE_PROBED_AT = time.monotonic()
  359. return True
  360. def restart_safe_gateway_child_argv(
  361. command: list[str], *, unit_suffix: str, require_restart_safe_scope: bool,
  362. outlives_parent: bool = False,
  363. ) -> GatewayChildDispatch:
  364. """Place a managed-systemd gateway child outside the gateway cgroup.
  365. A systemd-supervised gateway restart kills every process in the service
  366. cgroup, so children that must survive it run in a transient user scope.
  367. Hosts with no user systemd session (containers, LXCs without linger) cannot
  368. create one; hard-failing there is a silent cron outage, so callers state the
  369. policy: ``require_restart_safe_scope=True`` raises
  370. :class:`RestartSafeScopeUnavailable` (kanban's long-lived workers), ``False``
  371. degrades to a direct external subprocess with a once-per-process warning
  372. (cron, behind ``cron.require_restart_safe_scope``).
  373. ``outlives_parent=True`` (fire-and-forget kanban workers): any *other*
  374. systemd unit — a ``Type=oneshot`` dispatch timer, an operator's sequencer
  375. service — tears its cgroup down when it exits, so the child is scope-wrapped
  376. there too (#113612). Whether that unit actually kills its children
  377. (``KillMode``, lifetime) is not knowable here, so without a user bus it
  378. degrades with a loud warning instead of refusing: a long-lived
  379. ``Type=simple`` sequencer without linger keeps working. A cron job blocks its
  380. caller until it finishes and never needs this.
  381. """
  382. if not _IS_LINUX:
  383. return GatewayChildDispatch("in_process", command)
  384. if not os.environ.get("INVOCATION_ID"):
  385. return GatewayChildDispatch("in_process", command)
  386. supervised_gateway = _is_supervised_gateway_process()
  387. if not supervised_gateway and not outlives_parent:
  388. return GatewayChildDispatch("in_process", command)
  389. def _degrade(detail: str) -> GatewayChildDispatch:
  390. if supervised_gateway:
  391. if require_restart_safe_scope:
  392. # Stored as the cron execution's error and shown on the job row: name the remedy.
  393. raise RestartSafeScopeUnavailable(
  394. f"cannot create restart-safe systemd scope for gateway child: {detail}"
  395. )
  396. _warn_scope_degraded_once(f"managed gateway: {detail}", consequence=_CRON_DEGRADED_CONSEQUENCE)
  397. else:
  398. _warn_scope_degraded_once(f"systemd unit dispatch: {detail}", consequence=_UNIT_DEGRADED_CONSEQUENCE)
  399. return GatewayChildDispatch("degraded", command)
  400. if not _systemd_run_user_scope_available():
  401. return _degrade(
  402. "systemd-run --user --scope is unavailable (usually no reachable user D-Bus session at "
  403. f"/run/user/{os.getuid()}/bus). On a system-level service install, run " # windows-footgun: ok — behind the _IS_LINUX return above
  404. "`sudo loginctl enable-linger <gateway-user>` and restart the gateway."
  405. )
  406. scoped = _build_systemd_scope_argv(command, unit_suffix=unit_suffix)
  407. if scoped == command:
  408. return _degrade("systemd-run disappeared after the availability probe")
  409. return GatewayChildDispatch("scoped", scoped)
  410. def _stop_systemd_unit(unit_name: str) -> bool:
  411. """Stop a transient systemd user scope by unit name.
  412. Reaps the *entire* cgroup — catching double-forked descendants reparented to init
  413. inside the scope that survive a plain PID signal (SIGTERM all, SIGKILL after
  414. ``TimeoutStopSec``). True if stopped or already gone; False if ``systemctl`` is
  415. unavailable or the stop failed.
  416. See #70716.
  417. """
  418. import shutil
  419. binary = shutil.which("systemctl")
  420. if binary is None:
  421. return False
  422. try:
  423. result = subprocess.run(
  424. [binary, "--user", "stop", unit_name],
  425. capture_output=True,
  426. timeout=15,
  427. stdin=subprocess.DEVNULL,
  428. env=systemd_user_bus_env(),
  429. check=False,
  430. )
  431. if result.returncode != 0:
  432. stderr = (result.stderr or b"").decode(errors="replace").strip()
  433. if any(marker in stderr.lower() for marker in ("not loaded", "not found", "does not exist")):
  434. return True
  435. logger.debug("systemctl --user stop %s exited %d: %s", unit_name, result.returncode, stderr)
  436. return False
  437. return True
  438. except Exception as exc:
  439. logger.debug("systemctl --user stop %s failed: %s", unit_name, exc)
  440. return False
  441. def format_uptime_short(seconds: int) -> str:
  442. s = max(0, int(seconds))
  443. if s < 60:
  444. return f"{s}s"
  445. mins, secs = divmod(s, 60)
  446. if mins < 60:
  447. return f"{mins}m {secs}s"
  448. hours, mins = divmod(mins, 60)
  449. return f"{hours}h {mins}m"
  450. def _not_found(session_id: str) -> dict:
  451. return {"status": "not_found", "error": f"No process with ID {session_id}"}
  452. def _output_tail(session: "ProcessSession", n: int) -> str:
  453. """Last *n* chars of the session output with ANSI sequences stripped."""
  454. from tools.ansi_strip import strip_ansi
  455. return strip_ansi(session.output_buffer[-n:])
  456. def _completion_output(session: "ProcessSession") -> dict:
  457. """``output`` sized by the session's ``completion_output_chars`` plus ``output_cut`` when trimmed.
  458. Shared by the completion notification AND the wait/poll/kill snapshots: a bot in an api_server
  459. or one-shot session cannot receive notifications and polls instead, so the polled result must
  460. carry the same whole reply and the same cut marker (#115334)."""
  461. limit = session.completion_output_chars or COMPLETION_OUTPUT_CHARS
  462. cut = len(session.output_buffer) - limit
  463. return {"output": _output_tail(session, limit), **({"output_cut": cut} if cut > 0 else {})}
  464. @dataclass
  465. class ProcessSession:
  466. """A tracked background process with output buffering."""
  467. id: str # "proc_xxxxxxxxxxxx"
  468. command: str
  469. task_id: str = "" # Task/sandbox isolation key (CONTAINER key,
  470. # may be collapsed by _resolve_container_task_id)
  471. owner_task_id: str = "" # RAW spawning task id ("sa-..."); ownership
  472. # checks must use this, not task_id
  473. session_key: str = "" # Gateway session key (reset protection)
  474. pid: Optional[int] = None
  475. process: Optional[subprocess.Popen] = None # Popen handle (local only)
  476. env_ref: Any = None # Environment object (sandbox spawns)
  477. cwd: Optional[str] = None
  478. started_at: float = 0.0 # time.time() of spawn
  479. host_start_time: Optional[int] = None # kernel start ticks (/proc/<pid>/stat f22) — PID-reuse guard
  480. exited: bool = False
  481. exited_at: float = 0.0 # time.time() of the FIRST move to finished (0 = unknown)
  482. exit_code: Optional[int] = None # None while running
  483. completion_reason: str = "exited" # exited|killed|lost|failed_start|already_exited
  484. termination_source: str = "" # process.kill|kill_all|backend_lost|failed_start
  485. output_buffer: str = "" # Rolling tail (last max_output_chars)
  486. max_output_chars: int = MAX_OUTPUT_CHARS
  487. detached: bool = False # Recovered from checkpoint (no pipe)
  488. pid_scope: str = "host" # "host" for local/PTY PIDs, "sandbox" for env-local PIDs
  489. systemd_unit: str = "" # transient scope unit name when spawned under systemd-run
  490. wsl_chain: bool = False # spawned via wsl[.exe]: the host PID is the short-lived
  491. # launcher; Linux-side workers outlive it (#120546)
  492. handoff_note: str = "" # why a subagent handed this process to its parent (rides the notice)
  493. persist_on_release: bool = False # opt out of agent-lifecycle cleanup (release()/turn-abandon kill
  494. # sweeps), per terminal(background=true, persist_on_release=true) (#41225)
  495. output_log: str = "" # file the process writes to (worker spawns; no pipe to lose)
  496. pty_fallback: str = "" # why a pty=true local spawn runs in pipe mode instead
  497. # Watcher/notification routing (persisted for crash recovery)
  498. watcher_platform: str = ""
  499. watcher_chat_id: str = ""
  500. watcher_user_id: str = ""
  501. watcher_user_name: str = ""
  502. watcher_thread_id: str = ""
  503. watcher_message_id: str = "" # Triggering message id — reply anchor for topic routing
  504. watcher_interval: int = 0 # 0 = no watcher configured
  505. # Session-db id of the spawning conversation; lets the gateway drop completions whose
  506. # session was closed at a user boundary (/new) instead of injecting into the NEW one.
  507. parent_session_id: str = ""
  508. notify_on_complete: bool = False # Queue agent notification on exit
  509. completion_output_chars: int = 0 # Output chars the completion carries; 0 = COMPLETION_OUTPUT_CHARS
  510. watch_patterns: list[str] = field(default_factory=list)
  511. heartbeat_seconds: int = 0 # 0 = off; else a "heartbeat" event every N s while running
  512. total_output_chars: int = 0 # Chars ever ingested (the buffer is a rolling tail)
  513. _heartbeat_last: float = field(default=0.0, repr=False) # time of the last heartbeat (or spawn)
  514. _heartbeat_total_at_last: int = field(default=0, repr=False) # total_output_chars at that moment
  515. _heartbeat_seq: int = field(default=0, repr=False)
  516. _watch_hits: int = field(default=0, repr=False) # total matches delivered
  517. _watch_suppressed: int = field(default=0, repr=False) # matches dropped by rate limit
  518. _watch_disabled: bool = field(default=False, repr=False) # permanently killed after strike limit
  519. # Rate-limit window state (see WATCH_*). A strike is a WINDOW with drops, not a drop.
  520. _watch_cooldown_until: float = field(default=0.0, repr=False)
  521. _watch_strike_candidate: bool = field(default=False, repr=False)
  522. _watch_consecutive_strikes: int = field(default=0, repr=False)
  523. _completion_event: threading.Event = field(default_factory=threading.Event, repr=False)
  524. # Set by the _move_to_finished call that moved the session out of _running; only that call
  525. # publishes the completion, so a duplicate finisher must not release waiters early.
  526. _finish_claimed: bool = field(default=False, repr=False)
  527. # The last stop of ``systemd_unit`` failed, so its descendants may still be alive; bulk
  528. # cleanup retries the stop even after the session finished.
  529. _scope_stop_pending: bool = field(default=False, repr=False)
  530. _lock: threading.Lock = field(default_factory=threading.Lock)
  531. _reader_thread: Optional[threading.Thread] = field(default=None, repr=False)
  532. _reader_finish_requested: threading.Event = field(default_factory=threading.Event, repr=False)
  533. _reader_selectable: bool = field(default=False, repr=False)
  534. _pty: Any = field(default=None, repr=False) # ptyprocess handle (use_pty=True)
  535. _owned_pgid: Optional[int] = field(default=None, repr=False) # see ProcessOwnershipMixin
  536. def __post_init__(self):
  537. # A session built without an explicit owner is owned by its own task, so ownership checks compare
  538. # ``owner_task_id`` alone instead of repeating an ``or task_id`` fallback at every call site.
  539. if not self.owner_task_id:
  540. self.owner_task_id = self.task_id
  541. def append_output(self, text: str) -> None:
  542. """Append to the rolling output buffer under the session lock, keeping the tail."""
  543. with self._lock:
  544. self._append_locked(text)
  545. def append_output_if_running(self, text: str) -> bool:
  546. """Append unless the session has exited. Decided under the lock a kill holds while it
  547. snapshots the output and sets ``exited``, so a chunk is either in the kill's receipt or
  548. dropped, never added after it."""
  549. with self._lock:
  550. if self.exited:
  551. return False
  552. self._append_locked(text)
  553. return True
  554. def _append_locked(self, text: str) -> None:
  555. self.output_buffer += text
  556. self.total_output_chars += len(text)
  557. if len(self.output_buffer) > self.max_output_chars:
  558. self.output_buffer = self.output_buffer[-self.max_output_chars:]
  559. def mark_exited(self, exit_code, reason: str = "exited", source: str = "") -> None:
  560. """Record an exit. A kill that raced the observer already recorded its own
  561. exit_code/reason; never overwrite it."""
  562. self.exited = True
  563. if self.completion_reason != "killed":
  564. self.exit_code = exit_code
  565. self.completion_reason = reason
  566. if source:
  567. self.termination_source = source
  568. # Watcher routing fields, in event-dict key order (``watcher_<key>`` on the session).
  569. _WATCHER_ROUTE_KEYS = ("platform", "chat_id", "user_id", "user_name", "thread_id", "message_id")
  570. # Session fields persisted verbatim in the crash-recovery checkpoint (plus
  571. # ``session_id``; ``command`` is redacted and ``owner_task_id`` defaulted on write).
  572. _CHECKPOINT_FIELDS = (
  573. "command", "pid", "pid_scope", "host_start_time", "systemd_unit", "wsl_chain", "cwd",
  574. "started_at", "task_id", "owner_task_id", "session_key",
  575. *(f"watcher_{k}" for k in _WATCHER_ROUTE_KEYS), "watcher_interval",
  576. "parent_session_id", "notify_on_complete", "completion_output_chars", "watch_patterns",
  577. "heartbeat_seconds", "persist_on_release", "output_log")
  578. _CHECKPOINT_DEFAULTS = {
  579. f.name: ([] if f.name == "watch_patterns" else f.default)
  580. for f in ProcessSession.__dataclass_fields__.values()
  581. if f.name in _CHECKPOINT_FIELDS
  582. }
  583. _WSL_CHAIN_NOTE = (
  584. "Spawned via a wsl[.exe] launcher: the recorded host PID is the short-lived "
  585. "launcher, not the Linux-side workers. Inspect them with `wsl -e ps` / "
  586. "`wsl --list --running` from the host."
  587. )
  588. class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOutputLogMixin, ProcessOwnershipMixin):
  589. """In-memory registry of running and finished background processes.
  590. Thread-safe: accessed from executor threads (terminal_tool, process handlers),
  591. the gateway asyncio loop (watchers, reset checks) and the cleanup thread."""
  592. _SHELL_NOISE_SUBSTRINGS = (
  593. "no job control in this shell", "cannot set terminal process group",
  594. "tcsetattr: Inappropriate ioctl for device")
  595. # Class default so registries built via __new__ (tests) still restore on first drain.
  596. _completions_restored = False
  597. def __init__(self):
  598. self._running: dict[str, ProcessSession] = {}
  599. self._finished: dict[str, ProcessSession] = {}
  600. self._lock = threading.Lock()
  601. # Side-channel for check_interval watchers (gateway reads after agent run)
  602. self.pending_watchers: list[dict[str, Any]] = []
  603. # Unified queue for all background events (distinguished by "type"); the CLI
  604. # process_loop and the gateway drain it after each agent turn to trigger new turns.
  605. import queue as _queue_mod
  606. self.completion_queue: _queue_mod.Queue = _queue_mod.Queue()
  607. # Durable delegation completions are NOT rehydrated here: this constructor runs at
  608. # import time in every process that touches the tools graph (cron clients, `hermes
  609. # sessions list`, workers), and restoring means writing the canonical ledger. The
  610. # owner replays them explicitly (GatewayRunner._start_recover_previous_run) and the
  611. # first consumer through restore_completions() (#123265). Importing the module is
  612. # side-effect free and keeps its import-order contract for later completion writers.
  613. import tools.async_delegation
  614. # Completions the agent already consumed via wait()/read_log() (output in
  615. # hand): drain loops AND gateway/tui watchers skip them.
  616. self._completion_consumed: set = set()
  617. # Sessions merely *observed* exited via poll(). poll() is read-only and must NOT
  618. # mark consumed (a status check would suppress the watcher's autonomous delivery
  619. # turn), but the CLI has the poll result inline in the same turn, so
  620. # drain_notifications() skips these to avoid a duplicate [SYSTEM: ...];
  621. # gateway/tui watchers deliberately ignore this set.
  622. # See #8228.
  623. self._poll_observed: set = set()
  624. # Global watch-match circuit breaker across all sessions.
  625. self._global_watch_lock = threading.Lock()
  626. self._global_watch_window_start = self._global_watch_tripped_until = 0.0
  627. self._global_watch_window_hits = self._global_watch_suppressed_during_trip = 0
  628. # Driver-installed sinks (desktop gateway): on_output(session, chunk) streams
  629. # live output from reader threads; on_close(session_or_none, process_id) drops
  630. # a read-only terminal tab without killing the process.
  631. self.on_output = None
  632. self.on_close = None
  633. self._heartbeat_thread: Optional[threading.Thread] = None
  634. # ── heartbeat ───────────────────────────────────────────────────────────
  635. def arm_heartbeat(self, session: ProcessSession, seconds: int) -> int:
  636. """Enable periodic heartbeat events for ``session``; returns the effective interval."""
  637. seconds = max(int(seconds), HEARTBEAT_MIN_SECONDS)
  638. session.heartbeat_seconds = seconds
  639. session._heartbeat_last = time.time()
  640. # The output baseline stays at spawn (field default 0), never here: the spawn call
  641. # arms the heartbeat only after its bookkeeping, and a fast-starting process has
  642. # already written its first lines by then. Those lines belong to the first heartbeat.
  643. self._ensure_heartbeat_thread()
  644. return seconds
  645. def _ensure_heartbeat_thread(self) -> None:
  646. with self._lock:
  647. if self._heartbeat_thread is not None and self._heartbeat_thread.is_alive():
  648. return
  649. self._heartbeat_thread = threading.Thread(
  650. target=self._heartbeat_loop, name="process-heartbeat", daemon=True)
  651. self._heartbeat_thread.start()
  652. def _heartbeat_loop(self) -> None:
  653. """One daemon thread for every heartbeat session: reader threads block on the pipe and
  654. cannot keep time, and a per-process timer would leak one thread per job."""
  655. while True:
  656. time.sleep(HEARTBEAT_TICK_SECONDS)
  657. now = time.time()
  658. with self._lock:
  659. due = [s for s in self._running.values()
  660. if s.heartbeat_seconds > 0 and not s.exited
  661. and now - s._heartbeat_last >= s.heartbeat_seconds]
  662. for session in due:
  663. self._emit_heartbeat(session, now)
  664. def _emit_heartbeat(self, session: ProcessSession, now: float) -> None:
  665. """Queue a heartbeat carrying the output since the last one. A tick with nothing new is
  666. skipped outright: every queued event costs the owner a full model turn, and "still running,
  667. no output" is already visible on the process surfaces (status stack, /agents dock)."""
  668. session._heartbeat_last = now
  669. with session._lock:
  670. delta = session.total_output_chars - session._heartbeat_total_at_last
  671. output = session.output_buffer[-delta:] if delta > 0 else ""
  672. session._heartbeat_total_at_last = session.total_output_chars
  673. if not output:
  674. return
  675. output, omitted = heartbeat_output_tail(output)
  676. session._heartbeat_seq += 1
  677. notification = {
  678. **self._watch_event_base(session),
  679. "type": "heartbeat",
  680. "seq": session._heartbeat_seq,
  681. "interval": session.heartbeat_seconds,
  682. "elapsed": int(now - session.started_at) if session.started_at else 0,
  683. "output": output,
  684. "omitted": omitted,
  685. "started_at": session.started_at,
  686. }
  687. _redact_process_result(notification)
  688. self.completion_queue.put(notification)
  689. @staticmethod
  690. def _clean_shell_noise(text: str) -> str:
  691. """Strip shell startup warnings from the beginning of output."""
  692. lines = text.split("\n")
  693. while lines and any(noise in lines[0] for noise in ProcessRegistry._SHELL_NOISE_SUBSTRINGS):
  694. lines.pop(0)
  695. return "\n".join(lines)
  696. def _emit_output(self, session: ProcessSession, chunk: str) -> None:
  697. """Forward a chunk to the live-output sink; called from reader threads, never raises."""
  698. sink = self.on_output
  699. if sink is None or not chunk:
  700. return
  701. with suppress(Exception):
  702. sink(session, chunk)
  703. def _check_watch_patterns(self, session: ProcessSession, new_text: str) -> None:
  704. """Scan a freshly-read chunk for watch patterns and queue notifications.
  705. Per-session rate limiting (see WATCH_* constants): one match per cooldown
  706. window, a match inside the window is one strike, WATCH_STRIKE_LIMIT consecutive
  707. strikes or WATCH_LIFETIME_MAX_HITS total deliveries disable watching and
  708. promote the session to notify_on_complete."""
  709. if not session.watch_patterns or session._watch_disabled:
  710. return
  711. # Late chunks after the reader declared exit are post-exit noise; dropping them
  712. # avoids stale notifications minutes after the process ended.
  713. if session.exited:
  714. return
  715. hits = [ # (first matching pattern, line) — one match per line
  716. (next(p for p in session.watch_patterns if p in line), line.rstrip())
  717. for line in new_text.splitlines() if any(p in line for p in session.watch_patterns)]
  718. if not hits:
  719. return
  720. matched_pattern = hits[0][0]
  721. matched_lines = [line for _, line in hits]
  722. now = time.time()
  723. with session._lock:
  724. if session._watch_cooldown_until and now < session._watch_cooldown_until:
  725. # Inside the cooldown: drop, count one strike per window, disable +
  726. # promote once the strike limit is hit.
  727. session._watch_suppressed += len(matched_lines)
  728. if session._watch_strike_candidate:
  729. return
  730. session._watch_strike_candidate = True
  731. session._watch_consecutive_strikes += 1
  732. if session._watch_consecutive_strikes < WATCH_STRIKE_LIMIT:
  733. return
  734. session._watch_disabled = True
  735. # Promote so the agent still gets exactly one notification on exit,
  736. # plus exactly one summary so it sees why things went quiet.
  737. session.notify_on_complete = True
  738. self._emit_watch_disabled(
  739. session, session._watch_suppressed,
  740. f"{WATCH_STRIKE_LIMIT} consecutive rate-limit windows triggered "
  741. f"(min spacing {WATCH_MIN_INTERVAL_SECONDS}s). ")
  742. return
  743. # Cooldown expired. A prior window with no drops resets the
  744. # consecutive-strike counter (healthy cadence again).
  745. if session._watch_cooldown_until and not session._watch_strike_candidate:
  746. session._watch_consecutive_strikes = 0
  747. session._watch_strike_candidate = False
  748. # Emit and start a new cooldown window.
  749. session._watch_cooldown_until = now + WATCH_MIN_INTERVAL_SECONDS
  750. session._watch_hits += 1
  751. suppressed = session._watch_suppressed
  752. session._watch_suppressed = 0
  753. # Lifetime cap: this match is still delivered, but no further ones.
  754. lifetime_exhausted = session._watch_hits >= WATCH_LIFETIME_MAX_HITS
  755. if lifetime_exhausted:
  756. session._watch_disabled = True
  757. session.notify_on_complete = True
  758. output = "\n".join(matched_lines[:20])
  759. if len(output) > 2000:
  760. output = output[:2000] + "\n...(truncated)"
  761. if self._global_watch_admit(now):
  762. notification = {
  763. **self._watch_event_base(session),
  764. "type": "watch_match",
  765. "pattern": matched_pattern,
  766. "output": output,
  767. "suppressed": suppressed,
  768. }
  769. _redact_process_result(notification)
  770. self.completion_queue.put(notification)
  771. # Even when the breaker drops the final match, still explain the silence.
  772. if lifetime_exhausted:
  773. self._emit_watch_disabled(
  774. session, 0, f"reached the lifetime cap of {WATCH_LIFETIME_MAX_HITS} delivered matches. ",
  775. )
  776. def _emit_watch_disabled(self, session: ProcessSession, suppressed: int, why: str) -> None:
  777. """Queue the one-shot watch_disabled summary (strike-limit or lifetime-cap path)."""
  778. self.completion_queue.put({
  779. **self._watch_event_base(session),
  780. "type": "watch_disabled",
  781. "suppressed": suppressed,
  782. "message": (
  783. f"Watch patterns disabled for process {session.id} — {why}"
  784. f"Falling back to notify_on_complete semantics; you'll get "
  785. f"exactly one notification when the process exits."),
  786. })
  787. @staticmethod
  788. def _watch_event_base(session: ProcessSession) -> dict:
  789. """Session identity + watcher routing fields shared by every watch event."""
  790. return {
  791. "event_id": uuid.uuid4().hex,
  792. "started_at": session.started_at,
  793. "parent_session_id": session.parent_session_id,
  794. "session_id": session.id,
  795. "session_key": session.session_key,
  796. "task_id": session.task_id,
  797. "owner_task_id": session.owner_task_id,
  798. "command": session.command,
  799. **{key: getattr(session, f"watcher_{key}") for key in _WATCHER_ROUTE_KEYS},
  800. }
  801. @staticmethod
  802. def _global_watch_event(type_: str, message: str, **extra) -> dict:
  803. """Unaddressed (all-sessions) watch breaker event."""
  804. return {
  805. "session_id": "", "session_key": "", "command": "", "type": type_, **extra,
  806. "message": message,
  807. "platform": "", "chat_id": "", "user_id": "", "user_name": "", "thread_id": "",
  808. }
  809. def _global_watch_admit(self, now: float) -> bool:
  810. """True if this watch_match may pass the global breaker.
  811. In cooldown: drop and count. Otherwise slide the rolling window; exceeding
  812. the cap trips the breaker for WATCH_GLOBAL_COOLDOWN_SECONDS with ONE
  813. "tripped" summary, and the cooldown's end emits ONE "released" summary."""
  814. events = [] # summary events, queued outside the lock
  815. with self._global_watch_lock:
  816. # Handle cooldown expiry first so we can emit the release summary.
  817. if self._global_watch_tripped_until and now >= self._global_watch_tripped_until:
  818. suppressed = self._global_watch_suppressed_during_trip
  819. self._global_watch_tripped_until = 0.0
  820. self._global_watch_suppressed_during_trip = 0
  821. self._global_watch_window_start, self._global_watch_window_hits = now, 0
  822. if suppressed > 0:
  823. events.append(self._global_watch_event(
  824. "watch_overflow_released",
  825. f"Watch-pattern notifications resumed. "
  826. f"{suppressed} match event(s) were suppressed during the flood.",
  827. suppressed=suppressed))
  828. if self._global_watch_tripped_until and now < self._global_watch_tripped_until:
  829. # Still in cooldown — drop and count.
  830. self._global_watch_suppressed_during_trip += 1
  831. admit = False
  832. else:
  833. if now - self._global_watch_window_start >= WATCH_GLOBAL_WINDOW_SECONDS:
  834. self._global_watch_window_start, self._global_watch_window_hits = now, 0
  835. admit = self._global_watch_window_hits < WATCH_GLOBAL_MAX_PER_WINDOW
  836. if admit:
  837. self._global_watch_window_hits += 1
  838. else:
  839. self._global_watch_tripped_until = now + WATCH_GLOBAL_COOLDOWN_SECONDS
  840. self._global_watch_suppressed_during_trip += 1
  841. events.append(self._global_watch_event(
  842. "watch_overflow_tripped",
  843. f"Watch-pattern overflow: >{WATCH_GLOBAL_MAX_PER_WINDOW} "
  844. f"notifications in {WATCH_GLOBAL_WINDOW_SECONDS}s across all processes. "
  845. f"Suppressing further watch_match events for "
  846. f"{WATCH_GLOBAL_COOLDOWN_SECONDS}s."))
  847. for msg in events:
  848. self.completion_queue.put(msg)
  849. return admit
  850. @staticmethod
  851. def _is_host_pid_alive(pid: Optional[int]) -> bool:
  852. """Best-effort liveness check for host-visible PIDs."""
  853. if not pid:
  854. return False
  855. # ``os.kill(pid, 0)`` is NOT a no-op on Windows (bpo-14484) — use the
  856. # cross-platform existence check.
  857. from gateway.status import _pid_exists
  858. return _pid_exists(pid)
  859. @staticmethod
  860. def _safe_host_start_time(pid: Optional[int]) -> Optional[int]:
  861. """Kernel start ticks for a host PID, or None when unavailable."""
  862. try:
  863. from gateway.status import get_process_start_time
  864. return get_process_start_time(pid) if pid else None
  865. except Exception:
  866. return None
  867. @classmethod
  868. def _host_pid_is_ours(cls, pid: Optional[int], expected_start: Optional[int]) -> bool:
  869. """True only if ``pid`` is alive AND still the process we spawned.
  870. The kernel recycles PIDs, so a stored number can later name an unrelated
  871. process (seen in the wild: a browser's session leader tree-killed). The kernel
  872. start time captured at spawn must match the live one; with no baseline
  873. (legacy checkpoints, no ``/proc``) degrade to a bare liveness check."""
  874. return cls._is_host_pid_alive(pid) and (
  875. expected_start is None or cls._safe_host_start_time(pid) == expected_start)
  876. def _detached_host_fate(self, pid: Optional[int], expected_start: Optional[int]) -> str:
  877. """How a recovered host PID should be supervised.
  878. ``running`` — alive and still ours (start time matches, no baseline, or
  879. the start-time probe could not be read). Re-attach; do not invent an exit.
  880. ``reused`` — alive, and the start time positively differs. The number
  881. belongs to someone else; close our entry and never signal that PID.
  882. ``gone`` — the PID is not alive. Prune it; no exit status was collected.
  883. """
  884. if self._host_pid_is_ours(pid, expected_start):
  885. return "running"
  886. if not pid or not self._is_host_pid_alive(pid):
  887. return "gone"
  888. if expected_start is None:
  889. return "running"
  890. current = self._safe_host_start_time(pid)
  891. if current is None or current == expected_start:
  892. return "running"
  893. return "reused"
  894. def _refresh_detached_session(self, session: Optional[ProcessSession]) -> Optional[ProcessSession]:
  895. """Re-attach, close, or prune a recovered host-PID session.
  896. A completion is not queued here: recovery has no waitable handle, so it
  897. never collected an exit status.
  898. """
  899. if session is None or session.exited or not session.detached or session.pid_scope != "host" or session.output_log:
  900. return session # a log-backed adoption's reader collects the recorded exit itself
  901. fate = self._detached_host_fate(session.pid, session.host_start_time)
  902. if fate == "running":
  903. return session
  904. if fate == "gone":
  905. return self._prune_uncollected_detached(session)
  906. self._close_reused_detached(session)
  907. return session
  908. def _close_reused_detached(self, session: ProcessSession) -> None:
  909. """Close an entry whose PID was recycled onto another process.
  910. The stranger is not signalled, and no completion is queued: we never
  911. collected an exit status for the process we spawned.
  912. """
  913. with session._lock:
  914. if session.exited:
  915. return
  916. session.exited = True
  917. with self._lock:
  918. if session.id in self._running:
  919. session.exited_at = time.time()
  920. self._running.pop(session.id, None)
  921. self._finished[session.id] = session
  922. self._write_checkpoint()
  923. session._completion_event.set()
  924. def _prune_uncollected_detached(self, session: ProcessSession) -> Optional[ProcessSession]:
  925. """Drop a recovered entry whose PID is gone, without inventing an exit.
  926. An owned systemd scope stays reachable so kill can still reap it. A
  927. scope-less entry is removed: poll/list must not report a collected exit.
  928. """
  929. with session._lock:
  930. session.exited = True
  931. with self._lock:
  932. self._running.pop(session.id, None)
  933. if session.systemd_unit:
  934. self._finished[session.id] = session
  935. else:
  936. self._finished.pop(session.id, None)
  937. self._write_checkpoint()
  938. session._completion_event.set()
  939. return session if session.systemd_unit else None
  940. def _uncollected_gone(self, session: Optional[ProcessSession]) -> bool:
  941. """True when a detached entry was closed without a collected exit status
  942. and the PID is no longer alive. List/poll must not report that as exited."""
  943. return bool(
  944. session is not None and session.exited and session.exit_code is None
  945. and session.detached and session.pid_scope == "host"
  946. and not self._is_host_pid_alive(session.pid))
  947. @staticmethod
  948. def _config_value(section: str, key: str, fallback):
  949. """``config.yaml`` value for ``section.key``, else the DEFAULT_CONFIG value.
  950. Raises if config is unreadable; callers wrap with their own hard fallback so
  951. registry code paths never crash on a broken config file."""
  952. from hermes_cli.config import DEFAULT_CONFIG, cfg_get, read_raw_config
  953. val = cfg_get(read_raw_config(), section, key)
  954. return DEFAULT_CONFIG[section][key] if val is None else val
  955. @staticmethod
  956. def _config_seconds(key: str, fallback: float) -> float:
  957. """``terminal.<key>`` as a non-negative float (0 disables); *fallback* if unreadable."""
  958. try:
  959. return max(float(ProcessRegistry._config_value("terminal", key, fallback)), 0.0)
  960. except Exception:
  961. return fallback
  962. @staticmethod
  963. def _daemon_term_grace_seconds() -> float:
  964. """Grace (s) between SIGTERM and escalated SIGKILL; 0 disables escalation."""
  965. return ProcessRegistry._config_seconds("daemon_term_grace_seconds", 2.0)
  966. # ----- Spawn -----
  967. @staticmethod
  968. def _new_session(command, task_id, owner_task_id, session_key, cwd, **extra) -> ProcessSession:
  969. from gateway.session_context import get_session_env
  970. return ProcessSession(
  971. id=f"proc_{uuid.uuid4().hex[:12]}", command=command, task_id=task_id,
  972. owner_task_id=owner_task_id, session_key=session_key, cwd=cwd,
  973. parent_session_id=get_session_env("HERMES_SESSION_ID", ""),
  974. wsl_chain=_is_wsl_launcher_command(command),
  975. started_at=time.time(), **extra)
  976. @staticmethod
  977. def _env_temp_dir(env: Any) -> str:
  978. """Return the writable sandbox temp dir for env-backed background tasks."""
  979. get_temp_dir = getattr(env, "get_temp_dir", None)
  980. if callable(get_temp_dir):
  981. try:
  982. temp_dir = get_temp_dir()
  983. if isinstance(temp_dir, str) and temp_dir.startswith("/"):
  984. return temp_dir.rstrip("/") or "/"
  985. except Exception as exc:
  986. logger.debug("Could not resolve environment temp dir: %s", exc)
  987. return tempfile.gettempdir()
  988. def _scope_argv(self, session: ProcessSession, safe_command: str, unit_suffix: str, label: str) -> list[str]:
  989. """Login-shell argv for *safe_command* (parity with LocalEnvironment: rc files
  990. sourced, user tools on PATH), wrapped in a transient systemd scope when we are
  991. the supervised gateway (own cgroup: an OOM kills only the worker, not the
  992. gateway and its messaging control plane)."""
  993. argv = [_find_shell(), "-lic", f"set +m; {safe_command}"]
  994. # This applies to both pipe mode and the PTY path above. See #70716.
  995. in_supervised_gateway = _IS_LINUX and _is_supervised_gateway_process()
  996. if in_supervised_gateway and _systemd_run_user_scope_available():
  997. session.systemd_unit = f"hermes-worker-{unit_suffix}.scope"
  998. return _build_systemd_scope_argv(argv, unit_suffix=unit_suffix)
  999. if in_supervised_gateway:
  1000. # Under a supervisor but no private cgroup: a worker OOM can still take
  1001. # the whole gateway down.
  1002. logger.debug(
  1003. "%s background executor not isolated in a systemd scope "
  1004. "(systemd-run --user unavailable); worker shares the gateway cgroup.", label)
  1005. return argv
  1006. @staticmethod
  1007. def _spawn_env(env_vars: dict) -> dict:
  1008. """Sanitized child env from the session's client toolchain; PYTHONUNBUFFERED so poll sees progress."""
  1009. from agent.session_client_env import toolchain_environ
  1010. env = _terminal_env(_sanitize_subprocess_env(toolchain_environ(), env_vars))
  1011. env["PYTHONUNBUFFERED"] = "1"
  1012. return env
  1013. def _track_started(self, session: ProcessSession, reader_target, reader_name: str, extra_args=()) -> None:
  1014. """Register before the reader can publish completion, even for an exited child."""
  1015. from contextvars import copy_context
  1016. # Reader completion must retain the producer's multiplex profile scope.
  1017. reader = threading.Thread(target=copy_context().run, args=(reader_target, session, *extra_args),
  1018. daemon=True, name=reader_name)
  1019. session._reader_thread = reader
  1020. with self._lock:
  1021. self._prune_if_needed()
  1022. # Completion takes this lock too. Starting here also leaves no
  1023. # ghost entry if the interpreter cannot start another thread.
  1024. reader.start()
  1025. self._running[session.id] = session
  1026. self._write_checkpoint()
  1027. def _spawn_local_pty(self, session: ProcessSession, safe_command: str, env_vars: dict) -> ProcessSession:
  1028. """PTY spawn for interactive CLI tools (Codex, Claude Code, REPLs).
  1029. Raises ImportError when no PTY backend is installed and re-raises any spawn
  1030. failure; ``spawn_local`` falls back to pipe mode in both cases."""
  1031. if _IS_WINDOWS:
  1032. from winpty import PtyProcess as _PtyProcessCls
  1033. else:
  1034. from ptyprocess import PtyProcess as _PtyProcessCls
  1035. pty_argv = self._scope_argv(session, safe_command, session.id, "PTY")
  1036. pty_env = self._spawn_env(env_vars)
  1037. if session.systemd_unit:
  1038. pty_env = systemd_user_bus_env(pty_env)
  1039. # A PTY is a real TTY, so pager-happy tools (git log/diff, man) WILL page and
  1040. # hang waiting for `q` — default them to cat, honoring any pager the user set.
  1041. pty_env.setdefault("GIT_PAGER", "cat")
  1042. pty_env.setdefault("PAGER", "cat")
  1043. pty_proc = _PtyProcessCls.spawn(pty_argv, cwd=session.cwd, env=pty_env, dimensions=(30, 120))
  1044. self._own_pty_job(session, pty_proc)
  1045. session.pid = pty_proc.pid
  1046. session.host_start_time = self._safe_host_start_time(session.pid)
  1047. session._pty = pty_proc
  1048. self._track_started(session, self._pty_reader_loop, f"proc-pty-reader-{session.id}")
  1049. return session
  1050. def spawn_local(
  1051. self, command: str, cwd: str | None = None, task_id: str = "", session_key: str = "",
  1052. env_vars: dict | None = None, use_pty: bool = False, owner_task_id: str = "",
  1053. persist_on_release: bool = False, restore_login_path: bool = False) -> ProcessSession:
  1054. """Spawn a background process locally (TERMINAL_ENV=local; other backends: spawn_via_env()).
  1055. ``use_pty`` asks ptyprocess/pywinpty for a pseudo-terminal; on failure: pipe + ``pty_fallback``.
  1056. ``persist_on_release``: no agent kill sweeps (#41225); ``restore_login_path``: re-add reset PATH dirs (#56634)."""
  1057. # Bash runs ``A && B &`` as a subshell holding our stdout pipe open while a server B lives;
  1058. # the rewriter makes it ``A && { B & }`` (#68915). Lazy: terminal_tool imports this module.
  1059. from tools.terminal_tool_sudo import _rewrite_compound_background as _rewrite_bg
  1060. safe_command = _append_dropped_path_entries(_rewrite_bg(command), self._spawn_env(env_vars).get("PATH", "") if restore_login_path else "")
  1061. session = self._new_session(command, task_id, owner_task_id, session_key, _resolve_safe_cwd(cwd or os.getcwd()),
  1062. persist_on_release=persist_on_release)
  1063. pty_scope_attempted = False
  1064. # A worker's pipe or PTY dies with the worker (the child's next write is SIGPIPE): write to a log.
  1065. log = worker_output_log(session.id)
  1066. if use_pty and log is None:
  1067. try:
  1068. return self._spawn_local_pty(session, safe_command, env_vars)
  1069. except ImportError:
  1070. session.pty_fallback = "the PTY library (ptyprocess/pywinpty) is not installed"
  1071. logger.warning("ptyprocess not installed, falling back to pipe mode")
  1072. except Exception as e:
  1073. session.pty_fallback = f"the PTY spawn failed ({e})"
  1074. logger.warning("PTY spawn failed (%s), falling back to pipe mode", e)
  1075. if session.systemd_unit:
  1076. pty_scope_attempted = True
  1077. if not _stop_systemd_unit(session.systemd_unit):
  1078. raise RuntimeError(
  1079. "PTY scope could not be reaped; refusing pipe fallback "
  1080. "to avoid duplicate command execution"
  1081. ) from e
  1082. session.systemd_unit = ""
  1083. # Pipe path (non-PTY or PTY fallback).
  1084. _popen_kwargs = {"creationflags": windows_hide_flags()} if _IS_WINDOWS else {}
  1085. unit_suffix = f"{session.id}-pipe-fallback" if pty_scope_attempted else session.id
  1086. spawn_argv = self._scope_argv(session, record_exit_command(log, safe_command) if log else safe_command,
  1087. unit_suffix, "Local")
  1088. spawn_env = self._spawn_env(env_vars)
  1089. if session.systemd_unit:
  1090. spawn_env = systemd_user_bus_env(spawn_env)
  1091. # start_new_session is REQUIRED with systemd-run --scope too: the scope does not
  1092. # give the worker a new session, so from an interactive TUI the worker would
  1093. # share the foreground process group and background spawns would stop the whole
  1094. # session (observed as dead TUIs in state T). Cgroup isolation is unaffected —
  1095. # the scope attaches to the invoked process, not the spawning session.
  1096. output = open_output_log(log) if log else subprocess.PIPE
  1097. try:
  1098. proc = self._spawn_job(
  1099. session, spawn_argv, persist_on_release, text=True, cwd=session.cwd, env=spawn_env,
  1100. encoding="utf-8", errors="replace", stdout=output, stderr=subprocess.STDOUT,
  1101. stdin=subprocess.DEVNULL, **_popen_kwargs)
  1102. finally:
  1103. if log:
  1104. os.close(output)
  1105. session.process, session.output_log = proc, str(log or "")
  1106. session.pid = proc.pid
  1107. session.host_start_time = self._safe_host_start_time(session.pid)
  1108. try:
  1109. self._track_started(session, self._log_reader_loop if log else self._reader_loop, f"proc-reader-{session.id}")
  1110. except Exception:
  1111. self._reap_untracked(session, proc)
  1112. raise
  1113. return session
  1114. def adopt_local(
  1115. self, proc: subprocess.Popen, *, command: str, cwd: Optional[str], task_id: str = "",
  1116. session_key: str = "", owner_task_id: str = "", output_so_far: str = "",
  1117. notify_on_complete: bool = True) -> ProcessSession:
  1118. """Take over a still-running foreground Popen as a tracked background session
  1119. (yield-to-background: the user sent a message while the command was running).
  1120. The caller has stopped its own drain thread; the registry's reader continues from
  1121. the pipe's current position and ``output_so_far`` seeds the buffer so nothing
  1122. already captured is lost."""
  1123. session = self._new_session(command, task_id, owner_task_id, session_key, cwd)
  1124. session.process = proc
  1125. session.pid = proc.pid
  1126. session.host_start_time = self._safe_host_start_time(session.pid)
  1127. session.notify_on_complete = notify_on_complete
  1128. handed_pgid = getattr(proc, "_hermes_owned_pgid", None)
  1129. self._take_handoff_group(session, proc)
  1130. if output_so_far:
  1131. session.append_output(output_so_far)
  1132. try:
  1133. self._track_started(session, self._reader_loop, f"proc-reader-{session.id}")
  1134. except Exception:
  1135. # The caller resumes its foreground wait: give it back its group, still covered.
  1136. self._return_handoff_group(session, proc, handed_pgid)
  1137. raise
  1138. return session
  1139. def spawn_via_env(
  1140. self, env: Any, command: str, cwd: str | None = None, task_id: str = "", session_key: str = "",
  1141. timeout: int = 10, owner_task_id: str = "", persist_on_release: bool = False) -> ProcessSession:
  1142. """Spawn a background process inside a non-local backend's sandbox.
  1143. The command is wrapped to capture its in-sandbox PID and redirect output to a
  1144. log file that later execute() calls poll. No live pipe or stdin, but it runs in
  1145. the correct sandbox context. ``persist_on_release`` keeps the process out of
  1146. agent-lifecycle kill sweeps (#41225)."""
  1147. session = self._new_session(command, task_id, owner_task_id, session_key, cwd, env_ref=env, pid_scope="sandbox",
  1148. persist_on_release=persist_on_release)
  1149. temp_dir = self._env_temp_dir(env)
  1150. log_path, pid_path, exit_path = (f"{temp_dir}/hermes_bg_{session.id}.{ext}" for ext in ("log", "pid", "exit"))
  1151. q = shlex.quote
  1152. bg_command = (
  1153. f"mkdir -p {q(temp_dir)} && "
  1154. f"( nohup bash -lc {q(command)} > {q(log_path)} 2>&1; "
  1155. f"rc=$?; printf '%s\\n' \"$rc\" > {q(exit_path)} ) & "
  1156. f"echo $! > {q(pid_path)} && cat {q(pid_path)}")
  1157. result = env.execute(bg_command, cwd=cwd or "", timeout=timeout, rewrite_compound_background=False)
  1158. output = result.get("output", "").strip()
  1159. session.pid = next((int(ln) for ln in map(str.strip, output.splitlines()) if ln.isdigit()), None)
  1160. # No PID from the wrapper (no bash in the image, sandbox gone, broken redirect): the
  1161. # command never started, so raise like a failed Popen instead of returning a session.
  1162. if session.pid is None:
  1163. raise RuntimeError(f"launch wrapper exited {result.get('returncode')} without a PID: "
  1164. f"{output or '(no output)'}")
  1165. self._track_started(
  1166. session, self._env_poller_loop, f"proc-poller-{session.id}", (env, log_path, pid_path, exit_path))
  1167. return session
  1168. # ----- Reader / Poller Threads -----
  1169. def _reader_loop(self, session: ProcessSession):
  1170. """Background thread: read stdout from a local Popen process.
  1171. ``buffer.read1(4096)`` not ``TextIOWrapper.read(4096)``: on pipes the latter
  1172. blocks until EOF, landing "live" output in one burst at exit. Orphaned-pipe
  1173. guard: a backgrounded grandchild (``node server.js &``) inherits our pipe's write
  1174. end so EOF never arrives while it lives, which would park this thread and never
  1175. fire ``notify_on_complete``; on POSIX we wait on an ``fd_wait`` selector (no FD_SETSIZE
  1176. cap, #94928) and stop draining shortly after the direct child exits.
  1177. Windows pipes lack select(), so the lazy ``_reconcile_local_exit`` is the net (#68915, #8340).
  1178. """
  1179. # ``bash -lic`` without a tty writes its startup warnings one write() per line, so the
  1180. # reader can wake between them; strip leading noise from every chunk until the
  1181. # process has produced real output, not just from the first read.
  1182. head_noise = True
  1183. # A split multibyte UTF-8 char would become U+FFFD with stateless decoding; the
  1184. # incremental decoder holds the partial sequence until the rest arrives.
  1185. decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") # (openclaw/openclaw#112325)
  1186. def _append_chunk(chunk: str):
  1187. nonlocal head_noise
  1188. if head_noise:
  1189. chunk = self._clean_shell_noise(chunk)
  1190. head_noise = not chunk.strip()
  1191. self._ingest_output(session, chunk)
  1192. _waiter = None
  1193. try:
  1194. proc = session.process
  1195. if proc is None or proc.stdout is None:
  1196. return
  1197. stdout = proc.stdout
  1198. raw_read = getattr(getattr(stdout, "buffer", None), "read1", None)
  1199. def _read_once():
  1200. """One 4 KiB read: decoded text ('' for a partial multibyte tail), None at EOF."""
  1201. if raw_read is None: # mocked/alternate streams without a raw buffer: less "live"
  1202. return stdout.read(4096) or None
  1203. raw = raw_read(4096)
  1204. return decoder.decode(raw) if raw else None
  1205. # select() needs a real OS fd; mocked streams (tests, adapters) may lack
  1206. # fileno() and use the blocking read instead.
  1207. try:
  1208. fd = stdout.fileno() if raw_read is not None and not _IS_WINDOWS else None
  1209. except Exception:
  1210. fd = None
  1211. if not (isinstance(fd, int) and fd >= 0):
  1212. fd = None
  1213. from tools.environments.fd_wait import open_fd_waiter
  1214. _waiter = open_fd_waiter(fd) if fd is not None else None
  1215. session._reader_selectable = _waiter is not None
  1216. idle_after_exit = 0
  1217. drained_after_request = 0
  1218. while True:
  1219. if _waiter is not None:
  1220. # Once a finish is requested, read only what is already buffered: the exited
  1221. # child's tail, never an orphaned grandchild's future writes.
  1222. wait_s = 0 if session._reader_finish_requested.is_set() else 0.2
  1223. ready = _waiter.wait(wait_s)
  1224. if not ready:
  1225. if session._reader_finish_requested.is_set():
  1226. break
  1227. # Direct child gone and pipe idle ~200ms: a few more cycles for a
  1228. # buffered tail, then stop rather than wait forever on an orphaned
  1229. # grandchild's pipe.
  1230. if proc.poll() is not None:
  1231. # See #68915.
  1232. idle_after_exit += 1
  1233. if idle_after_exit >= 3:
  1234. break
  1235. continue
  1236. chunk = _read_once()
  1237. if chunk is None:
  1238. break # true EOF — all writers closed
  1239. if chunk:
  1240. _append_chunk(chunk)
  1241. if session._reader_finish_requested.is_set():
  1242. # A single chunk is not the tail: the child can exit with a whole pipe buffer
  1243. # unread. The cap only stops a grandchild that never lets the pipe go empty.
  1244. drained_after_request += len(chunk)
  1245. if drained_after_request >= _FINAL_DRAIN_MAX_CHARS:
  1246. break
  1247. idle_after_exit = 0
  1248. except Exception as e:
  1249. logger.debug("Process stdout reader ended: %s", e)
  1250. finally:
  1251. if _waiter is not None:
  1252. _waiter.close()
  1253. self._finish_reader(
  1254. session, decoder, _append_chunk, "Process",
  1255. session.process.wait, lambda: session.process.returncode)
  1256. def _finish_reader(self, session, decoder, append, label, wait, exit_code) -> None:
  1257. """Reader-thread teardown: flush the decoder (a truncated multibyte tail becomes
  1258. one U+FFFD instead of vanishing), reap the child (no zombies), record the exit.
  1259. A process may close stdout long before it exits. The reader owns a dedicated
  1260. daemon thread, so it must keep waiting rather than publish a false completion
  1261. and discard the only ``Popen`` handle that can reap the child.
  1262. """
  1263. with suppress(Exception):
  1264. tail = decoder.decode(b"", final=True)
  1265. if tail:
  1266. append(tail)
  1267. try:
  1268. wait()
  1269. except Exception as e:
  1270. # A PTY child reaped by isalive() already has its exitstatus; only an
  1271. # unknown status must stay tracked for later reconciliation.
  1272. if exit_code() is None:
  1273. logger.warning("%s wait failed; leaving process tracked: %s", label, e)
  1274. return
  1275. logger.warning("%s wait failed; recording known exit status: %s", label, e)
  1276. self._finish_exited(session, exit_code())
  1277. _log_delta_command = staticmethod(log_delta_command)
  1278. def _env_poller_loop(self, session: ProcessSession, env: Any, log_path: str, pid_path: str, exit_path: str):
  1279. """Background thread: poll a sandbox log file for non-local backends."""
  1280. q = shlex.quote
  1281. # Byte offset already read from the log (bytes, not chars: the shell counts bytes).
  1282. prev_output_bytes = 0
  1283. while not session.exited:
  1284. time.sleep(2)
  1285. try:
  1286. # Read only the bytes written since the last poll.
  1287. raw = env.execute(self._log_delta_command(q(log_path), prev_output_bytes),
  1288. timeout=10).get("output", "")
  1289. header, _, delta = raw.partition("\n")
  1290. try:
  1291. size_str, offset_str = header.split()
  1292. new_size = int(size_str)
  1293. used_offset = int(offset_str)
  1294. except ValueError:
  1295. # No usable header (command failed, shell missing a tool): skip this
  1296. # poll rather than act on a half-read value.
  1297. new_size = None
  1298. used_offset = None
  1299. delta = ""
  1300. if new_size is not None:
  1301. if used_offset < prev_output_bytes:
  1302. # Log rotated/truncated: what we hold no longer lines up. Restart.
  1303. with session._lock:
  1304. if session.exited:
  1305. return
  1306. session.output_buffer = ""
  1307. prev_output_bytes = new_size
  1308. if delta:
  1309. # A kill that landed after this fetch already published its output
  1310. # snapshot: leave the output as the kill reported it (as the PTY reader does).
  1311. with session._lock:
  1312. if session.exited:
  1313. return
  1314. session.output_buffer += delta
  1315. if len(session.output_buffer) > session.max_output_chars:
  1316. session.output_buffer = session.output_buffer[-session.max_output_chars:]
  1317. self._check_watch_patterns(session, delta)
  1318. self._emit_output(session, delta)
  1319. check = env.execute(
  1320. f"kill -0 \"$(cat {q(pid_path)} 2>/dev/null)\" 2>/dev/null; echo $?", timeout=5)
  1321. check_output = check.get("output", "").strip()
  1322. if check_output and check_output.splitlines()[-1].strip() != "0":
  1323. # Exited -- read the exit code captured by the wrapper shell.
  1324. exit_str = env.execute(f"cat {q(exit_path)} 2>/dev/null", timeout=5).get("output", "").strip()
  1325. try:
  1326. exit_code = int(exit_str.splitlines()[-1].strip())
  1327. except (ValueError, IndexError):
  1328. exit_code = -1
  1329. self._finish_exited(session, exit_code)
  1330. return
  1331. except Exception:
  1332. # Environment might be gone (sandbox reaped, etc.). A kill that already
  1333. # settled the session keeps its result.
  1334. self._finish_exited(session, -1, reason="lost", source="backend_lost")
  1335. return
  1336. def _pty_reader_loop(self, session: ProcessSession):
  1337. """Background thread: read output from a PTY process."""
  1338. pty = session._pty
  1339. # Same split-multibyte handling as _reader_loop.
  1340. # PTY reads can split a multibyte UTF-8 character across chunks just like pipe reads — hold partial
  1341. # sequences until the rest arrives. (Ported from openclaw/openclaw#112325.)
  1342. decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
  1343. # Programs in a PTY can block waiting for replies to device-status / window-size /
  1344. # cursor-position / DEC private-mode queries. Answer the bounded set and strip the
  1345. # queries from captured output. POSIX only: Windows ConPTY is a real console host that
  1346. # answers itself (and pywinpty yields str chunks, not bytes).
  1347. responder = None
  1348. if not _IS_WINDOWS:
  1349. from tools.pty_query_responder import PtyQueryResponder
  1350. responder = PtyQueryResponder(rows=30, cols=120)
  1351. def ingest(text: str) -> None:
  1352. # A kill can leave this reader running while a detached descendant holds the
  1353. # slave open (_release_finished_handles defers the close to it). Keep draining,
  1354. # but leave the killed session's output as the kill reported it.
  1355. self._ingest_output(session, text, unless_exited=True)
  1356. try:
  1357. while pty.isalive():
  1358. try:
  1359. chunk = pty.read(4096)
  1360. if chunk:
  1361. # ptyprocess returns bytes; pywinpty returns str
  1362. if responder is not None and isinstance(chunk, bytes):
  1363. chunk, replies = responder.process(chunk)
  1364. if replies:
  1365. try:
  1366. pty.write(replies)
  1367. except Exception:
  1368. logger.debug(
  1369. "PTY query response write failed",
  1370. exc_info=True,
  1371. )
  1372. text = chunk if isinstance(chunk, str) else decoder.decode(chunk)
  1373. if text:
  1374. ingest(text)
  1375. except Exception: # EOFError included
  1376. break
  1377. except Exception as e:
  1378. logger.debug("PTY stdout reader ended: %s", e)
  1379. if responder is not None:
  1380. # A query prefix split across the final reads is plain output after all.
  1381. tail = decoder.decode(responder.flush())
  1382. if tail:
  1383. ingest(tail)
  1384. self._finish_reader(
  1385. session, decoder, ingest, "PTY",
  1386. pty.wait, lambda: pty.exitstatus if hasattr(pty, 'exitstatus') else -1)
  1387. def _ingest_output(self, session: ProcessSession, text: str, *, unless_exited: bool = False) -> None:
  1388. """Buffer a freshly-read chunk, then scan watch patterns and stream it live.
  1389. ``unless_exited`` drops the chunk once the session has exited (atomically with a kill)."""
  1390. if not unless_exited:
  1391. session.append_output(text)
  1392. elif not session.append_output_if_running(text):
  1393. return
  1394. self._check_watch_patterns(session, text)
  1395. self._emit_output(session, text)
  1396. def _finish_exited(self, session: ProcessSession, exit_code, reason: str = "exited", source: str = "") -> None:
  1397. """Mark a reader-observed exit (a raced kill keeps its own code/reason) and finish.
  1398. Under the session lock, like kill_process's commit: otherwise the kill can land between
  1399. mark_exited's ``killed`` check and its writes, and the reader overwrites the kill."""
  1400. with session._lock:
  1401. session.mark_exited(exit_code, reason, source)
  1402. self._move_to_finished(session)
  1403. def _move_to_finished(self, session: ProcessSession) -> bool:
  1404. """Move a session from running to finished.
  1405. Idempotent: kill_process() and the reader thread can both call this; only
  1406. the FIRST move enqueues the completion notification, so no duplicates.
  1407. Returns True when this call is the one that persisted the session."""
  1408. with self._lock:
  1409. was_running = session.id in self._running
  1410. if was_running:
  1411. session._finish_claimed = True
  1412. session.exited_at = time.time()
  1413. # Keep the session tracked until its result is durable. A finite
  1414. # parent must not observe completion and exit during this write.
  1415. save_completed_result(session)
  1416. self._running.pop(session.id)
  1417. self._finished[session.id] = session
  1418. self._release_job_group(session)
  1419. # Release the retained Popen/PTY handles now: otherwise every
  1420. # finished-but-unpruned session keeps its stdout pipe (or PTY master)
  1421. # FD open until FINISHED_TTL_SECONDS elapses, and heavy background
  1422. # churn can exhaust the gateway's FD limit. On the reader-thread path
  1423. # the pipe is already at EOF; on the kill/reconcile paths the reader
  1424. # may still be draining — its next read raises on the closed stream
  1425. # and the loop exits, dropping at most the unread tail of a process
  1426. # that was just killed. poll()/wait()/read_log() serve from the
  1427. # buffered ``output_buffer``, never from the pipe.
  1428. try:
  1429. self._publish_finished(session, was_running)
  1430. finally:
  1431. # The owner releases waiters only after its notice is queued (or its publish
  1432. # failed, so a waiter never parks forever). A duplicate leaves the event to it.
  1433. if was_running or not session._finish_claimed:
  1434. session._completion_event.set()
  1435. return was_running
  1436. def _publish_finished(self, session: ProcessSession, was_running: bool) -> None:
  1437. self._release_finished_handles(session)
  1438. self._write_checkpoint()
  1439. if was_running and session.notify_on_complete:
  1440. notification = {
  1441. "type": "completion",
  1442. "session_id": session.id,
  1443. "session_key": session.session_key,
  1444. "task_id": session.task_id,
  1445. "owner_task_id": session.owner_task_id,
  1446. "command": session.command,
  1447. **({"handoff_note": session.handoff_note} if session.handoff_note else {}),
  1448. **self._exit_fields(session),
  1449. # A consumer that relays the output (a bot DM's reply) must know it is not whole.
  1450. **_completion_output(session),
  1451. # Stable producer identity across checkpoint recovery (unlike a
  1452. # consumer-observed completion timestamp).
  1453. "started_at": session.started_at,
  1454. }
  1455. _redact_process_result(notification)
  1456. self.completion_queue.put(notification)
  1457. @staticmethod
  1458. def _exit_fields(session: ProcessSession) -> dict:
  1459. return {
  1460. "exit_code": session.exit_code,
  1461. "completion_reason": session.completion_reason,
  1462. "termination_source": session.termination_source,
  1463. }
  1464. def _release_finished_handles(self, session: ProcessSession):
  1465. """Close a finished session's OS handles (Popen pipes / PTY master).
  1466. Best-effort and idempotent: the session may have no local Popen (env
  1467. backends, detached recovery), or the handles may already be closed by
  1468. the reader loop / kill path. Closing a Popen's stream objects does not
  1469. kill anything — the child has already exited — it only releases the
  1470. parent's pipe FDs, which is exactly the retained-resource leak.
  1471. """
  1472. proc = session.process
  1473. if proc is not None:
  1474. for stream in (proc.stdout, proc.stderr, proc.stdin):
  1475. if stream is not None:
  1476. with suppress(OSError, ValueError): # a stdin flush can hit EPIPE
  1477. stream.close()
  1478. if session._pty is not None:
  1479. # A live ptyprocess reader sits in a blocking read holding the PTY file
  1480. # object's buffer lock, and that read only ends once every holder of
  1481. # the slave side is gone. A descendant that setsid()s past the kill
  1482. # keeps it open, so close() here would block forever (under _lock on
  1483. # the prune path). The reader closes the PTY itself via
  1484. # _finish_reader once its read ends. pywinpty reads don't block, so
  1485. # Windows closes here as before.
  1486. reader = session._reader_thread
  1487. if (not _IS_WINDOWS and reader is not None and reader.is_alive()
  1488. and reader is not threading.current_thread()):
  1489. return
  1490. # ptyprocess/pywinpty close() is idempotent (``closed`` flag) and
  1491. # closes the master fd exactly once; it raises only if the child
  1492. # ignores SIGKILL, which we don't want to surface on the finish path.
  1493. with suppress(Exception):
  1494. session._pty.close()
  1495. # ----- Query Methods -----
  1496. @staticmethod
  1497. def _reader_finalizing(session: ProcessSession) -> bool:
  1498. """A reconcile saw the direct child exit and set ``exited``, but the reader that owns the
  1499. output is still on its final drain: the buffer is not the whole output yet, so nothing
  1500. may consume it as the completion."""
  1501. reader = session._reader_thread
  1502. return (session._reader_finish_requested.is_set() and not session._completion_event.is_set()
  1503. and reader is not None and reader.is_alive())
  1504. def is_completion_consumed(self, session_id: str) -> bool:
  1505. """Check if a completion notification was already consumed via wait/log."""
  1506. return session_id in self._completion_consumed
  1507. def is_session_waiting(self, session_id: str) -> bool:
  1508. """Whether a goal loop (``hermes_cli.goals`` wait barrier) should stay parked on
  1509. this session: still running AND, with ``watch_patterns``, none matched yet (a
  1510. long-lived watcher unblocks on its trigger, not on exit). Unknown/exited/
  1511. already-fired sessions return False so a stale barrier can never wedge the loop."""
  1512. with self._lock:
  1513. session = (self._running.get(session_id) or self._finished.get(session_id)) if session_id else None
  1514. if session is None:
  1515. return False
  1516. with suppress(Exception):
  1517. refreshed = self._refresh_detached_session(session)
  1518. if refreshed is None:
  1519. return False
  1520. session = refreshed
  1521. return not session.exited and not (
  1522. session.watch_patterns and not session._watch_disabled and session._watch_hits > 0)
  1523. def wait_for_pending_completions(
  1524. self, task_id: Optional[str] = None, *, timeout: float | None = None, poll_interval: float = 1.0,
  1525. ) -> dict:
  1526. """Bounded linger for ``notify_on_complete`` background processes at one-shot exit.
  1527. A one-shot CLI run (``hermes -q/-Q/-z``) exits when its turn ends; a background
  1528. process it spawned still holds a stdout pipe owned by the dying parent and dies of
  1529. SIGPIPE seconds later (Bot Mode handoff replies were the visible casualty). Only
  1530. ``notify_on_complete`` processes carry a completion contract — servers/daemons/
  1531. watchers aren't the parent's to wait for. ``task_id=None`` waits on every tracked
  1532. process; ``timeout=None`` reads ``terminal.oneshot_completion_wait_seconds`` (``<= 0``
  1533. disables). Each pass re-reconciles child state so an orphaned-pipe exit can't wedge
  1534. the linger. Returns ``{"waited", "completed", "timed_out"}`` id lists.
  1535. Bot Mode handoff REPLIES are the visible casualty (#90879): a recipient invoked as ``hermes -p <bot>
  1536. chat -Q --query-file ...`` dispatches its reply via ``message_agent`` / ``bot_relay`` exactly this
  1537. way, then exits, and the reply process is destroyed ~3s later. The sender waits forever for a reply
  1538. that was already killed.
  1539. See #17327.
  1540. """
  1541. if timeout is None:
  1542. timeout = self._oneshot_completion_wait_seconds()
  1543. result: dict = {"waited": [], "completed": [], "timed_out": []}
  1544. with self._lock:
  1545. # `_finished` too: `_move_to_finished` pops a session from `_running` and enqueues its completion
  1546. # only after releasing handles and writing the checkpoint. A parent whose turn ends inside that
  1547. # window would otherwise see nothing pending, drain nothing and exit without the follow-up turn.
  1548. pending = [
  1549. s for store in (self._running, self._finished) for s in store.values()
  1550. if s.notify_on_complete and not s._completion_event.is_set()
  1551. and (task_id is None or s.owner_task_id == task_id)
  1552. ]
  1553. if not pending or timeout <= 0:
  1554. return result
  1555. result["waited"] = [s.id for s in pending]
  1556. logger.info(
  1557. "One-shot exit lingering (bounded %ss) for %d notify_on_complete "
  1558. "background process(es): %s",
  1559. timeout, len(pending), ", ".join(s.id for s in pending))
  1560. deadline = time.monotonic() + max(float(timeout), 0.0)
  1561. interval = max(float(poll_interval), 0.05)
  1562. try:
  1563. from tools.interrupt import is_interrupted as _is_interrupted
  1564. except Exception:
  1565. _is_interrupted = lambda: False
  1566. interrupted = False
  1567. for session in pending:
  1568. try:
  1569. while not session._completion_event.is_set():
  1570. if interrupted or _is_interrupted():
  1571. interrupted = True
  1572. break
  1573. remaining = deadline - time.monotonic()
  1574. if remaining <= 0:
  1575. break
  1576. # Reconcile first so orphaned-pipe and detached exits fire the event.
  1577. with suppress(Exception):
  1578. # Reconcile first: catches direct-child exits whose reader is blocked on a pipe held
  1579. # open by a descendant (#17327) and detached/env sessions, so the event actually
  1580. # fires.
  1581. # Reconcile against real child state before reading session.exited. Guards against
  1582. # orphaned-pipe reader hangs (issue #17327).
  1583. # Reconcile against real child state — guards against orphaned- pipe reader hangs
  1584. # where the reader is blocked but the direct child has already exited (issue
  1585. # #17327).
  1586. self._reconcile_local_exit(session)
  1587. if self._refresh_detached_session(session) is None:
  1588. break
  1589. if session._completion_event.is_set():
  1590. break
  1591. session._completion_event.wait(min(remaining, interval))
  1592. except KeyboardInterrupt:
  1593. # Stop waiting, but never let the interrupt skip the caller's durable
  1594. # teardown (session flush, end_session) that follows.
  1595. interrupted = True
  1596. result["completed" if session._completion_event.is_set() else "timed_out"].append(session.id)
  1597. if result["timed_out"]:
  1598. logger.warning(
  1599. "One-shot exit linger timed out after %ss with %d background "
  1600. "process(es) still running: %s — they may be killed when this "
  1601. "process exits.",
  1602. timeout, len(result["timed_out"]), ", ".join(result["timed_out"]))
  1603. return result
  1604. @staticmethod
  1605. def _oneshot_completion_wait_seconds() -> float:
  1606. """Linger (s) for one-shot exits with pending notify_on_complete processes; 0 disables."""
  1607. return ProcessRegistry._config_seconds("oneshot_completion_wait_seconds", 600.0)
  1608. def _drain_should_skip(self, session_id: str, *, skip_poll_observed: bool = True) -> bool:
  1609. """Skip a completion the CLI agent already has this turn — consumed via wait/log
  1610. or observed inline via poll(). Gateway/tui watchers check only
  1611. ``is_completion_consumed`` so a read-only poll never suppresses their turn.
  1612. Skips when the agent has either truly consumed the output (wait/log → ``_completion_consumed``) or
  1613. observed the exit inline via poll() (``_poll_observed``). In both cases the CLI agent already has
  1614. the result this turn, so injecting a [SYSTEM: ...] completion would be a duplicate (#8228).
  1615. """
  1616. return session_id in self._completion_consumed or (skip_poll_observed and session_id in self._poll_observed)
  1617. @staticmethod
  1618. def _surface_child_process_notifications() -> bool:
  1619. """``delegation.surface_child_process_notifications``; False on any config
  1620. error — never crash the drain loop."""
  1621. try:
  1622. return bool(ProcessRegistry._config_value("delegation", "surface_child_process_notifications", False))
  1623. except Exception:
  1624. return False
  1625. @staticmethod
  1626. def _owns_event(evt: dict, session_key: str, owns_event, is_async_delegation: bool) -> bool:
  1627. """Routing verdict for one drained event (see drain_notifications); False = requeue."""
  1628. evt_session_key = str(evt.get("session_key") or "")
  1629. requires_positive_proof = is_async_delegation or bool(evt_session_key or evt.get("origin_ui_session_id"))
  1630. if owns_event is not None and requires_positive_proof:
  1631. try:
  1632. return bool(owns_event(evt))
  1633. except Exception:
  1634. return False # fail closed — never leak on a broken check
  1635. if session_key and requires_positive_proof:
  1636. return evt_session_key == session_key
  1637. # Restored payloads from a previous process: an unfiltered drain cannot prove
  1638. # ownership, so leave them for the owner.
  1639. return not (is_async_delegation and evt.get("restored"))
  1640. def restore_completions(self) -> int:
  1641. """Rehydrate durable delegation completions from the LAUNCH profile's ledger, once per
  1642. process. Called by the first consumer that drains the queue (CLI/TUI drain, gateway boot,
  1643. TUI poller) so a mere ``import model_tools`` never touches state.db (#123265). The replay
  1644. always runs in the launch scope: the TUI poller / prompt_turn drain call this under the
  1645. session's profile binding, and a once-per-process replay taken under a secondary's scope
  1646. would leave the launch ledger unreplayed for the life of the process. Secondaries are
  1647. replayed by the gateway's ``_restore_secondary_completion_ledgers``."""
  1648. if self._completions_restored:
  1649. return 0
  1650. self._completions_restored = True
  1651. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  1652. token = set_hermes_home_override(None)
  1653. try:
  1654. from tools.async_delegation import restore_undelivered_completions
  1655. return restore_undelivered_completions(self.completion_queue)
  1656. except Exception as exc:
  1657. logger.warning("Could not restore async delegation completions: %s", exc)
  1658. return 0
  1659. finally:
  1660. reset_hermes_home_override(token)
  1661. def drain_notifications(
  1662. self, session_key: str = "", owns_event=None, *, skip_poll_observed: bool = True,
  1663. ) -> list[tuple[dict, str]]:
  1664. """Pop all pending events and return ``(raw_event, formatted_text)`` pairs.
  1665. Skips completions per ``_drain_should_skip`` (gateway/TUI pass
  1666. ``skip_poll_observed=False``). Routing (``_owns_event``): async-delegation events
  1667. always need ownership proof, ordinary events once they carry ``session_key`` or
  1668. ``origin_ui_session_id``; ``owns_event(evt)`` (strongest; the TUI passes a
  1669. compression-chain-aware check) consumes ONLY on True, ``session_key`` uses plain
  1670. equality; non-owned events are re-queued for their owner. No filter consumes
  1671. everything (legacy single-session) except restored delegation payloads (fail-closed)."""
  1672. self.restore_completions()
  1673. results: list[tuple[dict, str]] = []
  1674. requeue: list[dict] = []
  1675. # delegation.surface_child_process_notifications, read at most once per drain
  1676. # and only when an sa- event shows up.
  1677. surface_child: bool | None = None
  1678. while not self.completion_queue.empty():
  1679. try:
  1680. evt = self.completion_queue.get_nowait()
  1681. except Exception:
  1682. break
  1683. is_async_delegation = evt.get("type") == "async_delegation"
  1684. if not self._owns_event(evt, session_key, owns_event, is_async_delegation):
  1685. requeue.append(evt)
  1686. continue
  1687. # Routing happened first so a foreign session cannot drop the owner's
  1688. # event via its own consumed/observed state.
  1689. _evt_sid = evt.get("session_id", "")
  1690. if evt.get("type") == "completion" and self._drain_should_skip(
  1691. _evt_sid, skip_poll_observed=skip_poll_observed):
  1692. continue
  1693. # Subagent-owned process notifications are suppressed by default — the
  1694. # child's delegation result is the deliverable. Judge ownership on
  1695. # owner_task_id (RAW spawning id; task_id is the container key, collapsed
  1696. # by _resolve_container_task_id). Dropped, NOT requeued: children never
  1697. # drain, so a requeue would pin the event forever. 'async_delegation'
  1698. # is the result itself and is NEVER suppressed.
  1699. _evt_task_id = str(evt.get("owner_task_id") or evt.get("task_id") or "")
  1700. if is_child_process_notification(evt):
  1701. if surface_child is None:
  1702. surface_child = self._surface_child_process_notifications()
  1703. if not should_surface_notification(evt, surface_child=surface_child):
  1704. logger.debug(
  1705. "Suppressed subagent-owned process notification "
  1706. "(delegation.surface_child_process_notifications=false): "
  1707. "type=%s session_id=%s task_id=%s",
  1708. evt.get("type", "completion"), _evt_sid, _evt_task_id)
  1709. continue
  1710. if text := format_process_notification(evt):
  1711. results.append((evt, text))
  1712. for evt in requeue:
  1713. self.completion_queue.put(evt)
  1714. return results
  1715. # Minimum suffix chars for prefix resolution; "p"/"proc_1" are too collision-prone.
  1716. _MIN_PREFIX_CHARS = 4
  1717. def get(self, session_id: str) -> Optional[ProcessSession]:
  1718. """Session by full ID or unique prefix (``proc_4dae`` / bare ``4dae``, like git
  1719. short hashes); ambiguous or too-short prefixes resolve to None, never a guess."""
  1720. if not isinstance(session_id, str) or not session_id:
  1721. return None
  1722. with self._lock:
  1723. session = self._running.get(session_id) or self._finished.get(session_id)
  1724. if session is None:
  1725. session = load_completed_results(session_id).get(session_id)
  1726. return self._refresh_detached_session(session if session is not None else self._resolve_prefix(session_id))
  1727. def _resolve_prefix(self, session_id: str) -> Optional[ProcessSession]:
  1728. """Resolve a unique session-ID prefix (a bare hex tail is normalized to
  1729. ``proc_<tail>``); :meth:`get` tries exact first."""
  1730. query = session_id.strip() if isinstance(session_id, str) else ""
  1731. if not query:
  1732. return None
  1733. if not query.startswith("proc_"):
  1734. query = f"proc_{query}"
  1735. if len(query) - len("proc_") < self._MIN_PREFIX_CHARS:
  1736. return None
  1737. matches = load_completed_results(query)
  1738. with self._lock:
  1739. matches.update({
  1740. sid: s for store in (self._running, self._finished)
  1741. for sid, s in store.items() if sid.startswith(query)
  1742. })
  1743. return next(iter(matches.values())) if len(matches) == 1 else None
  1744. def _reconcile_local_exit(self, session: ProcessSession) -> None:
  1745. """Reconcile ``session.exited`` against the real child state.
  1746. The reader flips ``exited`` only at EOF; when the direct child has exited but a
  1747. descendant (e.g. a daemon from ``hermes update``) holds the pipe open, poll()
  1748. would report "running" forever. If ``Popen.poll()`` has an exit code, drain
  1749. readable bytes non-blocking and flip ``exited``. No-op for env/PTY, exited and
  1750. detached sessions.
  1751. The reader thread (`_reader_loop`) sets `session.exited = True` only in its `finally` block, which
  1752. runs when `stdout.read()` returns EOF. If the direct `Popen` child has exited but a descendant
  1753. process (e.g. a daemon spawned by `hermes update` restarting the gateway) is still holding the
  1754. stdout pipe open, the reader blocks forever and poll() keeps returning "running" indefinitely (issue
  1755. #17327 — 74 polls over 7 minutes on Feishu).
  1756. """
  1757. if session is None or session.exited:
  1758. return
  1759. proc = getattr(session, "process", None)
  1760. if proc is None:
  1761. return
  1762. try:
  1763. rc = proc.poll()
  1764. except Exception:
  1765. return
  1766. if rc is None:
  1767. return # Direct child still running — reader block is legitimate.
  1768. reader = session._reader_thread
  1769. if (
  1770. not _IS_WINDOWS
  1771. and session._reader_selectable
  1772. and reader is not None
  1773. and reader.is_alive()
  1774. ):
  1775. # The reader owns the pipe and completion payload. Asking it to
  1776. # finish avoids a competing TextIOWrapper read here racing the
  1777. # reader, publishing an empty owner-stamped result, then closing
  1778. # the pipe before the buffered tail is ingested. It wakes within
  1779. # the reader's bounded select interval and drains what is buffered.
  1780. session._reader_finish_requested.set()
  1781. # Flipping ``exited`` before that drain lands would hand poll()/wait() a truncated
  1782. # snapshot (and wait() marks it consumed), so give the reader a moment to publish.
  1783. if session._completion_event.wait(_READER_FINAL_DRAIN_WAIT_SECONDS):
  1784. return
  1785. with session._lock:
  1786. session.mark_exited(rc)
  1787. logger.info(
  1788. "Reconciled session %s: direct child exited with code %s; "
  1789. "reader will publish the owned completion after its final drain.",
  1790. session.id, rc)
  1791. return
  1792. # Best-effort non-blocking drain of whatever the reader hasn't consumed.
  1793. stdout = getattr(proc, "stdout", None)
  1794. if stdout is not None and not _IS_WINDOWS:
  1795. try:
  1796. import fcntl
  1797. fd = stdout.fileno()
  1798. flags = fcntl.fcntl(fd, fcntl.F_GETFL)
  1799. fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
  1800. try:
  1801. with suppress(BlockingIOError, OSError, ValueError):
  1802. chunk = stdout.read()
  1803. if chunk:
  1804. session.append_output(chunk if isinstance(chunk, str) else chunk.decode("utf-8", errors="replace"))
  1805. finally:
  1806. with suppress(Exception):
  1807. fcntl.fcntl(fd, fcntl.F_SETFL, flags)
  1808. except Exception as e:
  1809. logger.debug("Non-blocking drain failed for %s: %s", session.id, e)
  1810. with session._lock:
  1811. session.mark_exited(rc)
  1812. logger.info(
  1813. "Reconciled session %s: direct child exited with code %s but reader "
  1814. "was still blocked (orphaned pipe). Flipped to exited.",
  1815. session.id, rc)
  1816. self._move_to_finished(session)
  1817. @staticmethod
  1818. def _status_head(session: ProcessSession) -> dict:
  1819. return {"session_id": session.id, "command": session.command, "status": "exited" if session.exited else "running"}
  1820. def poll(self, session_id: str) -> dict:
  1821. """Check status and get new output for a background process."""
  1822. session = self.get(session_id)
  1823. if session is None or self._uncollected_gone(session):
  1824. return _not_found(session_id)
  1825. self._reconcile_local_exit(session) # orphaned-pipe reader guard
  1826. # Read with the preview under the lock the reader appends and exits under, so the
  1827. # finalization check describes the output actually returned.
  1828. with session._lock:
  1829. output_preview = _output_tail(session, 1000)
  1830. exited, finalizing = session.exited, self._reader_finalizing(session)
  1831. result = {
  1832. **self._status_head(session), "pid": session.pid,
  1833. "uptime_seconds": int(time.time() - session.started_at), "output_preview": output_preview}
  1834. if exited:
  1835. result.update(self._exit_fields(session))
  1836. # Read-only: record in _poll_observed (CLI inline dedup) but NOT in
  1837. # _completion_consumed, or a status check would suppress the watcher's
  1838. # autonomous delivery turn. See __init__. A preview taken before the reader's
  1839. # final drain is partial and must not suppress the full completion.
  1840. if not finalizing:
  1841. self._poll_observed.add(session_id)
  1842. if session.detached and not session.output_log:
  1843. result.update(detached=True, note="Process recovered after restart -- output history unavailable")
  1844. return result
  1845. def read_log(self, session_id: str, offset: int | None = None, limit: int = 200) -> dict:
  1846. """Read the full output log with optional pagination by lines."""
  1847. from tools.ansi_strip import strip_ansi
  1848. session = self.get(session_id)
  1849. if session is None:
  1850. return _not_found(session_id)
  1851. with session._lock:
  1852. full_output = strip_ansi(session.output_buffer)
  1853. exited, finalizing = session.exited, self._reader_finalizing(session)
  1854. lines = full_output.splitlines()
  1855. total_lines = len(lines)
  1856. # An explicit offset=0 means "start from the first line" — previously it was conflated with the
  1857. # default and silently returned the TAIL instead of the head (same falsy-coercion class as the
  1858. # wait() timeout guard; salvaged from PR #60004, credit @isheng-eqi).
  1859. if offset is None and limit > 0:
  1860. selected = lines[-limit:]
  1861. observed_completion_output = bool(selected) or total_lines == 0
  1862. else:
  1863. offset = offset or 0
  1864. selected = lines[offset:offset + limit]
  1865. stop = slice(offset, offset + limit).indices(total_lines)[1]
  1866. observed_completion_output = total_lines == 0 or (bool(selected) and stop == total_lines)
  1867. result = {
  1868. **self._status_head(session), "output": "\n".join(selected),
  1869. "total_lines": total_lines, "showing": f"{len(selected)} lines"}
  1870. if exited and observed_completion_output and not finalizing:
  1871. self._completion_consumed.add(session_id)
  1872. return result
  1873. def wait(self, session_id: str, timeout: int | None = None) -> dict:
  1874. """Block until the process exits, the timeout elapses, the user interrupts, or a
  1875. mid-turn user message (steer/redirect → ``request_yield``) releases the wait.
  1876. ``timeout`` defaults to (and is clamped by) TERMINAL_TIMEOUT. Returns a dict
  1877. with status exited|timeout|interrupted|not_found|error and an output snapshot."""
  1878. from tools.interrupt import consume_yield as _consume_yield, is_interrupted as _is_interrupted
  1879. try:
  1880. max_timeout = int(os.getenv("TERMINAL_TIMEOUT", "180"))
  1881. except (ValueError, TypeError):
  1882. max_timeout = 180
  1883. # The schema says minimum=1 but not every caller enforces it; timeout=0 is
  1884. # falsy and would silently fall through to the default wait.
  1885. if timeout is not None and timeout <= 0:
  1886. return {"status": "error", "error": f"timeout must be positive (got {timeout})"}
  1887. timeout_note = None
  1888. effective_timeout = timeout or max_timeout
  1889. if timeout and timeout > max_timeout:
  1890. effective_timeout = max_timeout
  1891. timeout_note = f"Requested wait of {timeout}s was clamped to configured limit of {max_timeout}s"
  1892. session = self.get(session_id)
  1893. if session is None:
  1894. return _not_found(session_id)
  1895. deadline = time.monotonic() + effective_timeout
  1896. while time.monotonic() < deadline:
  1897. session = self._refresh_detached_session(session)
  1898. if session is None:
  1899. return _not_found(session_id)
  1900. self._reconcile_local_exit(session) # orphaned-pipe reader guard
  1901. result = None
  1902. if session.exited and not self._reader_finalizing(session):
  1903. self._completion_consumed.add(session_id)
  1904. result = self._exit_snapshot(session, "exited")
  1905. elif _is_interrupted():
  1906. result = {
  1907. "status": "interrupted", "command": session.command, "output": _output_tail(session, 1000),
  1908. "note": "User sent a new message -- wait interrupted"}
  1909. elif _consume_yield(threading.current_thread().ident):
  1910. # A steer/redirect landed mid-turn: redirect() asks tool workers to YIELD so
  1911. # the user's message is delivered instead of parked behind this wait. The
  1912. # process is untouched and still notify-tracked; the model should read the
  1913. # steer text and respond, not re-issue the wait (kimi-code#3697 class).
  1914. result = {
  1915. "status": "interrupted", "command": session.command, "output": _output_tail(session, 1000),
  1916. "process_running": True,
  1917. "note": ("User sent a new message -- wait released; the process is still "
  1918. "running and you will be notified on exit. Respond to the user now.")}
  1919. if result is not None:
  1920. if timeout_note:
  1921. result["timeout_note"] = timeout_note
  1922. return result
  1923. remaining = deadline - time.monotonic()
  1924. if remaining <= 0:
  1925. break
  1926. session._completion_event.wait(timeout=min(1.0, remaining))
  1927. result = {
  1928. "status": "timeout", "command": session.command, "output": _output_tail(session, 1000),
  1929. # Not a failure — models re-issued identical waits after misreading this as an error.
  1930. "process_running": True}
  1931. base_note = (
  1932. f"Wait window of {effective_timeout}s elapsed — the process is still running. This is not an error.")
  1933. if session.started_at:
  1934. base_note += f" Uptime: {int(time.time() - session.started_at)}s."
  1935. base_note += (
  1936. " notify_on_complete is set: you will be notified on exit — do more work instead of waiting again."
  1937. if session.notify_on_complete else
  1938. " Poll again later or use terminal(background=true, "
  1939. "notify_on_complete=true) next time for automatic notification.")
  1940. result["timeout_note"] = f"{timeout_note}. {base_note}" if timeout_note else base_note
  1941. return result
  1942. @staticmethod
  1943. def _exit_snapshot(session: ProcessSession, status: str) -> dict:
  1944. """Result dict for an exited session: exit metadata + the completion-sized output tail."""
  1945. return {
  1946. "status": status, "command": session.command,
  1947. **ProcessRegistry._exit_fields(session), **_completion_output(session)}
  1948. def kill_process(
  1949. self, session_id: str, *, source: str = "process.kill", consume_output: bool = True,
  1950. ) -> dict:
  1951. """Kill a background process and return its output snapshot.
  1952. ``consume_output`` is true for explicit tool/RPC kills (the caller sees the
  1953. output). Bulk cleanup passes false so it doesn't suppress an autonomous
  1954. completion notification — except abandoned-turn reaping (``kill_started_since``),
  1955. which passes true so a killed abandoned process can't revive stopped work."""
  1956. session = self.get(session_id)
  1957. if session is None:
  1958. return _not_found(session_id)
  1959. if session.exited:
  1960. # A double-forked descendant may still be alive in the systemd scope even
  1961. # though the main process exited — stop the scope to reap survivors.
  1962. # See #70716.
  1963. # If the worker was spawned in its own systemd scope (#70716), stop the entire unit to reap any
  1964. # double-forked descendants that were reparented inside the scope and survived the PID signal
  1965. # above (reviewer gap #2). ``systemctl --user stop`` sends SIGTERM to every process in the
  1966. # cgroup and escalates to SIGKILL after TimeoutStopSec. This is additive — the PID-based kill
  1967. # above already handled the main process; this catches stragglers.
  1968. scope_stopped = self._stop_scope(session)
  1969. with session._lock:
  1970. result = self._exit_snapshot(session, "already_exited")
  1971. finalizing = self._reader_finalizing(session)
  1972. if not scope_stopped:
  1973. result["scope_stop_failed"] = session.systemd_unit
  1974. # Only suppress the autonomous turn after its output is present in
  1975. # the explicit kill result, matching wait/log consumption.
  1976. if consume_output and not finalizing:
  1977. self._completion_consumed.add(session_id)
  1978. return result
  1979. try:
  1980. early = self._signal_kill(session, session_id, consume_output)
  1981. if early is not None:
  1982. return early
  1983. # Additive to the PID kill: stopping the scope reaps double-forked
  1984. # descendants reparented inside the cgroup.
  1985. scope_stopped = self._stop_scope(session)
  1986. # Post-kill verification (#115490): the signals above can leave
  1987. # survivors (SIGTERM-ignoring daemons, scope escapees). A kill that
  1988. # leaves a live tree must not write a killed receipt or prune the
  1989. # session — the survivors would become unmanageable. Keep the
  1990. # session running so it stays listed and killable.
  1991. with session._lock:
  1992. signal_race_exited = session.exited
  1993. # A reader that finalised the session mid-signal already proved
  1994. # real exit; only a still-running session needs tree-death proof.
  1995. survivors = [] if signal_race_exited else self._post_kill_survivors(session)
  1996. if survivors:
  1997. alive = ", ".join(map(str, survivors))
  1998. logger.warning(
  1999. "Kill incomplete for %s: %d process(es) still alive (%s) — session kept running",
  2000. session.id, len(survivors), alive)
  2001. return {
  2002. "status": "error",
  2003. "error": (
  2004. f"Kill incomplete: {len(survivors)} process(es) still alive "
  2005. f"({alive}); session running"),
  2006. "session_id": session.id, "survivors": survivors,
  2007. "process_running": True}
  2008. # Descendants outside the root's tree are visible only to the scope; a scope that did
  2009. # not stop leaves nothing proving they are gone, even when the root exited meanwhile.
  2010. # The session keeps its unit, so a later kill retries the stop.
  2011. if not scope_stopped:
  2012. with session._lock:
  2013. root_exited = session.exited
  2014. logger.warning("Kill incomplete for %s: systemd scope %s did not stop",
  2015. session.id, session.systemd_unit)
  2016. return {
  2017. "status": "error",
  2018. "error": f"Kill incomplete: systemd scope {session.systemd_unit} did not stop",
  2019. "session_id": session.id, "scope_stop_failed": session.systemd_unit,
  2020. "process_running": not root_exited}
  2021. # Capture output, mark consumed, THEN expose ``exited`` to watcher tasks —
  2022. # closes the delayed-notification race without losing the transcript.
  2023. with session._lock:
  2024. output = _completion_output(session)
  2025. if consume_output:
  2026. self._completion_consumed.add(session_id)
  2027. session.exited = True
  2028. session.exit_code = -15 # SIGTERM
  2029. session.completion_reason = "killed"
  2030. session.termination_source = source
  2031. # The reader thread can finalise the session while the signal path
  2032. # blocks in the SIGKILL grace window: its ``save_completed_result``
  2033. # then persists this kill as a plain ``exited``. Re-write the receipt
  2034. # so the durable record matches what the caller was told.
  2035. if not self._move_to_finished(session):
  2036. save_completed_result(session)
  2037. self._write_checkpoint()
  2038. return {
  2039. "status": "killed", "session_id": session.id, "completion_reason": session.completion_reason,
  2040. "termination_source": session.termination_source, **output}
  2041. except Exception as e:
  2042. return {"status": "error", "error": str(e)}
  2043. @staticmethod
  2044. def _stop_scope(session: ProcessSession) -> bool:
  2045. """Stop the session's systemd scope: True when it has none or the stop worked. Every kill
  2046. path stops the scope through here, so a failed stop always leaves it marked for retry."""
  2047. if not session.systemd_unit:
  2048. return True
  2049. stopped = _stop_systemd_unit(session.systemd_unit)
  2050. session._scope_stop_pending = not stopped
  2051. return stopped
  2052. def _signal_kill(self, session: ProcessSession, session_id: str, consume_output: bool) -> Optional[dict]:
  2053. """Deliver the kill via PTY, local Popen tree, sandbox exec or recovered host
  2054. PID. Returns a final result dict when the kill cannot proceed (recycled/dead
  2055. recovered PID, or no runtime handle), else None."""
  2056. if session._pty:
  2057. try:
  2058. session._pty.terminate(force=True)
  2059. except Exception:
  2060. if session.pid:
  2061. os.kill(session.pid, signal.SIGTERM)
  2062. elif session.process:
  2063. # Tree kill: on Windows Popen.terminate() only kills the shell wrapper and
  2064. # leaves Git Bash descendants behind.
  2065. self._terminate_host_pid(session.process.pid, session.host_start_time)
  2066. elif session.env_ref and session.pid:
  2067. session.env_ref.execute(f"kill {session.pid} 2>/dev/null", timeout=5)
  2068. elif session.detached and session.pid_scope == "host" and session.pid:
  2069. # Same fate as poll/list: a gone or reused PID means our process is
  2070. # gone — never tree-kill the stranger — but a live PID with an
  2071. # unreadable start time is still ours and must really be killed.
  2072. # If this recovered session also carries an owned systemd scope, stop that scope before
  2073. # returning: a daemonized descendant may still be alive there even though the wrapper PID exited
  2074. # or was recycled across the gateway restart (#70716, teknium1 review).
  2075. if self._detached_host_fate(session.pid, session.host_start_time) != "running":
  2076. scope_stopped = self._stop_scope(session)
  2077. with session._lock:
  2078. output = _completion_output(session)
  2079. if consume_output:
  2080. self._completion_consumed.add(session_id)
  2081. # No waitable handle, so this is not a collected exit. Close the
  2082. # entry without queueing a completion, and do not signal a PID
  2083. # whose start time does not match.
  2084. self._close_reused_detached(session)
  2085. result = {"status": "already_exited", "exit_code": session.exit_code, **output}
  2086. if not scope_stopped:
  2087. result["scope_stop_failed"] = session.systemd_unit
  2088. return result
  2089. # Identity was just proven above. Re-passing the start time would make
  2090. # an unreadable probe refuse the kill and leave the re-adopted child running.
  2091. self._terminate_host_pid(session.pid)
  2092. else:
  2093. return {
  2094. # Reject non-positive timeouts — the schema declares minimum=1, but not every caller
  2095. # enforces schemas before dispatch. timeout=0 is falsy, so without this guard it silently
  2096. # fell through (`0 or max_timeout`) to the DEFAULT wait instead of erroring. Salvaged from
  2097. # PR #60004 (credit @isheng-eqi).
  2098. "status": "error",
  2099. "error": "Recovered process cannot be killed after restart because "
  2100. "its original runtime handle is no longer available",
  2101. }
  2102. return None
  2103. def _stdin_op(self, session_id: str, pty_op, pipe_op, ok: dict) -> dict:
  2104. """``pty_op(pty)`` under PTY mode, else ``pipe_op(stdin)`` on the Popen pipe; *ok* on success."""
  2105. from tools.process_registry_stdin import already_exited_error, stdin_unavailable_error
  2106. session = self.get(session_id)
  2107. if session is None:
  2108. return _not_found(session_id)
  2109. if session.exited:
  2110. return {"status": "already_exited", "error": already_exited_error(session)}
  2111. try:
  2112. if session._pty:
  2113. pty_op(session._pty)
  2114. elif not session.process or not session.process.stdin:
  2115. return {"status": "error", "error": stdin_unavailable_error(session)}
  2116. else:
  2117. pipe_op(session.process.stdin)
  2118. return ok
  2119. except Exception as e:
  2120. return {"status": "error", "error": str(e)}
  2121. def write_stdin(self, session_id: str, data: str) -> dict:
  2122. """Send raw data to a running process's stdin (no newline appended)."""
  2123. def via_pty(pty):
  2124. # pywinpty expects str on Windows; ptyprocess expects bytes on POSIX.
  2125. if _IS_WINDOWS:
  2126. pty.write(data.decode("utf-8") if isinstance(data, bytes) else str(data))
  2127. else:
  2128. # surrogateescape: a PTY is a byte stream — round-trip the original
  2129. # bytes instead of crashing on surrogate content.
  2130. pty.write(data.encode("utf-8", "surrogateescape") if isinstance(data, str) else data)
  2131. def via_pipe(stdin):
  2132. stdin.write(data)
  2133. stdin.flush()
  2134. return self._stdin_op(session_id, via_pty, via_pipe, {"status": "ok", "bytes_written": len(data)})
  2135. def submit_stdin(self, session_id: str, data: str = "") -> dict:
  2136. """Send data + Enter. On a PTY Enter is the CR a keyboard sends: raw-mode TUIs (prompt_toolkit,
  2137. Ink) read ``\\n`` as Ctrl+J and canonical children still get ``\\n`` via ICRNL; a Windows PTY
  2138. gets ``\\r\\n`` (a bare ``\\n`` through pywinpty never ends a ConPTY line read). Pipes keep ``\\n``."""
  2139. session = self.get(session_id)
  2140. pty_enter = "\r\n" if _IS_WINDOWS else "\r"
  2141. return self.write_stdin(session_id, data + (pty_enter if session is not None and session._pty else "\n"))
  2142. def request_close_terminal(self, session_id: str) -> dict:
  2143. """Ask the desktop GUI to close this process's read-only terminal tab. Does NOT
  2144. kill the process — output keeps buffering and the tab can be reopened from the
  2145. status stack. Errors when no UI close sink is wired."""
  2146. if self.on_close is None:
  2147. return {"status": "error", "error": "close_terminal is only available in the Hermes desktop app."}
  2148. # The session may already be finished (or pruned) — the tab can still
  2149. # linger and be closed, so a missing session is not an error here.
  2150. try:
  2151. self.on_close(self.get(session_id), session_id)
  2152. except Exception as e:
  2153. return {"status": "error", "error": str(e)}
  2154. return {
  2155. "status": "ok", "closed": session_id,
  2156. "note": "Closed the read-only terminal tab. The process was not killed; "
  2157. "its output remains available and the user can reopen the tab "
  2158. "from the status stack."}
  2159. def close_stdin(self, session_id: str) -> dict:
  2160. """Close a running process's stdin / send EOF without killing the process."""
  2161. session = self.get(session_id)
  2162. msg = "EOF sent" if session is not None and session._pty else "stdin closed"
  2163. return self._stdin_op(
  2164. session_id, lambda pty: pty.sendeof(), lambda stdin: stdin.close(), {"status": "ok", "message": msg})
  2165. def count_running(self) -> int:
  2166. """O(1) running count for status-bar polling; dict ``len()`` is atomic, no lock."""
  2167. return len(self._running)
  2168. def list_sessions(self, task_id: str | None = None, session_key: str | None = None, *, include_retained: bool = False) -> list:
  2169. """Running and recently-finished processes for ``task_id`` and/or ``session_key``;
  2170. cross-task entries sharing the gateway session (a forgotten preview server
  2171. blocking session reset) are flagged ``"session_scoped": true``.
  2172. When ``task_id`` is given, processes that task spawned (its ``owner_task_id``) are included. When
  2173. ``session_key`` is also given, session-scoped background processes (``background: true``) registered
  2174. under that gateway session are surfaced too, even if they belong to a different task — so the agent
  2175. can discover a forgotten preview server that is blocking session reset (#29177).
  2176. """
  2177. # Only an explicit tool query reads historical receipts. Status bars and
  2178. # gateway liveness scans call this frequently and need the live registry.
  2179. sessions = load_completed_results() if include_retained else {}
  2180. with self._lock:
  2181. sessions.update(self._finished)
  2182. sessions.update(self._running)
  2183. all_sessions = [
  2184. refreshed for refreshed in (
  2185. self._refresh_detached_session(s) for s in sessions.values()
  2186. ) if refreshed is not None and not self._uncollected_gone(refreshed)
  2187. ]
  2188. if task_id or session_key:
  2189. all_sessions = [
  2190. s for s in all_sessions
  2191. if (task_id and s.owner_task_id == task_id)
  2192. or (session_key and s.session_key == session_key)
  2193. ]
  2194. result = []
  2195. for s in all_sessions:
  2196. # List-only refreshes must observe child exit even while descendants
  2197. # keep the capture pipe open; retain the existing completion owner.
  2198. self._reconcile_local_exit(s)
  2199. entry = {
  2200. "session_id": s.id,
  2201. "command": s.command[:200],
  2202. "cwd": s.cwd,
  2203. "pid": s.pid,
  2204. "owner_task_id": s.owner_task_id,
  2205. "started_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(s.started_at)),
  2206. "uptime_seconds": int(time.time() - s.started_at),
  2207. "status": "exited" if s.exited else "running",
  2208. "output_preview": s.output_buffer[-200:] if s.output_buffer else "",
  2209. }
  2210. # Flag processes surfaced only because they share the gateway session (not the current task) —
  2211. # these are the long-lived background processes a user may have forgotten about (#29177).
  2212. if task_id and session_key and s.owner_task_id != task_id and s.session_key == session_key:
  2213. entry["session_scoped"] = True
  2214. if s.wsl_chain:
  2215. entry["wsl_chain"] = True
  2216. entry["wsl_note"] = _WSL_CHAIN_NOTE
  2217. # Trigger metadata for goal-loop judges (a watcher may never exit).
  2218. if s.watch_patterns and not s._watch_disabled:
  2219. entry.update(watch_patterns=list(s.watch_patterns), watch_hit=s._watch_hits > 0)
  2220. if s.notify_on_complete:
  2221. entry["notify_on_complete"] = True
  2222. if s.persist_on_release:
  2223. entry["persist_on_release"] = True
  2224. if s.exited:
  2225. entry["exit_code"] = s.exit_code
  2226. entry["exited_at"] = s.exited_at
  2227. entry["completion_reason"] = s.completion_reason
  2228. if s.detached:
  2229. entry["detached"] = True
  2230. result.append(entry)
  2231. return result
  2232. # ----- Session/Task Queries (for gateway integration) -----
  2233. def _any_running(self, predicate) -> bool:
  2234. """True if any still-running session satisfies *predicate*, after refreshing
  2235. detached sessions so a finished-but-unreaped process reads as inactive."""
  2236. with self._lock:
  2237. sessions = list(self._running.values())
  2238. for session in sessions:
  2239. self._refresh_detached_session(session)
  2240. with self._lock:
  2241. return any(not s.exited and predicate(s) for s in self._running.values())
  2242. def has_active_processes(self, task_id: str) -> bool:
  2243. """Whether any process for ``task_id`` is still running. Ownership is
  2244. ``owner_task_id`` (the raw spawning id) like the other task-scoped queries:
  2245. ``task_id`` on a session is the collapsed container key, shared across
  2246. turns and delegate children, so a container-key match alone would miss a
  2247. delegate child's own background work (#120546)."""
  2248. return self._any_running(lambda s: s.owner_task_id == task_id)
  2249. def running_owned_by(self, owner_task_id: str) -> list[ProcessSession]:
  2250. """Running processes whose RAW spawning owner is ``owner_task_id``."""
  2251. with self._lock:
  2252. return [s for s in self._running.values() if s.owner_task_id == owner_task_id and not s.exited]
  2253. def unread_completions_owned_by(self, owner_task_id: str) -> list[ProcessSession]:
  2254. """Exited ``notify_on_complete`` processes of ``owner_task_id`` whose result nobody read (no wait/log/poll).
  2255. A child's completion notice is suppressed in the parent, so an unread exit is otherwise lost silently."""
  2256. with self._lock:
  2257. return [s for s in self._finished.values()
  2258. if s.owner_task_id == owner_task_id and s.notify_on_complete
  2259. and s.exit_code is not None
  2260. and s.id not in self._completion_consumed and s.id not in self._poll_observed]
  2261. def transfer_ownership(self, session_id: str, *, from_owner: str, to_owner: str, to_task_id: str,
  2262. to_session_key: str, note: str = "") -> Optional[ProcessSession]:
  2263. """Move a RUNNING process from one owner to another under the registry lock. Ownership is the ``owner_task_id``
  2264. field: completion notices are stamped from it at exit time and teardown kills by it, so flipping it here is the
  2265. whole transfer. Returns the session, or None when it is unknown, already exited, or not owned by ``from_owner``
  2266. (the caller must not report a transfer that did not happen)."""
  2267. session = self.get(session_id)
  2268. with self._lock:
  2269. if session is None or session.exited or session.owner_task_id != from_owner:
  2270. return None
  2271. session.owner_task_id = to_owner
  2272. session.task_id = to_task_id
  2273. session.session_key = to_session_key
  2274. session.handoff_note = note
  2275. return session
  2276. def has_active_for_session(self, session_key: str, max_active_age: Optional[float] = None) -> bool:
  2277. """Active processes for a gateway session key. Processes older than
  2278. ``max_active_age`` seconds are ignored as stale so a forgotten ``http.server``
  2279. can't freeze session idle/daily reset forever; ``None`` keeps legacy behaviour
  2280. (any running process blocks)."""
  2281. now = time.time()
  2282. return self._any_running(
  2283. lambda s: s.session_key == session_key
  2284. and (max_active_age is None or (now - s.started_at) < max_active_age))
  2285. def has_any_active(self) -> bool:
  2286. """Whether ANY background process is running — scale-to-zero must not
  2287. suspend a gateway with live background work or the process is lost."""
  2288. return self._any_running(lambda s: True)
  2289. def snapshot_running_ids(self, task_id: str) -> frozenset[str]:
  2290. """Running IDs owned by ``task_id`` — a turn-boundary marker: on timeout
  2291. only processes absent from the starting snapshot belong to the abandoned
  2292. turn; older ones intentionally span turns and must survive. Ownership is
  2293. ``owner_task_id``: ``task_id`` is the container key (``session:<key>``,
  2294. ``default``), shared across turns and sessions, not the turn's id."""
  2295. return frozenset(s.id for s in self.running_owned_by(task_id))
  2296. # Kill sources that are agent-lifecycle cleanup (session end / compression / error
  2297. # recovery / turn abandon), not a deliberate stop: those must skip persist_on_release
  2298. # sessions. An explicit stop (process_manage kill, CLI /stop) passes its own source
  2299. # string and still reaches them.
  2300. _LIFECYCLE_KILL_SOURCES = frozenset({"kill_all", "gateway_turn_timeout", "agent_close"})
  2301. def kill_started_since(self, task_id: str, baseline_ids, *, source: str) -> int:
  2302. """Kill ``task_id`` processes created after ``baseline_ids``. Output is
  2303. consumed so an abandoned turn can't enqueue a follow-up reviving work the
  2304. timeout deliberately stopped."""
  2305. return self.kill_all(task_id, exclude_ids=frozenset(baseline_ids or ()), source=source, consume_output=True)
  2306. def kill_all(
  2307. self, task_id: Optional[str] = None, *, exclude_ids: frozenset = frozenset(),
  2308. source: str = "kill_all", consume_output: bool = False) -> int:
  2309. """Kill all running processes, optionally only those ``task_id`` spawned (its ``owner_task_id``).
  2310. Returns how many it stopped. A session whose systemd scope did not stop is not counted
  2311. (its descendants may live on), and a later sweep retries it even after it finished.
  2312. Sessions with ``persist_on_release=True`` are skipped when ``source`` is an
  2313. agent-lifecycle sweep (the default ``kill_all`` release path, a gateway turn
  2314. timeout, or ``agent_close``): an explicitly persisted background job must survive
  2315. session end / compression / error recovery (#41225). An explicit operator stop
  2316. (``process.kill`` via process_manage, CLI /stop) passes its own source and still
  2317. reaches them, so the user can always stop a persisted process on purpose."""
  2318. lifecycle = source in self._LIFECYCLE_KILL_SOURCES
  2319. with self._lock:
  2320. targets = [
  2321. s for s in (*self._running.values(), *self._finished.values())
  2322. if (task_id is None or s.owner_task_id == task_id)
  2323. and s.id not in exclude_ids and (not s.exited or s._scope_stop_pending)
  2324. and not (lifecycle and s.persist_on_release)
  2325. ]
  2326. results = [self.kill_process(s.id, source=source, consume_output=consume_output) for s in targets]
  2327. return sum(
  2328. r.get("status") in {"killed", "already_exited"} and "scope_stop_failed" not in r
  2329. for r in results)
  2330. # ----- Cleanup / Pruning -----
  2331. def _prune_if_needed(self):
  2332. """Drop expired, then oldest-over-cap finished sessions (hold _lock); keep pending scope stops and unpublished notices."""
  2333. from tools.process_registry_awaiting import retained_for_delivery
  2334. now = time.time()
  2335. prunable = {sid: s for sid, s in self._finished.items()
  2336. if not getattr(s, "_scope_stop_pending", False) and not retained_for_delivery(s)}
  2337. expired = [sid for sid, s in prunable.items() if (now - s.started_at) > FINISHED_TTL_SECONDS]
  2338. over_cap = len(self._running) + len(self._finished) - len(expired) >= MAX_PROCESSES
  2339. if over_cap and (survivors := [sid for sid in prunable if sid not in expired]):
  2340. expired.append(min(survivors, key=lambda sid: self._finished[sid].started_at))
  2341. for sid in expired:
  2342. # Belt-and-suspenders handle release: sessions normally arrive in
  2343. # _finished via _move_to_finished(), which already released their
  2344. # Popen/PTY handles — but any session inserted into _finished
  2345. # directly (defensive paths, historical checkpoints) would
  2346. # otherwise carry its OS handles to the grave unreleased. The
  2347. # release is idempotent, so double-closing is safe.
  2348. self._release_finished_handles(self._finished[sid])
  2349. del self._finished[sid]
  2350. # Belt-and-suspenders against module-lifetime growth: forget consumed /
  2351. # poll-observed marks for any session no longer tracked at all.
  2352. tracked = self._running.keys() | self._finished.keys()
  2353. self._completion_consumed &= tracked
  2354. self._poll_observed &= tracked
  2355. process_registry = ProcessRegistry()
  2356. # --- the "process_manage" tool schema + handler -----------------------------------
  2357. from tools.registry import registry, tool_error
  2358. PROCESS_SCHEMA = {
  2359. "name": "process_manage",
  2360. # The enum names the verbs; the description keeps only non-obvious semantics
  2361. # (write-vs-submit is the one real trap: a lone \n on a Windows PTY is not Enter).
  2362. # See #95681.
  2363. "description": (
  2364. "Poll, wait on, send input to, or kill terminal(background=true) processes. "
  2365. "Completed results remain retrievable by session_id when resuming their owning conversation "
  2366. "(up to 7 days, newest 64 results per profile; rolling output tail). "
  2367. "poll: status + new output. log: full output, paged. wait: block "
  2368. "until exit or timeout (partial output on timeout). write vs "
  2369. "submit: submit appends Enter — use it to answer prompts; write "
  2370. "sends raw bytes, no newline. close: EOF stdin. kill: terminate. "
  2371. "handoff (subagents only): transfer a running process you started to your parent agent, which then "
  2372. "receives its completion; `data` = one sentence on its purpose. Subagent-owned processes are otherwise "
  2373. "killed when the subagent finishes and their notifications never reach the parent."
  2374. ),
  2375. "parameters": {
  2376. "type": "object",
  2377. "properties": {
  2378. "action": {
  2379. "type": "string",
  2380. "enum": ["list", "poll", "log", "wait", "kill", "write", "submit", "close", "handoff"]
  2381. },
  2382. "session_id": {
  2383. "type": "string",
  2384. "description": "From terminal background output; any unique prefix works ('4dae' for proc_4dae56ca81f6). Required except for 'list'."
  2385. },
  2386. "data": {
  2387. "type": "string",
  2388. "description": "Stdin text for write/submit; purpose sentence for handoff."
  2389. },
  2390. "timeout": {
  2391. "type": "integer",
  2392. "description": "Max seconds for 'wait'.",
  2393. "minimum": 1
  2394. },
  2395. "offset": {
  2396. "type": "integer",
  2397. "description": "Log line offset (default: last 200)."
  2398. },
  2399. "limit": {
  2400. "type": "integer",
  2401. "description": "Max log lines.",
  2402. "minimum": 1
  2403. }
  2404. },
  2405. "required": ["action"]
  2406. }
  2407. }
  2408. def transform_process_output(output: str, *, command: str, returncode: Optional[int], task_id: str = "") -> str:
  2409. """``transform_terminal_output`` seam for background-process output — the poll/wait/log/kill
  2410. results and the completion/heartbeat/watch notifications — so a plugin that rewrites terminal
  2411. output sees the same command output whether it ran in the foreground or not (#70760).
  2412. Same helper as the foreground path; callers redact AFTER it, never before, so a replacement
  2413. the plugin returns is still masked. ``returncode`` is None while the process is running and
  2414. ``env_type`` is not recorded per process, so it is passed empty."""
  2415. from tools.terminal_tool_result import _apply_output_transform_hook
  2416. return _apply_output_transform_hook(command, output, returncode, task_id or "", "")
  2417. def _redact_process_result(result: dict) -> dict:
  2418. """Transform, then redact secrets from background-process output before it reaches the
  2419. model, session.db and CLI, mirroring the foreground ``terminal`` pipeline (hook first,
  2420. redaction after) so the two surfaces can't diverge. Respects ``security.redact_secrets``;
  2421. ``redact_terminal_output`` picks ``code_file`` from the recorded command.
  2422. The command string itself is also redacted in case it carried an inline credential. See #43025.
  2423. """
  2424. if not isinstance(result, dict):
  2425. return result
  2426. from agent.redact import redact_sensitive_text, redact_terminal_output
  2427. command = result.get("command") or ""
  2428. # The hook's task_id is the process OWNER's (poll/log/wait results carry only session_id).
  2429. task_id = str(result.get("task_id") or "")
  2430. if not task_id and (session := process_registry.get(str(result.get("session_id") or ""))) is not None:
  2431. task_id = str(getattr(session, "task_id", "") or "")
  2432. for key in ("output", "output_preview"):
  2433. if isinstance(value := result.get(key), str) and value:
  2434. value = transform_process_output(value, command=command, returncode=result.get("exit_code"), task_id=task_id)
  2435. result[key] = redact_terminal_output(value, command)
  2436. if isinstance(command, str) and command:
  2437. result["command"] = redact_sensitive_text(command, code_file=True)
  2438. return result
  2439. def _list_processes(task_id) -> dict:
  2440. # Also surface session-scoped background processes (e.g. a forgotten preview
  2441. # server): they share the gateway session_key and can block session reset.
  2442. session_key = ""
  2443. with suppress(Exception):
  2444. # See #29177.
  2445. from tools.approval_context import get_current_session_key
  2446. session_key = get_current_session_key(default="") or ""
  2447. return {"processes": [
  2448. _redact_process_result(p)
  2449. for p in process_registry.list_sessions(
  2450. task_id=task_id, session_key=session_key or None, include_retained=True)]}
  2451. # action -> (handler(session_id, args) -> dict, redact output?). Output-bearing
  2452. # actions are redacted; stdin actions return only status.
  2453. _SESSION_ACTIONS = {
  2454. "poll": (lambda sid, a: process_registry.poll(sid), True),
  2455. "log": (lambda sid, a: process_registry.read_log(sid, offset=a.get("offset"), limit=a.get("limit", 200)), True),
  2456. "wait": (lambda sid, a: process_registry.wait(sid, timeout=a.get("timeout")), True),
  2457. "kill": (lambda sid, a: process_registry.kill_process(sid), True),
  2458. "write": (lambda sid, a: process_registry.write_stdin(sid, str(a.get("data", ""))), False),
  2459. "submit": (lambda sid, a: process_registry.submit_stdin(sid, str(a.get("data", ""))), False),
  2460. "close": (lambda sid, a: process_registry.close_stdin(sid), False),
  2461. }
  2462. def _handoff_process(session_id: str, args: dict, task_id: Optional[str]) -> dict:
  2463. """Subagent-only: transfer a running background process to the parent agent so its completion is delivered THERE
  2464. (child-owned process notices are suppressed and child teardown kills what it owns). Validated against the live spawn
  2465. tree: the caller must be a registered child and must own the process; anything else is an error, never a silent
  2466. no-op, so a PID mentioned in prose can't masquerade as a transfer."""
  2467. from tools.delegate_tool_registry import _active_subagents, _active_subagents_lock
  2468. from tools.terminal_tool import _resolve_container_task_id
  2469. with _active_subagents_lock:
  2470. record = _active_subagents.get(str(task_id or ""))
  2471. child = record.get("agent") if record else None
  2472. parent_ref = getattr(child, "_delegate_parent_ref", None)
  2473. parent = parent_ref() if callable(parent_ref) else None
  2474. if parent is None:
  2475. return {"error": "handoff is only available to a running subagent with a live parent; you are not one."}
  2476. parent_owner = str(getattr(parent, "_current_task_id", "") or getattr(parent, "session_id", "") or "")
  2477. if not parent_owner:
  2478. return {"error": "parent has no process owner id yet; retry after the parent's turn has started."}
  2479. handed = getattr(child, "_handed_off_processes", None)
  2480. if handed is None:
  2481. handed = child._handed_off_processes = []
  2482. if len(handed) >= _MAX_HANDOFFS_PER_CHILD:
  2483. return {"error": f"handoff cap reached ({_MAX_HANDOFFS_PER_CHILD} per subagent); wait on or kill the rest yourself."}
  2484. note = str(args.get("data") or "").strip()
  2485. if not note:
  2486. return {"error": "handoff requires `data`: one sentence saying what the process is for and what the parent should do with its result."}
  2487. session = process_registry.transfer_ownership(
  2488. session_id, from_owner=str(task_id or ""), to_owner=parent_owner,
  2489. to_task_id=_resolve_container_task_id(parent_owner),
  2490. to_session_key=str(getattr(parent, "session_id", "") or ""), note=note)
  2491. if session is None:
  2492. return {"error": f"cannot hand off {session_id}: not a running process you own (already exited? read its result "
  2493. "with poll/log and report it instead)."}
  2494. handed.append({"session_id": session.id, "command": session.command, "note": note})
  2495. return {"status": "handed_off", "session_id": session.id, "command": session.command,
  2496. "note": "Your parent now owns this process and will receive its completion; you will not. Mention the handoff "
  2497. "in your final answer."}
  2498. _MAX_HANDOFFS_PER_CHILD = 3
  2499. def _handle_process(args, **kw):
  2500. action = args.get("action", "")
  2501. # Coerce to string — some models send session_id as an integer
  2502. session_id = str(args.get("session_id", "")) if args.get("session_id") is not None else ""
  2503. if action == "list":
  2504. return json.dumps(_list_processes(kw.get("task_id")), ensure_ascii=False)
  2505. if action == "handoff":
  2506. if not session_id:
  2507. return tool_error("session_id is required for handoff")
  2508. return json.dumps(_handoff_process(session_id, args, kw.get("task_id")), ensure_ascii=False)
  2509. if action in _SESSION_ACTIONS:
  2510. if not session_id:
  2511. return tool_error(f"session_id is required for {action}")
  2512. handler, redact = _SESSION_ACTIONS[action]
  2513. from tools.process_registry_wait_policy import apply_wait_poll_policy
  2514. result = apply_wait_poll_policy(action, session_id, handler(session_id, args))
  2515. return json.dumps(_redact_process_result(result) if redact else result, ensure_ascii=False)
  2516. return tool_error(f"Unknown process action: {action}. Use: list, poll, log, wait, kill, write, submit, close, handoff")
  2517. registry.register(
  2518. name="process_manage",
  2519. toolset="terminal",
  2520. schema=PROCESS_SCHEMA,
  2521. handler=_handle_process,
  2522. emoji="⚙️",
  2523. )