base_output.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486
  1. """Process-handle plumbing for ``tools.environments.base``.
  2. Bounded output capture (head/tail window + disk spill), the ``ProcessHandle``
  3. duck type, the SDK adapter ``_ThreadedProcessHandle``, stdin piping, and the
  4. stdout drain thread used by ``BaseEnvironment._wait_for_process``.
  5. """
  6. import codecs
  7. import os
  8. import subprocess
  9. import threading
  10. import time
  11. from collections import deque
  12. from pathlib import Path
  13. from typing import IO, Callable, Protocol
  14. from hermes_constants import get_hermes_home
  15. from tools.environments.fd_wait import open_fd_waiter
  16. from tools.tool_output_truncate import head_tail_split, truncation_notice
  17. from hermes_cli._subprocess_compat import windows_hide_flags
  18. # Sentinel capacity for full-fidelity capture: large enough that the collector
  19. # never evicts, so bounded and unbounded modes share one code path.
  20. _UNBOUNDED_CAPTURE_CHARS = 2**63 - 1
  21. _SPILL_MAX_AGE_S = 7 * 86400
  22. class _BoundedOutputCollector:
  23. """Retain a bounded 40/60 head-tail window of streamed text. When ``spill_path`` is set,
  24. the FULL stream is also teed to that file once eviction begins (up to ``_SPILL_CAP_CHARS``)
  25. so a truncated foreground result is recoverable without re-running."""
  26. # Hard ceiling on spill file size; protects disk from runaway output.
  27. _SPILL_CAP_CHARS = 5_000_000
  28. def __init__(self, max_chars: int, spill_path: Path | None = None):
  29. self.max_chars = max(1, int(max_chars))
  30. self._head_limit = int(self.max_chars * 0.4)
  31. self._tail_limit = self.max_chars - self._head_limit
  32. self._head: list[str] = []
  33. self._tail: deque[str] = deque()
  34. self._head_chars = 0
  35. self._tail_chars = 0
  36. self._total_chars = 0
  37. self._lock = threading.Lock()
  38. self._spill_path = spill_path
  39. self._spill_fh: IO[str] | None = None
  40. self._spill_chars = 0
  41. self._spill_capped = False
  42. def _maybe_spill(self, text: str) -> None:
  43. """Tee ``text`` to the spill file (opened lazily on first overflow)."""
  44. if self._spill_path is None or self._spill_capped:
  45. return
  46. try:
  47. if self._spill_fh is None:
  48. from tools.spill_safety import ensure_spill_dir, open_exclusive
  49. # Raw pre-redaction output: private perms + symlink-refusing
  50. # exclusive create (a planted link must fail, never redirect).
  51. ensure_spill_dir(self._spill_path.parent, private=True)
  52. self._spill_fh = open_exclusive(self._spill_path, private=True, errors="replace")
  53. # Backfill what's retained so the file holds the stream from byte 0.
  54. backlog = "".join(self._head) + "".join(self._tail)
  55. self._spill_fh.write(backlog)
  56. self._spill_chars = len(backlog)
  57. budget = self._SPILL_CAP_CHARS - self._spill_chars
  58. if budget <= 0 or len(text) > budget:
  59. self._spill_fh.write(text[:max(0, budget)])
  60. self._spill_fh.write("\n... [spill capped at 5,000,000 chars] ...\n")
  61. self._spill_capped = True
  62. else:
  63. self._spill_fh.write(text)
  64. self._spill_chars += len(text)
  65. except OSError:
  66. # Disk trouble must never break command execution.
  67. self._spill_capped = True
  68. def close_spill(self) -> str | None:
  69. """Close the spill file and return its path if it was used."""
  70. with self._lock:
  71. if self._spill_fh is None:
  72. return None
  73. try:
  74. self._spill_fh.close()
  75. except OSError:
  76. pass
  77. self._spill_fh = None
  78. return str(self._spill_path)
  79. @property
  80. def buffered_chars(self) -> int:
  81. with self._lock:
  82. return self._head_chars + self._tail_chars
  83. @property
  84. def total_chars(self) -> int:
  85. with self._lock:
  86. return self._total_chars
  87. def append(self, text: str) -> None:
  88. if not text:
  89. return
  90. with self._lock:
  91. text_len = len(text)
  92. # Spill tee activates at the first overflow, then mirrors every chunk.
  93. if self._spill_path is not None and (
  94. self._spill_fh is not None or self._total_chars + text_len > self.max_chars):
  95. self._maybe_spill(text)
  96. self._total_chars += text_len
  97. start = 0
  98. if self._head_chars < self._head_limit:
  99. take = min(self._head_limit - self._head_chars, text_len)
  100. if take:
  101. self._head.append(text[:take])
  102. self._head_chars += take
  103. start = take
  104. remaining = text_len - start
  105. if remaining <= 0 or self._tail_limit <= 0:
  106. return
  107. if remaining >= self._tail_limit:
  108. self._tail.clear()
  109. self._tail.append(text[-self._tail_limit :])
  110. self._tail_chars = self._tail_limit
  111. return
  112. chunk = text[start:]
  113. self._tail.append(chunk)
  114. self._tail_chars += len(chunk)
  115. while (excess := self._tail_chars - self._tail_limit) > 0:
  116. first = self._tail[0]
  117. if len(first) <= excess:
  118. self._tail.popleft()
  119. self._tail_chars -= len(first)
  120. else:
  121. self._tail[0] = first[excess:]
  122. self._tail_chars -= excess
  123. def render(self, *, suffix: str = "") -> str:
  124. """Render within ``max_chars``, preserving a required status suffix."""
  125. with self._lock:
  126. if len(suffix) >= self.max_chars:
  127. return suffix[-self.max_chars :]
  128. head = "".join(self._head)
  129. tail = "".join(self._tail)
  130. available = self.max_chars - len(suffix)
  131. if self._total_chars <= available:
  132. return head + tail + suffix
  133. # The notice length depends on the omitted count, which depends on
  134. # the notice length; iterate to a fixed point (converges quickly).
  135. notice = ""
  136. for _ in range(4):
  137. omitted = max(0, self._total_chars - max(0, available - len(notice)))
  138. updated = truncation_notice(omitted, self._total_chars)
  139. if updated == notice:
  140. break
  141. notice = updated
  142. content_budget = max(0, available - len(notice))
  143. head_chars, tail_chars = head_tail_split(content_budget)
  144. rendered_tail = tail[-tail_chars:] if tail_chars else ""
  145. return head[:head_chars] + notice[:available] + rendered_tail + suffix
  146. def _new_output_collector(proc, bounded_capture: bool) -> _BoundedOutputCollector:
  147. """Build the collector for one ``_wait_for_process`` call. ``bounded_capture`` (foreground
  148. terminal path only) caps retention at ``tool_output.max_bytes`` and tees overflow to a
  149. spill file under ``$HERMES_HOME/cache/terminal-output`` (created only on actual overflow;
  150. spills older than 7 days are pruned opportunistically). Otherwise the collector is
  151. effectively unbounded so internal consumers keep full-fidelity output."""
  152. if not bounded_capture:
  153. return _BoundedOutputCollector(_UNBOUNDED_CAPTURE_CHARS)
  154. try:
  155. from tools.tool_output_limits import get_max_bytes
  156. capture_limit = get_max_bytes()
  157. except Exception:
  158. capture_limit = 50_000
  159. spill_path = None
  160. try:
  161. spill_dir = get_hermes_home() / "cache" / "terminal-output"
  162. spill_path = spill_dir / f"out-{int(time.time())}-{os.getpid()}-{id(proc) & 0xffff:x}.log"
  163. if spill_dir.is_dir():
  164. cutoff = time.time() - _SPILL_MAX_AGE_S
  165. for old in spill_dir.glob("out-*.log"):
  166. try:
  167. if old.stat().st_mtime < cutoff:
  168. old.unlink()
  169. except OSError:
  170. pass
  171. except Exception:
  172. spill_path = None
  173. return _BoundedOutputCollector(capture_limit, spill_path=spill_path)
  174. def _finalize_wait_result(collector: _BoundedOutputCollector, rendered: str, returncode: int | None) -> dict:
  175. """Assemble a wait result, attaching spill metadata when overflow occurred."""
  176. result = {"output": rendered, "returncode": returncode}
  177. spill = collector.close_spill()
  178. if spill:
  179. result["output_total_chars"] = collector.total_chars
  180. result["full_output_path"] = spill
  181. return result
  182. # --- Stdin / spawn helpers ---
  183. def _pipe_stdin(proc: subprocess.Popen, data: str) -> None:
  184. """Write *data* to proc.stdin on a daemon thread to avoid pipe-buffer deadlocks.
  185. Writes go through ``proc.stdin.buffer`` as UTF-8 bytes we encode ourselves: Windows
  186. text-mode stdin would translate ``\\n`` -> ``\\r\\n`` and corrupt every write_file/patch
  187. payload. Encoding uses ``surrogateescape`` (exact inverse of the read-side decode);
  188. surrogates outside U+DC80-U+DCFF raise, the error is recorded on
  189. ``proc._hermes_stdin_errors`` (surfaced by ``_wait_for_process`` as ``stdin_error``) and
  190. stdin is still closed in ``finally`` so the child sees EOF instead of hanging.
  191. """
  192. errors: list[BaseException] = []
  193. proc._hermes_stdin_errors = errors
  194. def _write():
  195. if proc.stdin is None:
  196. errors.append(RuntimeError("process stdin unavailable"))
  197. return
  198. # Resolve the target BEFORE encoding: a failed encode must still
  199. # reach the finally-close, or the child hangs on EOF forever.
  200. target = getattr(proc.stdin, "buffer", proc.stdin)
  201. try:
  202. raw = data.encode("utf-8", "surrogateescape") if isinstance(data, str) else data
  203. written = target.write(raw)
  204. if written != len(raw):
  205. # A short write from a buffered writer is a real failure.
  206. raise RuntimeError(f"short stdin write: {written} of {len(raw)} bytes")
  207. except (BrokenPipeError, OSError):
  208. pass # child closed stdin early — normal
  209. except Exception as exc:
  210. # Only reachable with out-of-range surrogates (e.g. literal U+D800).
  211. errors.append(exc)
  212. finally:
  213. try:
  214. target.close()
  215. except Exception:
  216. pass
  217. thread = threading.Thread(target=_write, daemon=True)
  218. proc._hermes_stdin_thread = thread
  219. thread.start()
  220. def _popen_bash(cmd: list[str], stdin_data: str | None = None, **kwargs) -> subprocess.Popen:
  221. """Spawn a subprocess with standard stdout/stderr/stdin setup; *stdin_data* is written
  222. asynchronously via :func:`_pipe_stdin`. Backends with special Popen needs (e.g. local's
  223. ``preexec_fn``) can bypass this and call :func:`_pipe_stdin` directly."""
  224. kwargs.setdefault("creationflags", windows_hide_flags())
  225. proc = subprocess.Popen(
  226. cmd,
  227. stdout=subprocess.PIPE,
  228. stderr=subprocess.STDOUT,
  229. stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL,
  230. text=True, encoding="utf-8", errors="replace",
  231. **kwargs)
  232. if stdin_data is not None:
  233. _pipe_stdin(proc, stdin_data)
  234. return proc
  235. # --- ProcessHandle protocol ---
  236. class ProcessHandle(Protocol):
  237. """Duck type every backend's _run_bash() must return. subprocess.Popen satisfies this
  238. natively; SDK backends (Modal, Daytona) return _ThreadedProcessHandle."""
  239. def poll(self) -> int | None: ...
  240. def kill(self) -> None: ...
  241. def wait(self, timeout: float | None = None) -> int: ...
  242. @property
  243. def stdout(self) -> IO[str] | None: ...
  244. @property
  245. def returncode(self) -> int | None: ...
  246. class _ThreadedProcessHandle:
  247. """Adapter for SDK backends (Modal, Daytona) that have no real subprocess: runs a blocking
  248. ``exec_fn() -> (output_str, exit_code)`` on a background thread behind a ProcessHandle
  249. interface. ``cancel_fn`` is invoked on ``kill()`` for backend-specific cancellation."""
  250. def __init__(self, exec_fn: Callable[[], tuple[str, int]], cancel_fn: Callable[[], None] | None = None):
  251. self._cancel_fn = cancel_fn
  252. self._done = threading.Event()
  253. self._returncode: int | None = None
  254. # Pipe for stdout — the drain thread in _wait_for_process reads the read end.
  255. read_fd, write_fd = os.pipe()
  256. self._stdout = os.fdopen(read_fd, "r", encoding="utf-8-sig", errors="replace")
  257. self._write_fd = write_fd
  258. def _worker():
  259. try:
  260. output, exit_code = exec_fn()
  261. self._returncode = exit_code
  262. try:
  263. os.write(self._write_fd, output.encode("utf-8", errors="replace"))
  264. except OSError:
  265. pass
  266. except Exception:
  267. self._returncode = 1
  268. finally:
  269. try:
  270. os.close(self._write_fd)
  271. except OSError:
  272. pass
  273. self._done.set()
  274. threading.Thread(target=_worker, daemon=True).start()
  275. @property
  276. def stdout(self):
  277. return self._stdout
  278. @property
  279. def returncode(self) -> int | None:
  280. return self._returncode
  281. def poll(self) -> int | None:
  282. return self._returncode if self._done.is_set() else None
  283. def kill(self):
  284. if self._cancel_fn:
  285. try:
  286. self._cancel_fn()
  287. except Exception:
  288. pass
  289. def wait(self, timeout: float | None = None) -> int:
  290. self._done.wait(timeout=timeout)
  291. return self._returncode
  292. # --- Stdout drain thread ---
  293. def _drain_stdout(proc: ProcessHandle, output: _BoundedOutputCollector, stop: threading.Event | None = None) -> None:
  294. """Drain ``proc.stdout`` into *output* until EOF or shortly after exit.
  295. ``for line in proc.stdout`` would block on ``readline()`` until EOF, and a backgrounded
  296. grandchild (``cmd &``, ``setsid cmd & disown``) inherits the pipe's write end — so the
  297. tool would hang for the grandchild's lifetime. Instead we ``select()`` with a short poll
  298. and stop ~300ms after bash exits even if the pipe has not EOF'd. Raw 4096-byte ``os.read``
  299. chunks can split a multibyte UTF-8 sequence, so an incremental decoder with
  300. ``errors="replace"`` buffers partial sequences across chunks. Streams without a real
  301. integer ``fileno()`` (mocks, in-memory adapters) are iterated to EOF instead — otherwise
  302. the thread would die silently and lose all output. ``select()`` does not work on pipe fds
  303. on Windows, so :func:`_drain_fd_windows` polls ``PeekNamedPipe`` there instead.
  304. """
  305. stream = proc.stdout
  306. if stream is None:
  307. return
  308. # Non-blocking drain via select(). The old pattern — ``for line in proc.stdout`` — blocks on
  309. # ``readline()`` until the pipe reaches EOF. When the user's command backgrounds a process (``cmd &``,
  310. # ``setsid cmd & disown``, etc.), that backgrounded grandchild inherits the write-end of our stdout pipe
  311. # via ``fork()``. Even after ``bash`` itself exits, the pipe stays open because the grandchild still
  312. # holds it — so the drain thread never returns and the tool hangs for the full lifetime of the
  313. # grandchild (issue #8340: users reported indefinite hangs when restarting uvicorn with ``setsid ... &
  314. # disown``). The fix: select() with a short poll interval, and stop draining shortly after ``bash``
  315. # exits even if the pipe hasn't EOF'd yet. Any output the grandchild writes after that point goes to an
  316. # orphaned pipe (harmless — the kernel reaps it when our end closes). Decoding: we ``os.read()`` raw
  317. # bytes in fixed-size chunks (4096) so a single multibyte UTF-8 character can split across reads. An
  318. # incremental decoder buffers partial sequences across chunks, and ``errors="replace"`` mirrors the
  319. # baseline ``TextIOWrapper`` (which was constructed with ``encoding="utf-8", errors="replace"`` on
  320. # ``Popen``) so binary or mis-encoded output is preserved with U+FFFD substitution rather than
  321. # clobbering the whole buffer.
  322. decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
  323. try:
  324. fd = stream.fileno()
  325. except Exception: # mocks / in-memory adapters without a real descriptor
  326. fd = None
  327. try:
  328. if not isinstance(fd, int) or fd < 0:
  329. for piece in stream:
  330. if piece is not None:
  331. output.append(decoder.decode(piece) if isinstance(piece, bytes) else str(piece))
  332. elif os.name == "nt":
  333. _drain_fd_windows(proc, fd, output, decoder, stop)
  334. else:
  335. _drain_fd_select(proc, fd, output, decoder, stop)
  336. except Exception:
  337. pass # closed fd / broken stream: keep what was captured
  338. finally:
  339. # With errors="replace" this emits U+FFFD for a final incomplete sequence.
  340. try:
  341. tail = decoder.decode(b"", final=True)
  342. if tail:
  343. output.append(tail)
  344. except Exception:
  345. pass
  346. def _drain_fd_select(proc, fd: int, output: _BoundedOutputCollector, decoder, stop=None) -> None:
  347. """POSIX drain: selector readiness (no FD_SETSIZE ceiling, #94928), stopping ~300ms after
  348. bash exits with the pipe idle, or when *stop* is set (the pipe is being handed to another
  349. reader — yield-to-background)."""
  350. waiter = open_fd_waiter(fd)
  351. if waiter is None:
  352. return # fd already closed
  353. try:
  354. _drain_fd_waiter(proc, fd, waiter, output, decoder, stop)
  355. finally:
  356. waiter.close()
  357. def _drain_fd_waiter(proc, fd: int, waiter, output: _BoundedOutputCollector, decoder, stop) -> None:
  358. idle_after_exit = 0
  359. while True:
  360. if stop is not None and stop.is_set():
  361. return
  362. if waiter.wait(0.1):
  363. try:
  364. chunk = os.read(fd, 4096)
  365. except (ValueError, OSError):
  366. return
  367. if not chunk:
  368. return # true EOF — all writers closed
  369. output.append(decoder.decode(chunk))
  370. idle_after_exit = 0
  371. elif proc.poll() is not None:
  372. # bash is gone and the pipe was idle ~100ms; allow two more cycles
  373. # for a buffered tail, then stop (a grandchild may hold the pipe).
  374. idle_after_exit += 1
  375. if idle_after_exit >= 3:
  376. return
  377. def _drain_fd_windows(proc, fd: int, output: _BoundedOutputCollector, decoder, stop=None) -> None:
  378. """Windows drain: poll ``PeekNamedPipe`` because ``select`` cannot poll pipes.
  379. ``os.read`` on a Windows pipe blocks until data is available or every writer
  380. closes the pipe. A backgrounded grandchild can retain a writer indefinitely,
  381. so use the Win32 pipe API to check availability before reading and apply the
  382. same post-exit idle bound as the POSIX drain.
  383. """
  384. import ctypes
  385. import msvcrt
  386. try:
  387. raw_handle = msvcrt.get_osfhandle(fd)
  388. except OSError:
  389. return
  390. if raw_handle in (-1, 0):
  391. return
  392. handle = ctypes.c_void_p(raw_handle)
  393. kernel32 = ctypes.windll.kernel32
  394. available = ctypes.c_ulong(0)
  395. idle_after_exit = 0
  396. while True:
  397. if stop is not None and stop.is_set():
  398. return
  399. try:
  400. ok = kernel32.PeekNamedPipe(handle, None, 0, None, ctypes.byref(available), None)
  401. except OSError:
  402. return
  403. if not ok:
  404. return
  405. if available.value:
  406. try:
  407. chunk = os.read(fd, min(int(available.value), 4096))
  408. except (ValueError, OSError):
  409. return
  410. if not chunk:
  411. return
  412. output.append(decoder.decode(chunk))
  413. idle_after_exit = 0
  414. continue
  415. if proc.poll() is not None:
  416. idle_after_exit += 1
  417. if idle_after_exit >= 3:
  418. return
  419. time.sleep(0.1)
  420. def _start_drain_thread(
  421. proc: ProcessHandle, output: _BoundedOutputCollector, stop: threading.Event | None = None,
  422. ) -> threading.Thread:
  423. """Start the daemon thread running :func:`_drain_stdout`; *stop* ends it early."""
  424. thread = threading.Thread(target=_drain_stdout, args=(proc, output, stop), daemon=True)
  425. thread.start()
  426. return thread