| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135 |
- """File-backed output for background processes a registered worker starts.
- A managed worker is a per-turn interpreter. A stdout pipe whose read end that worker's reader
- thread owns closes when the worker exits, so the child's next write raises SIGPIPE
- (``broken_pipe``) and everything it printed is gone. Under a worker the child writes to a
- profile-scoped log instead and its shell records the exit status beside it (the contract the
- env-backend wrapper already uses), so whichever interpreter adopts the checkpoint entry next
- (the session's next worker, the owner at startup) tails the same file and collects the exit.
- """
- import codecs
- from contextlib import suppress
- import logging
- import os
- from pathlib import Path
- import time
- from typing import Optional
- from hermes_dir_acl import make_user_dir
- from hermes_constants import get_hermes_home
- logger = logging.getLogger("tools.process_registry")
- _TAIL_SECONDS = 0.2
- # After the process ended: an orphaned grandchild may keep appending; stop draining after this.
- _FINAL_DRAIN_MAX_BYTES = 1 << 20
- def output_dir() -> Path:
- return get_hermes_home() / "logs" / "process-output"
- def worker_output_log(session_id: str) -> Optional[Path]:
- """The log a worker-spawned process writes to; None outside a registered worker."""
- from agent.runtime_session_store import is_worker_process
- if not is_worker_process():
- return None
- directory = output_dir()
- make_user_dir(directory, mode=0o700, parents=True, exist_ok=True)
- from tools.process_registry_results import RESULT_RETENTION_SECONDS
- # Only a recorded exit retires a log: a quiet process can run longer than the retention.
- cutoff = time.time() - RESULT_RETENTION_SECONDS
- for exited in directory.glob("proc_*.exit"):
- with suppress(FileNotFoundError):
- if exited.stat().st_mtime < cutoff:
- exited.with_suffix(".log").unlink(missing_ok=True)
- exited.unlink()
- return directory / f"{session_id}.log"
- def adoptable_log(path) -> str:
- """A checkpointed log path, only when it names this profile's output directory."""
- log = Path(str(path or ""))
- return str(log) if path and log.parent == output_dir() and log.suffix == ".log" else ""
- def open_output_log(log: Path) -> int:
- """Owner-only append descriptor for the child's stdout+stderr (the parent closes its copy)."""
- return os.open(log, os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, "O_BINARY", 0), 0o600)
- def _exit_path(log) -> Path:
- return Path(log).with_suffix(".exit")
- def record_exit_command(log: Path, command: str) -> str:
- """Prefix ``command`` so the shell writes its exit status beside the log: an interpreter
- that adopts the process after the spawner exited has no Popen to wait on."""
- import shlex
- target = shlex.quote(_exit_path(log).as_posix())
- return f"__hermes_exit_file={target}; trap 'printf \"%s\\n\" \"$?\" > \"$__hermes_exit_file\"' EXIT\n{command}"
- def recorded_exit_code(log) -> Optional[int]:
- try:
- return int(_exit_path(log).read_text(encoding="utf-8-sig").strip())
- except (OSError, ValueError):
- return None
- class ProcessOutputLogMixin:
- """The subclass supplies ``_ingest_output``, ``_clean_shell_noise``, ``_finish_reader``,
- ``_finish_exited`` and ``_detached_host_fate``. See ProcessRegistry."""
- def _log_reader_loop(self, session) -> None:
- """Tail ``session.output_log`` until its process ends. The spawner decides the exit from
- its Popen; an adopter (no Popen) from the recorded status, else ``lost`` (-1)."""
- decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
- proc = session.process
- # poll()/wait() reconcile asks this reader to finish instead of reading a pipe itself.
- session._reader_selectable = True
- head_noise = True
- def ingest(text: str) -> None:
- nonlocal head_noise
- if head_noise: # ``bash -lic`` startup warnings, as the pipe reader strips them
- text = self._clean_shell_noise(text)
- head_noise = not text.strip()
- self._ingest_output(session, text)
- def running() -> bool:
- if session._reader_finish_requested.is_set():
- return False
- if proc is not None:
- return proc.poll() is None
- return self._detached_host_fate(session.pid, session.host_start_time) == "running"
- try:
- with open(session.output_log, "rb") as stream:
- drained = 0
- while True:
- # Decided BEFORE the read: once the process is gone every byte it wrote is readable.
- alive = running()
- chunk = stream.read(65536)
- if chunk:
- if text := decoder.decode(chunk):
- ingest(text)
- drained += 0 if alive else len(chunk)
- if drained < _FINAL_DRAIN_MAX_BYTES:
- continue
- if not alive:
- break
- time.sleep(_TAIL_SECONDS)
- except OSError as exc:
- logger.warning("Process output log %s unreadable: %s", session.output_log, exc)
- if proc is not None:
- self._finish_reader(session, decoder, ingest, "Log", proc.wait, lambda: proc.returncode)
- return
- if tail := decoder.decode(b"", final=True):
- ingest(tail)
- code = recorded_exit_code(session.output_log)
- if code is None:
- # SIGKILL / exec past the trap: the spawner is gone and no status was recorded.
- self._finish_exited(session, -1, reason="lost", source="backend_lost")
- else:
- self._finish_exited(session, code)
|