process_registry_output_log.py 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135
  1. """File-backed output for background processes a registered worker starts.
  2. A managed worker is a per-turn interpreter. A stdout pipe whose read end that worker's reader
  3. thread owns closes when the worker exits, so the child's next write raises SIGPIPE
  4. (``broken_pipe``) and everything it printed is gone. Under a worker the child writes to a
  5. profile-scoped log instead and its shell records the exit status beside it (the contract the
  6. env-backend wrapper already uses), so whichever interpreter adopts the checkpoint entry next
  7. (the session's next worker, the owner at startup) tails the same file and collects the exit.
  8. """
  9. import codecs
  10. from contextlib import suppress
  11. import logging
  12. import os
  13. from pathlib import Path
  14. import time
  15. from typing import Optional
  16. from hermes_dir_acl import make_user_dir
  17. from hermes_constants import get_hermes_home
  18. logger = logging.getLogger("tools.process_registry")
  19. _TAIL_SECONDS = 0.2
  20. # After the process ended: an orphaned grandchild may keep appending; stop draining after this.
  21. _FINAL_DRAIN_MAX_BYTES = 1 << 20
  22. def output_dir() -> Path:
  23. return get_hermes_home() / "logs" / "process-output"
  24. def worker_output_log(session_id: str) -> Optional[Path]:
  25. """The log a worker-spawned process writes to; None outside a registered worker."""
  26. from agent.runtime_session_store import is_worker_process
  27. if not is_worker_process():
  28. return None
  29. directory = output_dir()
  30. make_user_dir(directory, mode=0o700, parents=True, exist_ok=True)
  31. from tools.process_registry_results import RESULT_RETENTION_SECONDS
  32. # Only a recorded exit retires a log: a quiet process can run longer than the retention.
  33. cutoff = time.time() - RESULT_RETENTION_SECONDS
  34. for exited in directory.glob("proc_*.exit"):
  35. with suppress(FileNotFoundError):
  36. if exited.stat().st_mtime < cutoff:
  37. exited.with_suffix(".log").unlink(missing_ok=True)
  38. exited.unlink()
  39. return directory / f"{session_id}.log"
  40. def adoptable_log(path) -> str:
  41. """A checkpointed log path, only when it names this profile's output directory."""
  42. log = Path(str(path or ""))
  43. return str(log) if path and log.parent == output_dir() and log.suffix == ".log" else ""
  44. def open_output_log(log: Path) -> int:
  45. """Owner-only append descriptor for the child's stdout+stderr (the parent closes its copy)."""
  46. return os.open(log, os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, "O_BINARY", 0), 0o600)
  47. def _exit_path(log) -> Path:
  48. return Path(log).with_suffix(".exit")
  49. def record_exit_command(log: Path, command: str) -> str:
  50. """Prefix ``command`` so the shell writes its exit status beside the log: an interpreter
  51. that adopts the process after the spawner exited has no Popen to wait on."""
  52. import shlex
  53. target = shlex.quote(_exit_path(log).as_posix())
  54. return f"__hermes_exit_file={target}; trap 'printf \"%s\\n\" \"$?\" > \"$__hermes_exit_file\"' EXIT\n{command}"
  55. def recorded_exit_code(log) -> Optional[int]:
  56. try:
  57. return int(_exit_path(log).read_text(encoding="utf-8-sig").strip())
  58. except (OSError, ValueError):
  59. return None
  60. class ProcessOutputLogMixin:
  61. """The subclass supplies ``_ingest_output``, ``_clean_shell_noise``, ``_finish_reader``,
  62. ``_finish_exited`` and ``_detached_host_fate``. See ProcessRegistry."""
  63. def _log_reader_loop(self, session) -> None:
  64. """Tail ``session.output_log`` until its process ends. The spawner decides the exit from
  65. its Popen; an adopter (no Popen) from the recorded status, else ``lost`` (-1)."""
  66. decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
  67. proc = session.process
  68. # poll()/wait() reconcile asks this reader to finish instead of reading a pipe itself.
  69. session._reader_selectable = True
  70. head_noise = True
  71. def ingest(text: str) -> None:
  72. nonlocal head_noise
  73. if head_noise: # ``bash -lic`` startup warnings, as the pipe reader strips them
  74. text = self._clean_shell_noise(text)
  75. head_noise = not text.strip()
  76. self._ingest_output(session, text)
  77. def running() -> bool:
  78. if session._reader_finish_requested.is_set():
  79. return False
  80. if proc is not None:
  81. return proc.poll() is None
  82. return self._detached_host_fate(session.pid, session.host_start_time) == "running"
  83. try:
  84. with open(session.output_log, "rb") as stream:
  85. drained = 0
  86. while True:
  87. # Decided BEFORE the read: once the process is gone every byte it wrote is readable.
  88. alive = running()
  89. chunk = stream.read(65536)
  90. if chunk:
  91. if text := decoder.decode(chunk):
  92. ingest(text)
  93. drained += 0 if alive else len(chunk)
  94. if drained < _FINAL_DRAIN_MAX_BYTES:
  95. continue
  96. if not alive:
  97. break
  98. time.sleep(_TAIL_SECONDS)
  99. except OSError as exc:
  100. logger.warning("Process output log %s unreadable: %s", session.output_log, exc)
  101. if proc is not None:
  102. self._finish_reader(session, decoder, ingest, "Log", proc.wait, lambda: proc.returncode)
  103. return
  104. if tail := decoder.decode(b"", final=True):
  105. ingest(tail)
  106. code = recorded_exit_code(session.output_log)
  107. if code is None:
  108. # SIGKILL / exec past the trap: the spawner is gone and no status was recorded.
  109. self._finish_exited(session, -1, reason="lost", source="backend_lost")
  110. else:
  111. self._finish_exited(session, code)