process_registry_ownership.py 5.8 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697
  1. """Which local background jobs die with their owner, for ProcessRegistry (process_identity layer 4)."""
  2. import subprocess
  3. import threading
  4. from contextlib import suppress
  5. from typing import Any
  6. from hermes_cli.process_identity import own_process_groups, release_process_groups, spawn_owned
  7. class ProcessOwnershipMixin:
  8. """A local job's process group goes to the owner-death supervisor while the job's own process runs,
  9. so it dies with this process however this process dies (SIGKILL, OOM, crash): the agent-lifecycle
  10. sweeps only run while we are alive, and a killed CLI/TUI/worker left every job running on init.
  11. ``ProcessSession._owned_pgid`` is that group, cleared by the one release so a recycled pgid is
  12. never unregistered on someone else's behalf. The subclass supplies ``_terminate_host_pid``."""
  13. # Set by recover_from_checkpoint: this host re-adopts the jobs a crashed predecessor left
  14. # running, so its own jobs must outlive its crash the same way (see _owns_job).
  15. _jobs_outlive_host = False
  16. _owned_pgid_lock = threading.Lock()
  17. def _owns_job(self, persist_on_release: bool) -> bool:
  18. """Not owned, on purpose: ``persist_on_release`` jobs, which must outlive the agent (#41225);
  19. every job of a gateway, which re-adopts its jobs from the checkpoint after a crash (flagged by
  20. recover_from_checkpoint, which every gateway boot runs, checkpoint or not; a supervised gateway
  21. also before it gets there). The classic CLI, the TUI, the Desktop, ACP, cron and messaging all
  22. run their tools in a gateway, so their background jobs are not owned; a process that is not a
  23. gateway (a standalone serve with no session authority, the slash worker, an embedder of
  24. AIAgent) owns its jobs, and foreground commands are owned everywhere. Also not owned: every job of a
  25. managed worker, a per-turn interpreter whose jobs the next turn's worker adopts with their
  26. output log (worker_output_log); and Windows, where the layer-3 job object covers ownership."""
  27. from agent.runtime_session_store import is_worker_process
  28. from tools.process_registry import _IS_WINDOWS, _is_supervised_gateway_process
  29. return not (_IS_WINDOWS or persist_on_release or self._jobs_outlive_host
  30. or is_worker_process() or _is_supervised_gateway_process())
  31. def _spawn_job(self, session: Any, argv: list[str], persist_on_release: bool, **popen_kwargs) -> subprocess.Popen:
  32. """Start a pipe job in its own session, owned unless ``_owns_job`` says it must outlive us."""
  33. if self._owns_job(persist_on_release):
  34. proc = spawn_owned(argv, purpose="terminal-background", **popen_kwargs)
  35. session._owned_pgid = proc.pid
  36. return proc
  37. # noqa: subprocess-stdin — a forwarder: spawn_local passes stdin=DEVNULL in popen_kwargs
  38. return subprocess.Popen(argv, start_new_session=True, **popen_kwargs) # detached-spawn: ok — see _owns_job
  39. def _own_pty_job(self, session: Any, pty_proc: Any) -> None:
  40. """The PTY child leads its own session and group (pty.fork). Its hangup reaches only jobs that
  41. keep the default SIGHUP action; `nohup`/trap-HUP servers outlived a killed owner."""
  42. if self._owns_job(session.persist_on_release):
  43. own_process_groups([pty_proc.pid])
  44. session._owned_pgid = pty_proc.pid
  45. def _take_handoff_group(self, session: Any, proc: subprocess.Popen) -> None:
  46. """A foreground command that yields to background hands its supervised group over with the
  47. pipe (LocalEnvironment._run_bash): the job keeps it, unless jobs here must outlive us."""
  48. owned_pgid, proc._hermes_owned_pgid = getattr(proc, "_hermes_owned_pgid", None), None
  49. if owned_pgid is not None and self._owns_job(False):
  50. session._owned_pgid = owned_pgid
  51. elif owned_pgid is not None:
  52. release_process_groups([owned_pgid])
  53. def _return_handoff_group(self, session: Any, proc: subprocess.Popen, owned_pgid: Any) -> None:
  54. """The handoff failed (the reader could not start): the foreground wait resumes and must own
  55. the group again, covered by the supervisor, so its own completion releases it."""
  56. with self._owned_pgid_lock:
  57. session._owned_pgid = None
  58. if owned_pgid is not None:
  59. own_process_groups([owned_pgid]) # re-covers a group the gateway path released
  60. proc._hermes_owned_pgid = owned_pgid
  61. def _release_job_group(self, session: Any) -> None:
  62. """The job's own process exited or was killed: stop covering its group. What it left behind on
  63. purpose (`server &`) is no longer this job's, exactly as before."""
  64. with self._owned_pgid_lock:
  65. owned_pgid, session._owned_pgid = session._owned_pgid, None
  66. if owned_pgid is not None:
  67. release_process_groups([owned_pgid])
  68. def _reap_untracked(self, session: Any, proc: subprocess.Popen) -> None:
  69. """Post-Popen setup failed: kill the orphaned subprocess (and any setsid
  70. descendants) so nothing leaks untracked."""
  71. from tools.process_registry import _stop_systemd_unit
  72. with suppress(Exception):
  73. if session.systemd_unit:
  74. # Scope teardown is the authoritative cleanup for the worker cgroup;
  75. # the systemd-run wrapper PID is terminated below as fallback.
  76. _stop_systemd_unit(session.systemd_unit)
  77. # _terminate_host_pid revalidates the kernel start time captured at
  78. # spawn before signalling: pgid == pid would only prove the pid leads a
  79. # group, and a reaped child can be recycled onto an unrelated leader.
  80. self._terminate_host_pid(proc.pid, session.host_start_time)
  81. with suppress(Exception):
  82. proc.wait(timeout=5)
  83. self._release_job_group(session)