| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151 |
- """Contain each managed router tree without adopting the owner process."""
- from __future__ import annotations
- from collections.abc import Mapping
- import ctypes
- from ctypes import wintypes
- import subprocess
- import sys
- import threading
- import psutil
- class _BasicLimits(ctypes.Structure):
- _fields_ = [
- ("PerProcessUserTimeLimit", ctypes.c_longlong),
- ("PerJobUserTimeLimit", ctypes.c_longlong),
- ("LimitFlags", wintypes.DWORD),
- ("MinimumWorkingSetSize", ctypes.c_size_t),
- ("MaximumWorkingSetSize", ctypes.c_size_t),
- ("ActiveProcessLimit", wintypes.DWORD),
- ("Affinity", ctypes.c_size_t),
- ("PriorityClass", wintypes.DWORD),
- ("SchedulingClass", wintypes.DWORD),
- ]
- class _IoCounters(ctypes.Structure):
- _fields_ = [(name, ctypes.c_ulonglong) for name in (
- "ReadOperationCount", "WriteOperationCount", "OtherOperationCount",
- "ReadTransferCount", "WriteTransferCount", "OtherTransferCount",
- )]
- class _ExtendedLimits(ctypes.Structure):
- _fields_ = [
- ("BasicLimitInformation", _BasicLimits),
- ("IoInfo", _IoCounters),
- ("ProcessMemoryLimit", ctypes.c_size_t),
- ("JobMemoryLimit", ctypes.c_size_t),
- ("PeakProcessMemoryUsed", ctypes.c_size_t),
- ("PeakJobMemoryUsed", ctypes.c_size_t),
- ]
- class _WindowsJob:
- def __init__(self):
- self._lock = threading.Lock()
- self._api = ctypes.WinDLL("kernel32", use_last_error=True)
- for name, args, result in (
- ("CreateJobObjectW", [ctypes.c_void_p, wintypes.LPCWSTR], wintypes.HANDLE),
- ("SetInformationJobObject", [wintypes.HANDLE, ctypes.c_int,
- ctypes.c_void_p, wintypes.DWORD], wintypes.BOOL),
- ("AssignProcessToJobObject", [wintypes.HANDLE, wintypes.HANDLE], wintypes.BOOL),
- ("CloseHandle", [wintypes.HANDLE], wintypes.BOOL),
- ):
- fn = getattr(self._api, name)
- fn.argtypes = args
- fn.restype = result
- # NULL security attributes create a non-inheritable, unnamed owner handle.
- self._handle = self._api.CreateJobObjectW(None, None)
- if not self._handle:
- raise ctypes.WinError(ctypes.get_last_error())
- try:
- limits = _ExtendedLimits()
- # Neither BREAKAWAY_OK nor SILENT_BREAKAWAY_OK: descendants stay contained.
- limits.BasicLimitInformation.LimitFlags = 0x2000 # KILL_ON_JOB_CLOSE
- if not self._api.SetInformationJobObject(
- self._handle, 9, ctypes.byref(limits), ctypes.sizeof(limits)):
- raise ctypes.WinError(ctypes.get_last_error())
- except BaseException:
- self.close()
- raise
- def assign(self, proc: subprocess.Popen) -> None:
- # Popen retains the original process handle, avoiding a PID-reuse race.
- if not self._api.AssignProcessToJobObject(self._handle, int(proc._handle)):
- raise ctypes.WinError(ctypes.get_last_error())
- def close(self) -> None:
- """Terminate the contained tree; repeated closes are harmless."""
- with self._lock:
- if self._handle is not None:
- if not self._api.CloseHandle(self._handle):
- raise ctypes.WinError(ctypes.get_last_error())
- self._handle = None
- _CREDENTIAL_ENV_MARKERS = ("_API_KEY", "_TOKEN", "_SECRET", "PASSWORD", "_CREDENTIALS")
- def server_child_env(base_env: Mapping[str, str]) -> dict[str, str]:
- """Return the environment a native inference child (llama-server) gets.
- Provider and tool credentials never belong in a native child that talks to nobody but
- us — and on Windows they are not merely leaked: the bundled OpenMP runtime died with
- STATUS_HEAP_CORRUPTION during initialisation with one `*_API_KEY` present and loaded fine
- with only that variable removed (#116109, confirmed on a model-free libomp.dll probe).
- Everything else (PATH, CUDA_*, HSA_*, OMP_*, TEMP, …) passes through untouched. Applied by
- the llama-server supervisor only: spawn_server is also the generic bounded-probe spawner
- (git / PowerShell / update probes), whose children legitimately need GH_TOKEN, HF_TOKEN, …
- """
- return {
- key: value for key, value in base_env.items()
- if not any(marker in key.upper() for marker in _CREDENTIAL_ENV_MARKERS)
- }
- def spawn_server(cmd, *, owned: bool = False, **kwargs) -> tuple[subprocess.Popen, _WindowsJob | None]:
- """Start a router, returning its process and an owner-held containment handle.
- Keep the job until shutdown and call close() to terminate the entire tree.
- Windows closes it automatically if the owner dies. On POSIX ``owned=True`` gives
- the same guarantee: the router leads its own process group (its model children
- stay in it) and that group goes to this process's owner-death supervisor, which
- kills it the moment the owner dies by any cause (#126631). Without it Popen's
- ordinary behavior applies (this is also the generic bounded-probe spawner, whose
- short-lived children need no supervision). Assignment happens before the
- child's first instruction.
- """
- if sys.platform != "win32":
- if not owned:
- return subprocess.Popen(cmd, **kwargs), None
- from hermes_cli.process_identity import spawn_owned
- return spawn_owned(cmd, purpose="llama-server", **kwargs), None
- job = _WindowsJob()
- proc = None
- try:
- kwargs["creationflags"] = kwargs.get("creationflags", 0) | 0x00000004 # CREATE_SUSPENDED
- proc = subprocess.Popen(cmd, **kwargs)
- job.assign(proc)
- psutil.Process(proc.pid).resume()
- return proc, job
- except BaseException:
- try:
- if proc is not None:
- # Assignment may have failed: closing an empty job is not enough.
- proc.kill()
- proc.wait(timeout=10)
- finally:
- try:
- job.close()
- finally:
- if proc is not None:
- for stream in (proc.stdin, proc.stdout, proc.stderr):
- if stream is not None:
- stream.close()
- proc._handle.Close()
- raise
|