env_probe.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247
  1. """Local-environment toolchain probe for the system prompt: when the terminal backend
  2. is local, one deterministic line about Python tooling (python3/python versions, missing
  3. pip, pip bound to another Python, PEP 668) so models don't discover it by hitting
  4. walls. Cached per process; "" when clean. Remote backends are skipped (the sandbox has
  5. its own probe in agent/prompt_builder). Toggle: ``agent.environment_probe`` in config.yaml."""
  6. from __future__ import annotations
  7. import logging
  8. import shutil
  9. import subprocess
  10. import tempfile
  11. import threading
  12. from typing import Optional
  13. from hermes_cli._subprocess_compat import windows_hide_flags
  14. logger = logging.getLogger(__name__)
  15. # Concurrency model: exactly ONE background worker runs the probe; ``_PROBE_DONE``
  16. # signals completion. Callers block at most ``_PROBE_WAIT_TIMEOUT`` s then fail open
  17. # with "" — a stuck probe (e.g. a Windows pipe wedged by an orphaned pip descendant)
  18. # can degrade only the probe line, never system-prompt construction.
  19. # Module-level cache. The probe result is deterministic for the lifetime of the process — Python install
  20. # state doesn't change mid-session in any way that would matter for the system prompt. See #67964.
  21. _CACHE_LOCK = threading.Lock()
  22. _CACHED_LINE: Optional[str] = None # None = not probed yet; "" = probed, nothing to say.
  23. _PROBE_DONE = threading.Event()
  24. _PROBE_THREAD: Optional[threading.Thread] = None
  25. _PROBE_GEN = 0 # bumped on reset so a stale worker can't publish into the fresh generation
  26. _PROBE_WAIT_TIMEOUT = 10.0 # healthy runtime ~0.5s
  27. _WAIT_ALREADY_TIMED_OUT = False # after one full wait, later callers only peek
  28. # Keep in sync with agent/prompt_builder.py:_REMOTE_TERMINAL_BACKENDS.
  29. # Duplicated rather than imported to avoid a circular import.
  30. _REMOTE_BACKENDS = frozenset({
  31. "docker", "modal", "daytona", "ssh", "managed_modal",
  32. "vercel_sandbox",
  33. })
  34. def _plugin_backend_is_remote(backend: str) -> bool:
  35. """Whether a plugin-registered terminal backend is remote (fail-soft)."""
  36. if not backend or backend in _REMOTE_BACKENDS or backend == "local":
  37. return False
  38. try:
  39. from agent.terminal_env_registry import provider_flag
  40. return bool(provider_flag(backend, "is_remote", False))
  41. except Exception:
  42. return False
  43. def _run(cmd: list[str], timeout: float = 3.0) -> tuple[int, str, str]:
  44. """Run a short subprocess -> (returncode, stdout, stderr); failures (binary
  45. missing, timeout, OSError) return (-1, "", "<reason>"). Output goes through temp
  46. files, not pipes, so ``timeout`` bounds the *whole* call even on native Windows: a
  47. console-script launcher (``pip.exe``) can spawn a descendant that inherits the
  48. captured handles and outlives its parent; with OS pipes ``communicate()``'s reader
  49. threads block until that grandchild closes the write end (a warm probe could hang
  50. ~28 min holding ``_CACHE_LOCK``). Temp files make ``wait()`` cover only the child."""
  51. try:
  52. with tempfile.TemporaryFile() as out_f, tempfile.TemporaryFile() as err_f:
  53. try:
  54. result = subprocess.run(
  55. cmd, stdout=out_f, stderr=err_f, timeout=timeout, check=False,
  56. # CREATE_NO_WINDOW (0 on POSIX): pythonw hosts would flash a console
  57. stdin=subprocess.DEVNULL, creationflags=windows_hide_flags())
  58. except subprocess.TimeoutExpired:
  59. return -1, "", "timeout"
  60. out_f.seek(0)
  61. err_f.seek(0)
  62. return (result.returncode, out_f.read().decode("utf-8", "replace").strip(),
  63. err_f.read().decode("utf-8", "replace").strip())
  64. except FileNotFoundError:
  65. return -1, "", "not found"
  66. except OSError as exc:
  67. return -1, "", f"oserror: {exc}"
  68. def _py_out(binary: str, *args: str) -> Optional[str]:
  69. """stdout of ``<binary> *args`` when the binary is on PATH and exits 0, else None."""
  70. if not shutil.which(binary):
  71. return None
  72. rc, out, _err = _run([binary, *args])
  73. return out if rc == 0 else None
  74. def _python_version_of(binary: str) -> Optional[str]:
  75. """Return a short version string like ``3.12.4`` for ``binary``, or None."""
  76. code = "import sys; print('.'.join(map(str, sys.version_info[:3])))"
  77. return _py_out(binary, "-c", code) or None
  78. def _has_pip_module(binary: str) -> bool:
  79. """True if ``<binary> -m pip --version`` succeeds."""
  80. return _py_out(binary, "-m", "pip", "--version") is not None
  81. def _detect_pep668(binary: str) -> bool:
  82. """True when ``<binary>`` is PEP-668 externally-managed (``EXTERNALLY-MANAGED``
  83. marker next to the stdlib, as Debian/Ubuntu ship)."""
  84. code = ("import os; print('yes' if os.path.exists(os.path.join("
  85. "os.path.dirname(os.__file__), 'EXTERNALLY-MANAGED')) else 'no')")
  86. return (_py_out(binary, "-c", code) or "").strip() == "yes"
  87. def _pip_python_version() -> Optional[str]:
  88. """If ``pip`` is on PATH, the Python version it's bound to — the trailing
  89. ``(python X.Y)`` of ``pip --version`` (e.g. ``"3.12"``), else None."""
  90. out = _py_out("pip", "--version") or ""
  91. if "(python " in out and out.endswith(")"):
  92. return out.rsplit("(python ", 1)[1][:-1].strip()
  93. return None
  94. def _resolve_terminal_backend() -> str:
  95. """Scope-aware terminal backend name (``local`` when unresolvable)."""
  96. try:
  97. from tools.terminal_scope import terminal_env
  98. return (terminal_env("TERMINAL_ENV") or "local").strip().lower()
  99. except Exception: # never let policy resolution break prompt building
  100. logger.debug("terminal backend resolution failed", exc_info=True)
  101. return "local"
  102. def _build_probe_line() -> str:
  103. """Build the one-liner; "" when nothing notable is detected — the goal is to
  104. save the model from an avoidable wall, not narrate a healthy environment."""
  105. py3_ver = _python_version_of("python3")
  106. py_ver = _python_version_of("python") # for systems with a `python` alias
  107. py3_has_pip = _has_pip_module("python3") if py3_ver else False
  108. pip_bound_to = _pip_python_version()
  109. py3_pep668 = _detect_pep668("python3") if py3_ver else False
  110. # Bare which() is correct here (unlike Hermes's own uv call sites): this reports
  111. # the environment *the model will see* in the terminal tool, whose PATH includes
  112. # the Hermes-managed $HERMES_HOME/bin via local.py.
  113. has_uv = shutil.which("uv") is not None
  114. mismatch = bool(pip_bound_to and py3_ver and not py3_ver.startswith(pip_bound_to))
  115. if py3_ver is not None and py3_has_pip and not mismatch and (not py3_pep668 or has_uv):
  116. return ""
  117. # Compact factual summary; ONE line so it doesn't dominate the prompt.
  118. bits: list[str] = []
  119. if py3_ver:
  120. bits.append(f"python3={py3_ver}" + ("" if py3_has_pip else " (no pip module)"))
  121. else:
  122. bits.append("python3=missing")
  123. if py_ver and py_ver != py3_ver:
  124. bits.append(f"python={py_ver}")
  125. elif not py_ver and py3_ver:
  126. # Common on Debian/Ubuntu — stop the model typing `python`.
  127. bits.append("python=missing (use python3)")
  128. if pip_bound_to:
  129. if mismatch:
  130. bits.append(f"pip→python{pip_bound_to} (mismatch)")
  131. elif not py3_has_pip:
  132. bits.append(f"pip→python{pip_bound_to}") # pip script works, `-m pip` doesn't
  133. elif not py3_has_pip:
  134. # (when `pip` is off PATH but `python3 -m pip` works, say nothing)
  135. bits.append("pip=missing")
  136. if py3_pep668:
  137. bits.append("PEP 668=yes (use venv or uv)")
  138. if has_uv:
  139. bits.append("uv=installed")
  140. return "Python toolchain: " + ", ".join(bits) + "."
  141. def get_environment_probe_line(*, force_refresh: bool = False) -> str:
  142. """Return the cached probe line (building it on first call); "" when the
  143. environment is clean, so the prompt assembler drops the section. Waits at most
  144. ``_PROBE_WAIT_TIMEOUT`` on the single worker, then fails open with "".
  145. ``force_refresh`` is for tests.
  146. A wedged probe subprocess (#67964) therefore can never block system-prompt construction — at worst the
  147. toolchain line is absent from prompts built while the probe is stuck.
  148. """
  149. global _WAIT_ALREADY_TIMED_OUT
  150. if force_refresh:
  151. _reset_cache_for_tests()
  152. # Resolve the backend HERE, in the caller's context: under gateway multiplexing the
  153. # routed profile's backend lives in the per-turn terminal scope, which the worker
  154. # thread does not inherit. Remote backends answer "" without consulting the cache
  155. # — the cached line describes the HOST toolchain.
  156. # See #68559.
  157. backend = _resolve_terminal_backend()
  158. if backend in _REMOTE_BACKENDS or _plugin_backend_is_remote(backend):
  159. return ""
  160. if _PROBE_DONE.is_set():
  161. return _CACHED_LINE or ""
  162. _ensure_probe_started()
  163. wait_timeout = 0.05 if _WAIT_ALREADY_TIMED_OUT else _PROBE_WAIT_TIMEOUT
  164. if not _PROBE_DONE.wait(timeout=wait_timeout):
  165. # Probe stuck or pathologically slow: the line is a nice-to-have,
  166. # blocking prompt construction is an outage. Fail open.
  167. if not _WAIT_ALREADY_TIMED_OUT:
  168. _WAIT_ALREADY_TIMED_OUT = True
  169. logger.warning(
  170. "env_probe did not finish within %.0fs; building the system "
  171. "prompt without the Python toolchain line", _PROBE_WAIT_TIMEOUT)
  172. return ""
  173. return _CACHED_LINE or ""
  174. def _probe_worker(gen: int) -> None:
  175. """Body of the single probe thread — computes and publishes the line."""
  176. global _CACHED_LINE
  177. try:
  178. line = _build_probe_line()
  179. except Exception as exc: # never let probe failure propagate
  180. logger.debug("env_probe failed: %s", exc)
  181. line = ""
  182. with _CACHE_LOCK:
  183. if gen != _PROBE_GEN:
  184. return # superseded by a reset (tests) — discard stale result
  185. _CACHED_LINE = line
  186. _PROBE_DONE.set()
  187. def _ensure_probe_started() -> None:
  188. """Start the probe worker if it isn't running and hasn't finished."""
  189. global _PROBE_THREAD
  190. with _CACHE_LOCK:
  191. if _PROBE_DONE.is_set() or (_PROBE_THREAD is not None and _PROBE_THREAD.is_alive()):
  192. return
  193. _PROBE_THREAD = threading.Thread(
  194. target=_probe_worker, args=(_PROBE_GEN,), name="env-probe", daemon=True)
  195. _PROBE_THREAD.start()
  196. def warm_environment_probe_async() -> None:
  197. """Start the probe in the background so the first system-prompt build doesn't
  198. pay the ~0.5s of subprocess calls on the time-to-first-token path. Idempotent;
  199. ``get_environment_probe_line`` waits (bounded) on the same worker."""
  200. _ensure_probe_started()
  201. def _reset_cache_for_tests() -> None:
  202. """Test helper — clear the cache between probe scenarios."""
  203. global _CACHED_LINE, _PROBE_THREAD, _PROBE_GEN, _WAIT_ALREADY_TIMED_OUT
  204. with _CACHE_LOCK:
  205. _CACHED_LINE = None
  206. _PROBE_DONE.clear()
  207. _PROBE_THREAD = None
  208. _PROBE_GEN += 1
  209. _WAIT_ALREADY_TIMED_OUT = False