runner.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264
  1. """Verification runner: bootstrap -> build -> test -> start in background ->
  2. readiness loop -> teardown (scoped port of grok-cli's verify flow).
  3. Commands come from the project's own recipe and run with ``shell=True`` on
  4. purpose: a developer tool running the project's own build commands in its own
  5. checkout — the same trust level as the terminal tool.
  6. """
  7. from __future__ import annotations
  8. import os
  9. import signal
  10. import subprocess
  11. import time
  12. import urllib.error
  13. import urllib.request
  14. from dataclasses import dataclass, field
  15. from pathlib import Path
  16. from typing import Any, Callable
  17. from agent.verify.recipes import Recipe
  18. from hermes_cli.process_identity import release_process_group_once_empty, spawn_owned
  19. DEFAULT_PHASE_TIMEOUT = 600.0
  20. DEFAULT_READY_TIMEOUT = 60.0
  21. _TAIL_CHARS = 2000
  22. PHASE_ORDER = ("bootstrap", "build", "test")
  23. # Project-authored shell commands; see module docstring.
  24. _SUBPROCESS_KW: dict[str, Any] = dict(
  25. shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
  26. text=True, errors="replace",
  27. )
  28. @dataclass
  29. class PhaseResult:
  30. phase: str
  31. command: str
  32. exit_code: int | None
  33. duration: float
  34. output_tail: str
  35. timed_out: bool = False
  36. @property
  37. def ok(self) -> bool:
  38. return self.exit_code == 0 and not self.timed_out
  39. def to_dict(self) -> dict[str, Any]:
  40. return {
  41. "phase": self.phase, "command": self.command, "exitCode": self.exit_code,
  42. "duration": round(self.duration, 3), "ok": self.ok, "timedOut": self.timed_out,
  43. "outputTail": self.output_tail,
  44. }
  45. @dataclass
  46. class ReadinessResult:
  47. url: str
  48. ready: bool
  49. status_code: int | None
  50. duration: float
  51. error: str | None = None
  52. output_tail: str = ""
  53. def to_dict(self) -> dict[str, Any]:
  54. return {
  55. "url": self.url, "ready": self.ready, "statusCode": self.status_code,
  56. "duration": round(self.duration, 3), "error": self.error, "outputTail": self.output_tail,
  57. }
  58. @dataclass
  59. class VerifyResult:
  60. recipe_name: str
  61. phases: list[PhaseResult] = field(default_factory=list)
  62. readiness: ReadinessResult | None = None
  63. @property
  64. def ok(self) -> bool:
  65. return all(p.ok for p in self.phases) and (self.readiness is None or self.readiness.ready)
  66. def to_dict(self) -> dict[str, Any]:
  67. return {
  68. "recipe": self.recipe_name, "ok": self.ok,
  69. "phases": [p.to_dict() for p in self.phases],
  70. "readiness": self.readiness.to_dict() if self.readiness else None,
  71. }
  72. def _tail(text: str, limit: int = _TAIL_CHARS) -> str:
  73. return text[-limit:] if len(text) > limit else text
  74. def _run_phase_command(
  75. phase: str, command: str, root: Path, timeout: float,
  76. on_output: Callable[[str], None] | None = None,
  77. ) -> PhaseResult:
  78. started = time.monotonic()
  79. try:
  80. proc = subprocess.run(command, cwd=str(root), timeout=timeout, **_SUBPROCESS_KW, check=False)
  81. output, exit_code, timed_out = proc.stdout or "", proc.returncode, False
  82. except subprocess.TimeoutExpired as exc:
  83. raw = exc.output
  84. output = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else (raw or "")
  85. exit_code, timed_out = None, True
  86. duration = time.monotonic() - started
  87. if on_output and output:
  88. on_output(output)
  89. return PhaseResult(phase, command, exit_code, duration, _tail(output), timed_out)
  90. def _poll_readiness(url: str, timeout: float, interval: float = 1.0) -> tuple[bool, int | None, str | None]:
  91. deadline = time.monotonic() + timeout
  92. last_error: str | None = None
  93. while time.monotonic() < deadline:
  94. try:
  95. with urllib.request.urlopen(url, timeout=5) as resp:
  96. return True, resp.status, None
  97. except urllib.error.HTTPError as exc:
  98. # The server answered — it is up, even if it returned 4xx/5xx.
  99. return True, exc.code, None
  100. except (urllib.error.URLError, OSError, TimeoutError) as exc:
  101. last_error = str(exc)
  102. time.sleep(interval)
  103. return False, None, last_error
  104. def _terminate_process_group(proc: subprocess.Popen) -> None:
  105. """SIGTERM the app's process group (``start_new_session=True`` on POSIX; just the
  106. direct child on Windows, which lacks ``os.killpg``), SIGKILL after 10s."""
  107. if proc.poll() is not None:
  108. return
  109. killpg = getattr(os, "killpg", None)
  110. getpgid = getattr(os, "getpgid", None)
  111. pgid = None
  112. if killpg is not None and getpgid is not None:
  113. try:
  114. pgid = getpgid(proc.pid)
  115. except (ProcessLookupError, PermissionError):
  116. pass
  117. if pgid is not None and pgid != proc.pid:
  118. # The child does not lead its own group, so it shares ours: killpg would
  119. # signal the whole runner process tree. Signal the direct child only.
  120. pgid = None
  121. def stop(sig: int, fallback: Callable[[], None]) -> None:
  122. if pgid is not None:
  123. killpg(pgid, sig) # windows-footgun: ok — POSIX-only branch (pgid only set when killpg exists)
  124. else:
  125. fallback()
  126. try:
  127. stop(signal.SIGTERM, proc.terminate)
  128. except (ProcessLookupError, PermissionError):
  129. return
  130. try:
  131. proc.wait(timeout=10)
  132. except subprocess.TimeoutExpired:
  133. try:
  134. stop(getattr(signal, "SIGKILL", signal.SIGTERM), proc.kill)
  135. except (ProcessLookupError, PermissionError):
  136. pass
  137. try:
  138. proc.wait(timeout=5)
  139. except subprocess.TimeoutExpired:
  140. pass
  141. def _run_start_phase(
  142. recipe: Recipe, root: Path, ready_timeout: float, port_override: int | None = None
  143. ) -> ReadinessResult:
  144. assert recipe.start is not None
  145. port = port_override or recipe.port or 8000
  146. url = f"http://127.0.0.1:{port}{recipe.readiness_path}"
  147. started = time.monotonic()
  148. # Owned: the app leads its own group (clean group teardown below) and that group dies with this
  149. # process if it is killed mid-poll, instead of leaving the dev server running on the port.
  150. proc = spawn_owned(recipe.start, purpose="verify-app", cwd=str(root), **_SUBPROCESS_KW)
  151. output = ""
  152. try:
  153. ready, status, error = _poll_readiness(url, ready_timeout)
  154. finally:
  155. _terminate_process_group(proc)
  156. if proc.poll() is not None:
  157. release_process_group_once_empty(proc.pid, proc=proc)
  158. try:
  159. output = proc.stdout.read() or "" if proc.stdout is not None else ""
  160. except (OSError, ValueError):
  161. output = ""
  162. return ReadinessResult(url, ready, status, time.monotonic() - started, error, _tail(output))
  163. def _compose_live_state_reason(root: Path) -> str | None:
  164. """Why ``docker compose build``/``up`` must not run at *root*, or ``None`` to proceed.
  165. Read-only ``docker compose ps`` probe. Only a missing docker binary proceeds -- the
  166. build phase would fail the same way, so there is nothing to protect. A hung daemon
  167. or a non-zero probe refuses: containers may be live and unobservable, which is
  168. exactly the #103567 loss window.
  169. """
  170. try:
  171. result = subprocess.run(
  172. ["docker", "compose", "ps", "--status", "running", "--format", "{{.Name}}"],
  173. cwd=root, capture_output=True, text=True, timeout=15, stdin=subprocess.DEVNULL,
  174. check=False,
  175. )
  176. except FileNotFoundError:
  177. return None
  178. except subprocess.TimeoutExpired:
  179. return "docker compose ps timed out after 15s; live containers cannot be ruled out"
  180. if result.returncode != 0:
  181. detail = (result.stderr or result.stdout or "").strip().splitlines()
  182. return f"docker compose ps failed (exit {result.returncode}): {detail[-1] if detail else 'no output'}"
  183. names = [line for line in result.stdout.splitlines() if line.strip()]
  184. return f"this compose project already has running container(s): {', '.join(names)}" if names else None
  185. def run_verify(
  186. root: Path, recipe: Recipe, phases: tuple[str, ...] | list[str] | None = None,
  187. phase_timeout: float = DEFAULT_PHASE_TIMEOUT, ready_timeout: float = DEFAULT_READY_TIMEOUT,
  188. skip_start: bool = False, port_override: int | None = None, stop_on_failure: bool = True,
  189. on_output: Callable[[str], None] | None = None,
  190. ) -> VerifyResult:
  191. """Run the selected command phases sequentially, then (unless ``skip_start`` or a
  192. phase failed) boot ``recipe.start``, poll readiness, and tear the process group down.
  193. A ``compose`` recipe refuses outright when the project already has running
  194. containers: ``docker compose build`` + ``up`` replaces them on an image-hash
  195. change, destroying any container-local state they carry -- this has caused a
  196. real state-loss incident (#103567). The check is best-effort and read-only
  197. (``docker compose ps``); when it cannot run at all, verify proceeds rather than
  198. blocking on an unrelated environment gap, matching every other recipe kind's
  199. behavior when its own tooling is unavailable."""
  200. root = Path(root)
  201. selected = tuple(phases) if phases else PHASE_ORDER + ("start",)
  202. result = VerifyResult(recipe_name=recipe.name)
  203. mutating = ("build" in selected) or ("start" in selected and not skip_start)
  204. if recipe.kind == "compose" and mutating:
  205. reason = _compose_live_state_reason(root)
  206. if reason:
  207. result.phases.append(PhaseResult(
  208. phase="build", command=recipe.build[0] if recipe.build else "docker compose build",
  209. exit_code=1, duration=0.0, output_tail=(
  210. f"Refusing to run: {reason}. `docker compose build` + `up` would replace "
  211. "live containers on an image-hash change, destroying any container-local "
  212. "state they carry. If you intend to rebuild this live deployment, run "
  213. "`docker compose build`/`up` yourself."
  214. ),
  215. ))
  216. return result
  217. for phase in PHASE_ORDER:
  218. if phase not in selected:
  219. continue
  220. for command in getattr(recipe, phase):
  221. phase_result = _run_phase_command(phase, command, root, phase_timeout, on_output)
  222. result.phases.append(phase_result)
  223. if not phase_result.ok and stop_on_failure:
  224. return result
  225. if not skip_start and "start" in selected and recipe.start and all(p.ok for p in result.phases):
  226. result.readiness = _run_start_phase(recipe, root, ready_timeout, port_override)
  227. return result