| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264 |
- """Verification runner: bootstrap -> build -> test -> start in background ->
- readiness loop -> teardown (scoped port of grok-cli's verify flow).
- Commands come from the project's own recipe and run with ``shell=True`` on
- purpose: a developer tool running the project's own build commands in its own
- checkout — the same trust level as the terminal tool.
- """
- from __future__ import annotations
- import os
- import signal
- import subprocess
- import time
- import urllib.error
- import urllib.request
- from dataclasses import dataclass, field
- from pathlib import Path
- from typing import Any, Callable
- from agent.verify.recipes import Recipe
- from hermes_cli.process_identity import release_process_group_once_empty, spawn_owned
- DEFAULT_PHASE_TIMEOUT = 600.0
- DEFAULT_READY_TIMEOUT = 60.0
- _TAIL_CHARS = 2000
- PHASE_ORDER = ("bootstrap", "build", "test")
- # Project-authored shell commands; see module docstring.
- _SUBPROCESS_KW: dict[str, Any] = dict(
- shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
- text=True, errors="replace",
- )
- @dataclass
- class PhaseResult:
- phase: str
- command: str
- exit_code: int | None
- duration: float
- output_tail: str
- timed_out: bool = False
- @property
- def ok(self) -> bool:
- return self.exit_code == 0 and not self.timed_out
- def to_dict(self) -> dict[str, Any]:
- return {
- "phase": self.phase, "command": self.command, "exitCode": self.exit_code,
- "duration": round(self.duration, 3), "ok": self.ok, "timedOut": self.timed_out,
- "outputTail": self.output_tail,
- }
- @dataclass
- class ReadinessResult:
- url: str
- ready: bool
- status_code: int | None
- duration: float
- error: str | None = None
- output_tail: str = ""
- def to_dict(self) -> dict[str, Any]:
- return {
- "url": self.url, "ready": self.ready, "statusCode": self.status_code,
- "duration": round(self.duration, 3), "error": self.error, "outputTail": self.output_tail,
- }
- @dataclass
- class VerifyResult:
- recipe_name: str
- phases: list[PhaseResult] = field(default_factory=list)
- readiness: ReadinessResult | None = None
- @property
- def ok(self) -> bool:
- return all(p.ok for p in self.phases) and (self.readiness is None or self.readiness.ready)
- def to_dict(self) -> dict[str, Any]:
- return {
- "recipe": self.recipe_name, "ok": self.ok,
- "phases": [p.to_dict() for p in self.phases],
- "readiness": self.readiness.to_dict() if self.readiness else None,
- }
- def _tail(text: str, limit: int = _TAIL_CHARS) -> str:
- return text[-limit:] if len(text) > limit else text
- def _run_phase_command(
- phase: str, command: str, root: Path, timeout: float,
- on_output: Callable[[str], None] | None = None,
- ) -> PhaseResult:
- started = time.monotonic()
- try:
- proc = subprocess.run(command, cwd=str(root), timeout=timeout, **_SUBPROCESS_KW, check=False)
- output, exit_code, timed_out = proc.stdout or "", proc.returncode, False
- except subprocess.TimeoutExpired as exc:
- raw = exc.output
- output = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else (raw or "")
- exit_code, timed_out = None, True
- duration = time.monotonic() - started
- if on_output and output:
- on_output(output)
- return PhaseResult(phase, command, exit_code, duration, _tail(output), timed_out)
- def _poll_readiness(url: str, timeout: float, interval: float = 1.0) -> tuple[bool, int | None, str | None]:
- deadline = time.monotonic() + timeout
- last_error: str | None = None
- while time.monotonic() < deadline:
- try:
- with urllib.request.urlopen(url, timeout=5) as resp:
- return True, resp.status, None
- except urllib.error.HTTPError as exc:
- # The server answered — it is up, even if it returned 4xx/5xx.
- return True, exc.code, None
- except (urllib.error.URLError, OSError, TimeoutError) as exc:
- last_error = str(exc)
- time.sleep(interval)
- return False, None, last_error
- def _terminate_process_group(proc: subprocess.Popen) -> None:
- """SIGTERM the app's process group (``start_new_session=True`` on POSIX; just the
- direct child on Windows, which lacks ``os.killpg``), SIGKILL after 10s."""
- if proc.poll() is not None:
- return
- killpg = getattr(os, "killpg", None)
- getpgid = getattr(os, "getpgid", None)
- pgid = None
- if killpg is not None and getpgid is not None:
- try:
- pgid = getpgid(proc.pid)
- except (ProcessLookupError, PermissionError):
- pass
- if pgid is not None and pgid != proc.pid:
- # The child does not lead its own group, so it shares ours: killpg would
- # signal the whole runner process tree. Signal the direct child only.
- pgid = None
- def stop(sig: int, fallback: Callable[[], None]) -> None:
- if pgid is not None:
- killpg(pgid, sig) # windows-footgun: ok — POSIX-only branch (pgid only set when killpg exists)
- else:
- fallback()
- try:
- stop(signal.SIGTERM, proc.terminate)
- except (ProcessLookupError, PermissionError):
- return
- try:
- proc.wait(timeout=10)
- except subprocess.TimeoutExpired:
- try:
- stop(getattr(signal, "SIGKILL", signal.SIGTERM), proc.kill)
- except (ProcessLookupError, PermissionError):
- pass
- try:
- proc.wait(timeout=5)
- except subprocess.TimeoutExpired:
- pass
- def _run_start_phase(
- recipe: Recipe, root: Path, ready_timeout: float, port_override: int | None = None
- ) -> ReadinessResult:
- assert recipe.start is not None
- port = port_override or recipe.port or 8000
- url = f"http://127.0.0.1:{port}{recipe.readiness_path}"
- started = time.monotonic()
- # Owned: the app leads its own group (clean group teardown below) and that group dies with this
- # process if it is killed mid-poll, instead of leaving the dev server running on the port.
- proc = spawn_owned(recipe.start, purpose="verify-app", cwd=str(root), **_SUBPROCESS_KW)
- output = ""
- try:
- ready, status, error = _poll_readiness(url, ready_timeout)
- finally:
- _terminate_process_group(proc)
- if proc.poll() is not None:
- release_process_group_once_empty(proc.pid, proc=proc)
- try:
- output = proc.stdout.read() or "" if proc.stdout is not None else ""
- except (OSError, ValueError):
- output = ""
- return ReadinessResult(url, ready, status, time.monotonic() - started, error, _tail(output))
- def _compose_live_state_reason(root: Path) -> str | None:
- """Why ``docker compose build``/``up`` must not run at *root*, or ``None`` to proceed.
- Read-only ``docker compose ps`` probe. Only a missing docker binary proceeds -- the
- build phase would fail the same way, so there is nothing to protect. A hung daemon
- or a non-zero probe refuses: containers may be live and unobservable, which is
- exactly the #103567 loss window.
- """
- try:
- result = subprocess.run(
- ["docker", "compose", "ps", "--status", "running", "--format", "{{.Name}}"],
- cwd=root, capture_output=True, text=True, timeout=15, stdin=subprocess.DEVNULL,
- check=False,
- )
- except FileNotFoundError:
- return None
- except subprocess.TimeoutExpired:
- return "docker compose ps timed out after 15s; live containers cannot be ruled out"
- if result.returncode != 0:
- detail = (result.stderr or result.stdout or "").strip().splitlines()
- return f"docker compose ps failed (exit {result.returncode}): {detail[-1] if detail else 'no output'}"
- names = [line for line in result.stdout.splitlines() if line.strip()]
- return f"this compose project already has running container(s): {', '.join(names)}" if names else None
- def run_verify(
- root: Path, recipe: Recipe, phases: tuple[str, ...] | list[str] | None = None,
- phase_timeout: float = DEFAULT_PHASE_TIMEOUT, ready_timeout: float = DEFAULT_READY_TIMEOUT,
- skip_start: bool = False, port_override: int | None = None, stop_on_failure: bool = True,
- on_output: Callable[[str], None] | None = None,
- ) -> VerifyResult:
- """Run the selected command phases sequentially, then (unless ``skip_start`` or a
- phase failed) boot ``recipe.start``, poll readiness, and tear the process group down.
- A ``compose`` recipe refuses outright when the project already has running
- containers: ``docker compose build`` + ``up`` replaces them on an image-hash
- change, destroying any container-local state they carry -- this has caused a
- real state-loss incident (#103567). The check is best-effort and read-only
- (``docker compose ps``); when it cannot run at all, verify proceeds rather than
- blocking on an unrelated environment gap, matching every other recipe kind's
- behavior when its own tooling is unavailable."""
- root = Path(root)
- selected = tuple(phases) if phases else PHASE_ORDER + ("start",)
- result = VerifyResult(recipe_name=recipe.name)
- mutating = ("build" in selected) or ("start" in selected and not skip_start)
- if recipe.kind == "compose" and mutating:
- reason = _compose_live_state_reason(root)
- if reason:
- result.phases.append(PhaseResult(
- phase="build", command=recipe.build[0] if recipe.build else "docker compose build",
- exit_code=1, duration=0.0, output_tail=(
- f"Refusing to run: {reason}. `docker compose build` + `up` would replace "
- "live containers on an image-hash change, destroying any container-local "
- "state they carry. If you intend to rebuild this live deployment, run "
- "`docker compose build`/`up` yourself."
- ),
- ))
- return result
- for phase in PHASE_ORDER:
- if phase not in selected:
- continue
- for command in getattr(recipe, phase):
- phase_result = _run_phase_command(phase, command, root, phase_timeout, on_output)
- result.phases.append(phase_result)
- if not phase_result.ok and stop_on_failure:
- return result
- if not skip_start and "start" in selected and recipe.start and all(p.ok for p in result.phases):
- result.readiness = _run_start_phase(recipe, root, ready_timeout, port_override)
- return result
|