batch_runner.py 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021
  1. #!/usr/bin/env python3
  2. """Batch Agent Runner — run the agent over a JSONL prompt dataset in parallel.
  3. Batches are processed by a multiprocessing pool with per-batch ``batch_N.jsonl`` output,
  4. checkpointing for ``--resume``, trajectories in from/value format, and tool-usage
  5. statistics aggregated across all batches. See ``main`` (fire CLI) for usage.
  6. """
  7. # hermes_bootstrap must be the very first import — UTF-8 stdio on Windows, no-op on POSIX.
  8. try:
  9. import hermes_bootstrap
  10. except ModuleNotFoundError as exc:
  11. # Partial ``hermes update`` (git reset landed, ``uv pip install -e .`` did not).
  12. if exc.name != "hermes_bootstrap":
  13. raise # the bootstrap exists but cannot load: skipping it would skip PM activation
  14. import json
  15. import logging
  16. import contextlib
  17. import os
  18. import time
  19. import traceback
  20. from datetime import datetime
  21. from multiprocessing import Lock, Pool
  22. from pathlib import Path
  23. from typing import Any, Dict, List, Optional, Tuple
  24. import fire
  25. from rich.console import Console
  26. from rich.progress import BarColumn, MofNCompleteColumn, Progress, SpinnerColumn, TextColumn, TimeRemainingColumn
  27. from model_tools import TOOL_TO_TOOLSET_MAP
  28. from run_agent import AIAgent
  29. from toolset_distributions import (
  30. list_distributions,
  31. sample_toolsets_from_distribution,
  32. validate_distribution,
  33. )
  34. logger = logging.getLogger(__name__)
  35. # Auto-derived from model_tools so it stays in sync as tools are added. Gives every
  36. # trajectory a consistent tool_stats schema (Arrow/Parquet for HF datasets) and filters
  37. # corrupted entries (hallucinated tool names) when combining trajectories.
  38. ALL_POSSIBLE_TOOLS = set(TOOL_TO_TOOLSET_MAP.keys())
  39. DEFAULT_TOOL_STATS = {'count': 0, 'success': 0, 'failure': 0}
  40. _REASONING_KEYS = ("total_assistant_turns", "turns_with_reasoning", "turns_without_reasoning")
  41. # BatchRunner.__init__ parameters stored as same-named attributes.
  42. _RUNNER_FIELDS = (
  43. "batch_size", "run_name", "distribution", "max_iterations", "base_url", "api_key", "model",
  44. "num_workers", "verbose", "ephemeral_system_prompt", "log_prefix_chars", "providers_allowed",
  45. "providers_ignored", "providers_order", "provider_sort", "openrouter_min_coding_score",
  46. "reasoning_config", "prefill_messages", "max_samples",
  47. )
  48. # BatchRunner attributes forwarded verbatim to every AIAgent in the worker config.
  49. _AGENT_PASSTHROUGH = (
  50. "base_url", "api_key", "ephemeral_system_prompt", "providers_allowed", "providers_ignored",
  51. "providers_order", "provider_sort", "openrouter_min_coding_score",
  52. "reasoning_config", "prefill_messages",
  53. # Without this, every batch task run is attributed to the "unknown" execution
  54. # surface in shared metrics even though "batch" is a first-class surface.
  55. "platform",
  56. )
  57. def _normalize_tool_stats(tool_stats: dict[str, dict[str, int]]) -> dict[str, dict[str, int]]:
  58. """All possible tools with zero defaults (consistent HF schema), plus any unexpected tools."""
  59. normalized = {
  60. tool: tool_stats[tool].copy() if tool in tool_stats else DEFAULT_TOOL_STATS.copy()
  61. for tool in ALL_POSSIBLE_TOOLS
  62. }
  63. for tool, stats in tool_stats.items():
  64. if tool not in normalized:
  65. normalized[tool] = stats.copy()
  66. return normalized
  67. def _normalize_tool_error_counts(tool_error_counts: dict[str, int]) -> dict[str, int]:
  68. """All possible tools with zero defaults, plus any unexpected tools."""
  69. normalized = {tool: tool_error_counts.get(tool, 0) for tool in ALL_POSSIBLE_TOOLS}
  70. for tool, count in tool_error_counts.items():
  71. if tool not in normalized:
  72. normalized[tool] = count
  73. return normalized
  74. def _merge_tool_stats(total: dict[str, dict[str, int]], tool_stats: dict[str, dict[str, int]]) -> None:
  75. """Add per-tool count/success/failure from *tool_stats* into *total* in place."""
  76. for tool_name, stats in tool_stats.items():
  77. agg = total.setdefault(tool_name, DEFAULT_TOOL_STATS.copy())
  78. agg["count"] += stats["count"]
  79. agg["success"] += stats["success"]
  80. agg["failure"] += stats["failure"]
  81. def _merge_reasoning_stats(total: dict[str, int], reasoning_stats: dict[str, Any]) -> None:
  82. """Add the turn counters from *reasoning_stats* into *total* in place."""
  83. for key in total:
  84. total[key] += reasoning_stats.get(key, 0)
  85. def _tool_call_succeeded(content) -> bool:
  86. """Judge a tool response: JSON with a non-null ``error`` / ``success: false`` fails;
  87. non-JSON fails only when empty or starting with ``Error:`` (no substring matching, to
  88. avoid false positives). Non-zero exit codes are NOT failures — the model self-corrects."""
  89. try:
  90. content_json = json.loads(content) if isinstance(content, str) else content
  91. except (json.JSONDecodeError, ValueError, TypeError):
  92. return bool(content) and not content.strip().lower().startswith("error:")
  93. if not isinstance(content_json, dict):
  94. return True
  95. if content_json.get("error") is not None:
  96. return False
  97. # Terminal wraps its response in a "content" field.
  98. inner_content = content_json.get("content")
  99. if isinstance(inner_content, dict) and inner_content.get("error") is not None:
  100. return False
  101. return content_json.get("success") is not False
  102. def _extract_tool_stats(messages: list[dict[str, Any]]) -> dict[str, dict[str, int]]:
  103. """Per-tool call counts and success/failure tallies from a message history."""
  104. tool_stats = {}
  105. tool_calls_map = {} # tool_call_id -> tool name
  106. for msg in messages:
  107. if msg["role"] == "assistant" and "tool_calls" in msg and msg["tool_calls"]:
  108. for tool_call in msg["tool_calls"]:
  109. if not tool_call or not isinstance(tool_call, dict): continue
  110. tool_name = tool_call["function"]["name"]
  111. tool_stats.setdefault(tool_name, DEFAULT_TOOL_STATS.copy())["count"] += 1
  112. tool_calls_map[tool_call["id"]] = tool_name
  113. elif msg["role"] == "tool":
  114. tool_call_id = msg.get("tool_call_id", "")
  115. is_success = _tool_call_succeeded(msg.get("content", ""))
  116. if tool_call_id in tool_calls_map:
  117. tool_stats[tool_calls_map[tool_call_id]]["success" if is_success else "failure"] += 1
  118. return tool_stats
  119. def _turn_has_reasoning(msg: dict[str, Any]) -> bool:
  120. """``<REASONING_SCRATCHPAD>`` in content, or a non-empty native ``reasoning`` field."""
  121. if "<REASONING_SCRATCHPAD>" in (msg.get("content", "") or ""):
  122. return True
  123. return bool(msg.get("reasoning", "").strip()) if msg.get("reasoning") else False
  124. def _extract_reasoning_stats(messages: list[dict[str, Any]]) -> dict[str, int]:
  125. """Count assistant turns with reasoning vs without."""
  126. assistant_turns = [msg for msg in messages if msg.get("role") == "assistant"]
  127. total = len(assistant_turns)
  128. with_reasoning = sum(1 for msg in assistant_turns if _turn_has_reasoning(msg))
  129. return {
  130. "total_assistant_turns": total,
  131. "turns_with_reasoning": with_reasoning,
  132. "turns_without_reasoning": total - with_reasoning,
  133. "has_any_reasoning": with_reasoning > 0,
  134. }
  135. def _failure_result(prompt_index: int, batch_num: int, error: str) -> dict[str, Any]:
  136. """Result dict for a prompt that produced no trajectory."""
  137. return {
  138. "success": False,
  139. "prompt_index": prompt_index,
  140. "error": error,
  141. "trajectory": None,
  142. "tool_stats": {},
  143. "toolsets_used": [],
  144. "metadata": {"batch_num": batch_num, "timestamp": datetime.now().isoformat()},
  145. }
  146. def _prepare_container_image(
  147. prompt_index: int, prompt_data: dict[str, Any], batch_num: int, task_id: str, config: dict[str, Any]
  148. ) -> Optional[dict[str, Any]]:
  149. """Register the dataset row's per-prompt container image (``image``/``docker_image``)
  150. for this task's sandbox (Docker, Modal, Daytona, or a plugin backend such as Singularity).
  151. For Docker the image is verified (local cache, then pull) before spending tokens on the
  152. agent loop; Modal pulls server-side so no local check. Returns a failure result when the
  153. pull fails, else ``None``.
  154. """
  155. container_image = prompt_data.get("image") or prompt_data.get("docker_image")
  156. if not container_image:
  157. return None
  158. env_type = os.getenv("TERMINAL_ENV", "local")
  159. if env_type == "docker":
  160. import subprocess as _sp
  161. try:
  162. probe = _sp.run(
  163. ["docker", "image", "inspect", container_image],
  164. capture_output=True, timeout=10,
  165. check=False,
  166. )
  167. if probe.returncode != 0:
  168. if config.get("verbose"):
  169. print(f" Prompt {prompt_index}: Pulling docker image {container_image}...", flush=True)
  170. pull = _sp.run(
  171. ["docker", "pull", container_image],
  172. capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=600,
  173. check=False,
  174. )
  175. if pull.returncode != 0:
  176. return _failure_result(
  177. prompt_index, batch_num,
  178. f"Docker image not available: {container_image}\n{pull.stderr[:500]}",
  179. )
  180. except FileNotFoundError:
  181. pass # Docker CLI not installed — skip check (e.g., Modal backend)
  182. except Exception as img_err:
  183. if config.get("verbose"):
  184. print(f" Prompt {prompt_index}: Docker image check failed: {img_err}", flush=True)
  185. from tools.terminal_tool import register_task_env_overrides
  186. overrides = {
  187. "docker_image": container_image,
  188. "modal_image": container_image,
  189. "daytona_image": container_image,
  190. # A plugin backend reads its own ``<backend>_image`` override (tools.terminal_tool._select_image).
  191. f"{env_type}_image": container_image,
  192. }
  193. if prompt_data.get("cwd"):
  194. overrides["cwd"] = prompt_data["cwd"]
  195. register_task_env_overrides(task_id, overrides)
  196. if config.get("verbose"):
  197. print(f" Prompt {prompt_index}: Using container image {container_image}")
  198. return None
  199. def _process_single_prompt(
  200. prompt_index: int,
  201. prompt_data: dict[str, Any],
  202. batch_num: int,
  203. config: dict[str, Any]
  204. ) -> dict[str, Any]:
  205. """Run the agent on one prompt; returns trajectory, stats and metadata (or a failure result)."""
  206. prompt = prompt_data["prompt"]
  207. task_id = f"task_{prompt_index}"
  208. failure = _prepare_container_image(prompt_index, prompt_data, batch_num, task_id, config)
  209. if failure is not None:
  210. return failure
  211. try:
  212. selected_toolsets = sample_toolsets_from_distribution(config["distribution"])
  213. if config.get("verbose"):
  214. print(f" Prompt {prompt_index}: Using toolsets {selected_toolsets}")
  215. agent = AIAgent(
  216. model=config["model"],
  217. max_iterations=config["max_iterations"],
  218. enabled_toolsets=selected_toolsets,
  219. save_trajectories=False, # We handle saving ourselves
  220. verbose_logging=config.get("verbose", False),
  221. log_prefix_chars=config.get("log_prefix_chars", 100),
  222. log_prefix=f"[B{batch_num}:P{prompt_index}]",
  223. skip_context_files=True, # Don't pollute trajectories with SOUL.md/AGENTS.md
  224. skip_memory=True, # Don't use persistent memory in batch runs
  225. **{key: config.get(key) for key in _AGENT_PASSTHROUGH if key != "platform"},
  226. # Batch is a first-class execution surface. Defaulting here (rather than
  227. # relying on the caller's config dict) keeps task-run telemetry attributable
  228. # even for callers that build a config without it.
  229. platform=config.get("platform") or "batch",
  230. )
  231. try:
  232. # task_id ensures each task gets its own isolated VM
  233. result = agent.run_conversation(prompt, task_id=task_id)
  234. # Stats before conversion — keep the original evaluation order.
  235. tool_stats = _extract_tool_stats(result["messages"])
  236. reasoning_stats = _extract_reasoning_stats(result["messages"])
  237. trajectory = agent._convert_to_trajectory_format(result["messages"], prompt, result["completed"])
  238. finally:
  239. # One agent per prompt, N prompts per batch process: an unclosed
  240. # agent per prompt leaks terminals/VMs/httpx clients for the
  241. # batch's whole run (#50197).
  242. with contextlib.suppress(Exception):
  243. agent.close()
  244. return {
  245. "success": True,
  246. "prompt_index": prompt_index,
  247. "trajectory": trajectory,
  248. "tool_stats": tool_stats,
  249. "reasoning_stats": reasoning_stats,
  250. "completed": result["completed"],
  251. # Sibling of the non-empty-response return below (#64686): the classifier's failure_reason must
  252. # survive the empty-response normalization path too, or downstream consumers (TUI billing
  253. # surface, transient-failure persistence) lose the structured reason exactly when the run
  254. # produced no text.
  255. "partial": result.get("partial", False),
  256. "api_calls": result["api_calls"],
  257. "toolsets_used": selected_toolsets,
  258. "metadata": {"batch_num": batch_num, "timestamp": datetime.now().isoformat(), "model": config["model"]},
  259. }
  260. except Exception as e:
  261. print(f"❌ Error processing prompt {prompt_index}: {e}")
  262. if config.get("verbose"):
  263. traceback.print_exc()
  264. return _failure_result(prompt_index, batch_num, str(e))
  265. def _append_jsonl(path: Path, row: dict[str, Any]) -> None:
  266. """Append one JSON row and fsync so a crash never loses an acknowledged prompt."""
  267. with open(path, 'a', encoding='utf-8') as f:
  268. f.write(json.dumps(row, ensure_ascii=False) + "\n")
  269. f.flush()
  270. os.fsync(f.fileno())
  271. def _process_batch_worker(args: tuple) -> dict[str, Any]:
  272. """Pool worker: process one batch of ``(index, prompt_data)`` sequentially.
  273. ``args`` is ``(batch_num, batch_data, output_dir, completed_prompts, config)``.
  274. """
  275. batch_num, batch_data, output_dir, completed_prompts_set, config = args
  276. output_dir = Path(output_dir)
  277. print(f"\n🔄 Batch {batch_num}: Starting ({len(batch_data)} prompts)")
  278. batch_output_file = output_dir / f"batch_{batch_num}.jsonl"
  279. prompts_to_process = [(idx, data) for idx, data in batch_data if idx not in completed_prompts_set]
  280. if not prompts_to_process:
  281. print(f"✅ Batch {batch_num}: Already completed (skipping)")
  282. return {"batch_num": batch_num, "processed": 0, "skipped": len(batch_data), "tool_stats": {}, "completed_prompts": []}
  283. print(f" Processing {len(prompts_to_process)} prompts (skipping {len(batch_data) - len(prompts_to_process)} already completed)")
  284. batch_tool_stats = {}
  285. batch_reasoning_stats = dict.fromkeys(_REASONING_KEYS, 0)
  286. completed_in_batch = []
  287. discarded_no_reasoning = 0
  288. for prompt_index, prompt_data in prompts_to_process:
  289. result = _process_single_prompt(prompt_index, prompt_data, batch_num, config)
  290. if result["success"] and result["trajectory"]:
  291. reasoning = result.get("reasoning_stats", {})
  292. if not reasoning.get("has_any_reasoning", True):
  293. print(f" 🚫 Prompt {prompt_index} discarded (no reasoning in any turn)")
  294. discarded_no_reasoning += 1
  295. completed_in_batch.append(prompt_index)
  296. # Tombstone row (#93527): resume filters by scanning batch_*.jsonl
  297. # rows for prompt content, so a discarded sample without a row would
  298. # be re-run at full cost on every restart. The merge step excludes
  299. # tombstones from trajectories.jsonl.
  300. _append_jsonl(batch_output_file, {
  301. "prompt_index": prompt_index,
  302. "discarded": "no_reasoning",
  303. "prompt": _entry_prompt_text(prompt_data),
  304. })
  305. continue
  306. # Normalize for a consistent schema across all entries.
  307. raw_tool_stats = result.get("tool_stats", {})
  308. raw_error_counts = {
  309. tool_name: stats.get("failure", 0)
  310. for tool_name, stats in raw_tool_stats.items()
  311. }
  312. _append_jsonl(batch_output_file, {
  313. "prompt_index": prompt_index,
  314. "conversations": result["trajectory"],
  315. "metadata": result["metadata"],
  316. "completed": result["completed"],
  317. "partial": result.get("partial", False), # True if stopped due to invalid tool calls
  318. "api_calls": result["api_calls"],
  319. "toolsets_used": result["toolsets_used"],
  320. "tool_stats": _normalize_tool_stats(raw_tool_stats), # {tool: {count, success, failure}}
  321. "tool_error_counts": _normalize_tool_error_counts(raw_error_counts) # {tool: failure_count}
  322. })
  323. _merge_tool_stats(batch_tool_stats, result.get("tool_stats", {}))
  324. _merge_reasoning_stats(batch_reasoning_stats, result.get("reasoning_stats", {}))
  325. # Only mark as completed if successfully saved (failed prompts can be retried on resume)
  326. if result["success"] and result["trajectory"]:
  327. completed_in_batch.append(prompt_index)
  328. status = "⚠️ partial" if result.get("partial") else "✅"
  329. print(f" {status} Prompt {prompt_index} completed")
  330. else:
  331. print(f" ❌ Prompt {prompt_index} failed (will retry on resume)")
  332. print(f"✅ Batch {batch_num}: Completed ({len(prompts_to_process)} prompts processed)")
  333. return {
  334. "batch_num": batch_num,
  335. "processed": len(prompts_to_process),
  336. "skipped": len(batch_data) - len(prompts_to_process),
  337. "tool_stats": batch_tool_stats,
  338. "reasoning_stats": batch_reasoning_stats,
  339. "discarded_no_reasoning": discarded_no_reasoning,
  340. "completed_prompts": completed_in_batch
  341. }
  342. def _entry_prompt_text(entry: dict) -> str:
  343. """Human prompt text from a dataset/trajectory entry: flat ``prompt``, ShareGPT
  344. ``conversations`` (from/value), chat ``conversations``/``messages`` (role/content),
  345. or a no-reasoning discard tombstone."""
  346. if not isinstance(entry, dict):
  347. return ""
  348. text = str(entry.get("prompt") or "").strip()
  349. if text:
  350. return text
  351. for key in ("conversations", "messages"):
  352. for msg in entry.get(key, []) or []:
  353. if not isinstance(msg, dict):
  354. continue
  355. role = msg.get("role") or msg.get("from")
  356. if role in {"user", "human"}:
  357. text = str(msg.get("content") or msg.get("value") or "").strip()
  358. if text:
  359. return text
  360. return ""
  361. def _banner(title: str) -> None:
  362. print("\n" + "=" * 70)
  363. print(title)
  364. print("=" * 70)
  365. def _chunk(entries: list[tuple[int, dict[str, Any]]], size: int) -> list[list[tuple[int, dict[str, Any]]]]:
  366. """Split ``(index, entry)`` tuples into batches of *size*, preserving original indices."""
  367. return [entries[i:i + size] for i in range(0, len(entries), size)]
  368. class BatchRunner:
  369. """Manages batch processing of agent prompts with checkpointing and statistics."""
  370. def __init__(
  371. self,
  372. dataset_file: str,
  373. batch_size: int,
  374. run_name: str,
  375. distribution: str = "default",
  376. max_iterations: int = 10,
  377. base_url: str | None = None,
  378. api_key: str | None = None,
  379. model: str = "claude-opus-4-20250514",
  380. num_workers: int = 4,
  381. verbose: bool = False,
  382. ephemeral_system_prompt: str | None = None,
  383. log_prefix_chars: int = 100,
  384. providers_allowed: list[str] | None = None,
  385. providers_ignored: list[str] | None = None,
  386. providers_order: list[str] | None = None,
  387. provider_sort: str | None = None,
  388. openrouter_min_coding_score: Optional[float] = None,
  389. reasoning_config: dict[str, Any] | None = None,
  390. prefill_messages: list[dict[str, Any]] | None = None,
  391. max_samples: int | None = None,
  392. ):
  393. """Load the dataset (truncated to *max_samples*), validate *distribution*, create batches.
  394. ``ephemeral_system_prompt`` is used during execution but NOT saved to trajectories.
  395. ``prefill_messages`` are prepended as few-shot context; Anthropic Sonnet/Opus 4.6+
  396. reject a trailing assistant-role prefill (400) — use user-role priming for those.
  397. """
  398. params = dict(locals())
  399. self.dataset_file = Path(dataset_file)
  400. for name in _RUNNER_FIELDS:
  401. setattr(self, name, params[name])
  402. # Batch runs are their own execution surface; declaring it here keeps every
  403. # worker's task-run telemetry attributable instead of falling back to "unknown".
  404. self.platform = "batch"
  405. if not validate_distribution(distribution):
  406. raise ValueError(f"Unknown distribution: {distribution}. Available: {list(list_distributions().keys())}")
  407. self.output_dir = Path("data") / run_name
  408. self.output_dir.mkdir(parents=True, exist_ok=True)
  409. self.checkpoint_file = self.output_dir / "checkpoint.json"
  410. self.stats_file = self.output_dir / "statistics.json"
  411. self.dataset = self._load_dataset()
  412. if self.max_samples and self.max_samples < len(self.dataset):
  413. full_count = len(self.dataset)
  414. self.dataset = self.dataset[:self.max_samples]
  415. print(f"✂️ Truncated dataset from {full_count} to {self.max_samples} samples (--max_samples)")
  416. self.batches = self._create_batches()
  417. print("📊 Batch Runner Initialized")
  418. print(f" Dataset: {self.dataset_file} ({len(self.dataset)} prompts)")
  419. print(f" Batch size: {self.batch_size}")
  420. print(f" Total batches: {len(self.batches)}")
  421. print(f" Run name: {self.run_name}")
  422. print(f" Distribution: {self.distribution}")
  423. print(f" Output directory: {self.output_dir}")
  424. print(f" Workers: {self.num_workers}")
  425. if self.ephemeral_system_prompt:
  426. prompt_preview = self.ephemeral_system_prompt[:60] + "..." if len(self.ephemeral_system_prompt) > 60 else self.ephemeral_system_prompt
  427. print(f" 🔒 Ephemeral system prompt: '{prompt_preview}'")
  428. def _load_dataset(self) -> list[dict[str, Any]]:
  429. """Load JSONL entries that have a ``prompt`` field; skip blank/invalid lines."""
  430. if not self.dataset_file.exists():
  431. raise FileNotFoundError(f"Dataset file not found: {self.dataset_file}")
  432. dataset = []
  433. with open(self.dataset_file, 'r', encoding='utf-8') as f:
  434. for line_num, line in enumerate(f, 1):
  435. line = line.strip()
  436. if not line:
  437. continue
  438. try:
  439. entry = json.loads(line)
  440. except json.JSONDecodeError as e:
  441. print(f"⚠️ Warning: Invalid JSON on line {line_num}: {e}")
  442. continue
  443. if not isinstance(entry, dict) or 'prompt' not in entry:
  444. print(f"⚠️ Warning: Line {line_num} missing 'prompt' field, skipping")
  445. continue
  446. dataset.append(entry)
  447. if not dataset:
  448. raise ValueError(f"No valid entries found in dataset file: {self.dataset_file}")
  449. return dataset
  450. def _create_batches(self) -> list[list[tuple[int, dict[str, Any]]]]:
  451. """Split the dataset into batches of ``(index, entry)`` tuples."""
  452. return _chunk(list(enumerate(self.dataset)), self.batch_size)
  453. def _empty_checkpoint(self) -> dict[str, Any]:
  454. return {"run_name": self.run_name, "completed_prompts": [], "batch_stats": {}, "last_updated": None}
  455. def _load_checkpoint(self) -> dict[str, Any]:
  456. """Checkpoint data (completed prompt indices), or an empty one if missing/unreadable."""
  457. if not self.checkpoint_file.exists():
  458. return self._empty_checkpoint()
  459. try:
  460. with open(self.checkpoint_file, 'r', encoding='utf-8') as f:
  461. return json.load(f)
  462. except Exception as e:
  463. print(f"⚠️ Warning: Failed to load checkpoint: {e}")
  464. return self._empty_checkpoint()
  465. def _save_checkpoint(self, checkpoint_data: dict[str, Any], lock: Optional[Lock] = None):
  466. """Atomically write *checkpoint_data* (stamped ``last_updated``), under *lock* if given."""
  467. checkpoint_data["last_updated"] = datetime.now().isoformat()
  468. from utils import atomic_json_write
  469. if lock:
  470. with lock:
  471. atomic_json_write(self.checkpoint_file, checkpoint_data)
  472. else:
  473. atomic_json_write(self.checkpoint_file, checkpoint_data)
  474. def _scan_completed_prompts_by_content(self) -> set:
  475. """Prompt texts already processed, scanned from every ``batch_*.jsonl``.
  476. Matching on content rather than index lets resume recover even when indices
  477. don't line up. Failed entries are skipped (retried); discard tombstones count
  478. as completed (#93527) — re-running would just re-discard.
  479. """
  480. completed_prompts = set()
  481. batch_files = sorted(self.output_dir.glob("batch_*.jsonl"))
  482. if not batch_files:
  483. return completed_prompts
  484. print(f"📂 Scanning {len(batch_files)} batch files for completed prompts...")
  485. for batch_file in batch_files:
  486. try:
  487. with open(batch_file, 'r', encoding='utf-8') as f:
  488. for line in f:
  489. try:
  490. entry = json.loads(line.strip())
  491. if not isinstance(entry, dict) or entry.get("failed", False):
  492. continue
  493. prompt_text = _entry_prompt_text(entry)
  494. if prompt_text:
  495. completed_prompts.add(prompt_text)
  496. except json.JSONDecodeError:
  497. continue
  498. except Exception as e:
  499. print(f" ⚠️ Warning: Error reading {batch_file.name}: {e}")
  500. return completed_prompts
  501. def _filter_dataset_by_completed(self, completed_prompts: set) -> tuple[list[dict], list[int]]:
  502. """Return ``([(index, entry)] not yet completed, [skipped indices])``."""
  503. filtered_dataset = []
  504. skipped_indices = []
  505. for idx, entry in enumerate(self.dataset):
  506. prompt_text = entry.get("prompt", "").strip()
  507. # Also check conversations format
  508. if not prompt_text:
  509. conversations = entry.get("conversations", [])
  510. for msg in conversations:
  511. role = msg.get("role") or msg.get("from")
  512. if role in {"user", "human"}:
  513. prompt_text = (msg.get("content") or msg.get("value", "")).strip()
  514. break
  515. if prompt_text in completed_prompts:
  516. skipped_indices.append(idx)
  517. else:
  518. filtered_dataset.append((idx, entry))
  519. return filtered_dataset, skipped_indices
  520. def _apply_resume(self) -> bool:
  521. """Rebuild ``self.batches`` from unprocessed prompts. False when nothing is left to run."""
  522. completed_prompt_texts = self._scan_completed_prompts_by_content()
  523. if not completed_prompt_texts:
  524. return True
  525. print(f" Found {len(completed_prompt_texts)} already-completed prompts by content matching")
  526. filtered_entries, skipped_indices = self._filter_dataset_by_completed(completed_prompt_texts)
  527. if not filtered_entries:
  528. print("\n✅ All prompts have already been processed!")
  529. return False
  530. self.batches = _chunk(filtered_entries, self.batch_size)
  531. _banner("📊 RESUME SUMMARY")
  532. print(f" Original dataset size: {len(self.dataset):,} prompts")
  533. print(f" Already completed: {len(skipped_indices):,} prompts")
  534. print(" ─────────────────────────────────────────")
  535. print(f" 🎯 RESUMING WITH: {len(filtered_entries):,} prompts")
  536. print(f" New batches created: {len(self.batches)}")
  537. print("=" * 70 + "\n")
  538. return True
  539. def _worker_config(self) -> dict[str, Any]:
  540. """Picklable agent configuration for worker processes.
  541. ``self.api_key`` may be a zero-arg callable (Azure Foundry Entra ID bearer provider
  542. from ``agent.azure_identity_adapter``), which is not safely picklable across the
  543. Pool boundary. Drop it and let each worker rebuild its own provider via
  544. ``resolve_runtime_provider()`` from ``model.auth_mode`` in config.yaml
  545. (azure-identity caches in-process, so each worker gets its own short-lived cache).
  546. """
  547. if callable(self.api_key) and not isinstance(self.api_key, str):
  548. worker_api_key = None
  549. print(
  550. "ℹ️ Detected Entra ID bearer provider — workers will rebuild "
  551. "credentials from config.yaml in each process.",
  552. flush=True,
  553. )
  554. else:
  555. worker_api_key = self.api_key
  556. config = {key: getattr(self, key) for key in _AGENT_PASSTHROUGH}
  557. config["api_key"] = worker_api_key
  558. for key in ("distribution", "model", "max_iterations", "verbose", "log_prefix_chars"):
  559. config[key] = getattr(self, key)
  560. return config
  561. def _run_pool(self, config, checkpoint_data, completed_prompts_set, checkpoint_lock) -> list[dict[str, Any]]:
  562. """Process all batches in a worker pool, checkpointing after each result."""
  563. print(f"\n🔧 Initializing {self.num_workers} worker processes...")
  564. with Pool(processes=self.num_workers) as pool:
  565. # output_dir as str for pickling
  566. tasks = [
  567. (batch_num, batch_data, str(self.output_dir), completed_prompts_set, config)
  568. for batch_num, batch_data in enumerate(self.batches)
  569. ]
  570. print(f"✅ Created {len(tasks)} batch tasks")
  571. print("🚀 Starting parallel batch processing...\n")
  572. # rich Progress gives a persistent bottom bar; stdout/stderr are NOT
  573. # redirected so worker prints stay visible.
  574. results = []
  575. console = Console(force_terminal=True)
  576. with Progress(
  577. SpinnerColumn(), TextColumn("[bold blue]📦 Batches"), BarColumn(bar_width=40),
  578. MofNCompleteColumn(), TextColumn("•"), TimeRemainingColumn(),
  579. console=console, refresh_per_second=2, transient=False,
  580. redirect_stdout=False, redirect_stderr=False,
  581. ) as progress:
  582. task = progress.add_task("Processing", total=len(tasks))
  583. # Temporarily suppress DEBUG logging to avoid bar interference
  584. root_logger = logging.getLogger()
  585. original_level = root_logger.level
  586. root_logger.setLevel(logging.WARNING)
  587. try:
  588. for result in pool.imap_unordered(_process_batch_worker, tasks):
  589. results.append(result)
  590. progress.update(task, advance=1)
  591. # Incremental checkpoint update (so resume works after crash)
  592. try:
  593. batch_num = result.get('batch_num')
  594. completed = result.get('completed_prompts', []) or []
  595. completed_prompts_set.update(completed)
  596. if isinstance(batch_num, int):
  597. checkpoint_data.setdefault('batch_stats', {})[str(batch_num)] = {
  598. key: result.get(key, 0) for key in ('processed', 'skipped', 'discarded_no_reasoning')
  599. }
  600. checkpoint_data['completed_prompts'] = sorted(completed_prompts_set)
  601. self._save_checkpoint(checkpoint_data, lock=checkpoint_lock)
  602. except Exception as ckpt_err:
  603. # Don't fail the run if checkpoint write fails
  604. print(f"⚠️ Warning: Failed to save incremental checkpoint: {ckpt_err}")
  605. except KeyboardInterrupt:
  606. print("\n⚠️ Interrupted — terminating batch workers...")
  607. pool.terminate()
  608. pool.join()
  609. raise
  610. except Exception as e:
  611. logger.error("Batch worker failed: %s", e, exc_info=True)
  612. pool.terminate()
  613. pool.join()
  614. raise
  615. finally:
  616. root_logger.setLevel(original_level)
  617. return results
  618. def _combine_batch_files(self) -> tuple[int, int]:
  619. """Merge ALL ``batch_*.jsonl`` (old runs + resume) into ``trajectories.jsonl``.
  620. Drops corrupted entries (hallucinated tool names, invalid JSON) and discard
  621. tombstones (#93527, resume bookkeeping only). Returns ``(kept, files_found)``.
  622. """
  623. combined_file = self.output_dir / "trajectories.jsonl"
  624. print(f"\n📦 Combining ALL batch files into {combined_file.name}...")
  625. total_entries = 0
  626. filtered_entries = 0
  627. tombstone_entries = 0
  628. batch_files_found = 0
  629. all_batch_files = sorted(self.output_dir.glob("batch_*.jsonl"))
  630. with open(combined_file, 'w', encoding='utf-8') as outfile:
  631. for batch_file in all_batch_files:
  632. batch_files_found += 1
  633. batch_num = batch_file.stem.split("_")[1] # Extract batch number for logging
  634. with open(batch_file, 'r', encoding='utf-8') as infile:
  635. for line in infile:
  636. total_entries += 1
  637. try:
  638. data = json.loads(line)
  639. if not isinstance(data, dict):
  640. filtered_entries += 1
  641. continue
  642. if data.get("discarded"):
  643. tombstone_entries += 1
  644. continue
  645. tool_stats = data.get('tool_stats', {})
  646. invalid_tools = [k for k in tool_stats if k not in ALL_POSSIBLE_TOOLS]
  647. if invalid_tools:
  648. filtered_entries += 1
  649. invalid_preview = invalid_tools[0][:50] + "..." if len(invalid_tools[0]) > 50 else invalid_tools[0]
  650. print(f" ⚠️ Filtering corrupted entry (batch {batch_num}): invalid tool '{invalid_preview}'")
  651. continue
  652. outfile.write(line)
  653. except json.JSONDecodeError:
  654. filtered_entries += 1
  655. print(f" ⚠️ Filtering invalid JSON entry (batch {batch_num})")
  656. if filtered_entries > 0:
  657. print(f"⚠️ Filtered {filtered_entries} corrupted entries out of {total_entries} total")
  658. kept = total_entries - filtered_entries - tombstone_entries
  659. print(f"✅ Combined {batch_files_found} batch files into trajectories.jsonl ({kept} entries)")
  660. return kept, batch_files_found
  661. def _print_summary(self, results, total_tool_stats, total_reasoning_stats, kept, batch_files_found, start_time) -> None:
  662. _banner("📊 BATCH PROCESSING COMPLETE")
  663. print(f"✅ Prompts processed this run: {sum(r.get('processed', 0) for r in results)}")
  664. print(f"✅ Total trajectories in merged file: {kept}")
  665. print(f"✅ Total batch files merged: {batch_files_found}")
  666. print(f"⏱️ Total duration: {round(time.time() - start_time, 2)}s")
  667. print("\n📈 Tool Usage Statistics:")
  668. print("-" * 70)
  669. if total_tool_stats:
  670. sorted_tools = sorted(total_tool_stats.items(), key=lambda x: x[1]["count"], reverse=True)
  671. print(f"{'Tool Name':<25} {'Count':<10} {'Success':<10} {'Failure':<10} {'Success Rate':<12}")
  672. print("-" * 70)
  673. for tool_name, stats in sorted_tools:
  674. print(f"{tool_name:<25} {stats['count']:<10} {stats['success']:<10} {stats['failure']:<10} {stats['success_rate']:.1f}%")
  675. else:
  676. print("No tool calls were made during this run.")
  677. total_discarded = sum(r.get("discarded_no_reasoning", 0) for r in results)
  678. print("\n🧠 Reasoning Coverage:")
  679. print("-" * 70)
  680. total_turns = total_reasoning_stats["total_assistant_turns"]
  681. with_reasoning = total_reasoning_stats["turns_with_reasoning"]
  682. without_reasoning = total_reasoning_stats["turns_without_reasoning"]
  683. if total_turns > 0:
  684. pct_with = round(with_reasoning / total_turns * 100, 1)
  685. pct_without = round(without_reasoning / total_turns * 100, 1)
  686. print(f" Total assistant turns: {total_turns:,}")
  687. print(f" With reasoning: {with_reasoning:,} ({pct_with}%)")
  688. print(f" Without reasoning: {without_reasoning:,} ({pct_without}%)")
  689. else:
  690. print(" No assistant turns recorded.")
  691. if total_discarded > 0:
  692. print(f" 🚫 Samples discarded (zero reasoning): {total_discarded:,}")
  693. print(f"\n💾 Results saved to: {self.output_dir}")
  694. print(" - Trajectories: trajectories.jsonl (combined)")
  695. print(" - Individual batches: batch_*.jsonl (for debugging)")
  696. print(f" - Statistics: {self.stats_file.name}")
  697. print(f" - Checkpoint: {self.checkpoint_file.name}")
  698. def run(self, resume: bool = False):
  699. """Run the batch pipeline; with *resume*, skip prompts already present in batch files."""
  700. _banner("🚀 Starting Batch Processing")
  701. if resume and not self._apply_resume():
  702. return
  703. # Load existing checkpoint (so resume doesn't clobber prior progress)
  704. checkpoint_data = self._load_checkpoint()
  705. if checkpoint_data.get("run_name") != self.run_name:
  706. checkpoint_data = self._empty_checkpoint()
  707. config = self._worker_config()
  708. # Index tracking is secondary to content matching (backward compatibility).
  709. completed_prompts_set = set(checkpoint_data.get("completed_prompts", []))
  710. start_time = time.time()
  711. # Checkpoint writes happen in the parent process; keep a lock for safety.
  712. checkpoint_lock = Lock()
  713. results = self._run_pool(config, checkpoint_data, completed_prompts_set, checkpoint_lock)
  714. total_tool_stats = {}
  715. total_reasoning_stats = dict.fromkeys(_REASONING_KEYS, 0)
  716. for batch_result in results:
  717. _merge_tool_stats(total_tool_stats, batch_result.get("tool_stats", {}))
  718. _merge_reasoning_stats(total_reasoning_stats, batch_result.get("reasoning_stats", {}))
  719. # Final checkpoint is best-effort; incremental writes already happened.
  720. try:
  721. checkpoint_data["completed_prompts"] = sorted(completed_prompts_set)
  722. self._save_checkpoint(checkpoint_data, lock=checkpoint_lock)
  723. except Exception as ckpt_err:
  724. print(f"⚠️ Warning: Failed to save final checkpoint: {ckpt_err}")
  725. for stats in total_tool_stats.values():
  726. total_calls = stats["success"] + stats["failure"]
  727. stats["success_rate"] = round(stats["success"] / total_calls * 100, 2) if total_calls > 0 else 0.0
  728. stats["failure_rate"] = round(stats["failure"] / total_calls * 100, 2) if total_calls > 0 else 0.0
  729. kept, batch_files_found = self._combine_batch_files()
  730. final_stats = {
  731. "run_name": self.run_name,
  732. "distribution": self.distribution,
  733. "total_prompts": len(self.dataset),
  734. "total_batches": len(self.batches),
  735. "batch_size": self.batch_size,
  736. # Snapshot the CLI-level credential/runtime fields BEFORE mutating them so a failed in-place
  737. # agent swap can roll the whole CLI back to the old working model. Otherwise the broken
  738. # credentials staged below leak into the next turn's resolution even though the agent itself
  739. # rolled back (#50163).
  740. # Snapshot CLI-level fields before mutation so a failed in-place swap rolls the whole CLI back
  741. # to the old working model (#50163).
  742. "model": self.model,
  743. "completed_at": datetime.now().isoformat(),
  744. "duration_seconds": round(time.time() - start_time, 2),
  745. "tool_statistics": total_tool_stats,
  746. "reasoning_statistics": total_reasoning_stats,
  747. "discarded_no_reasoning": sum(r.get("discarded_no_reasoning", 0) for r in results),
  748. }
  749. with open(self.stats_file, 'w', encoding='utf-8') as f:
  750. json.dump(final_stats, f, indent=2, ensure_ascii=False)
  751. self._print_summary(results, total_tool_stats, total_reasoning_stats, kept, batch_files_found, start_time)
  752. def _split_csv(value: Optional[str]) -> Optional[list[str]]:
  753. """Comma-separated CLI string to a list of stripped items; ``None`` when empty."""
  754. return [p.strip() for p in value.split(",")] if value else None
  755. def main(
  756. dataset_file: str | None = None,
  757. batch_size: int | None = None,
  758. run_name: str | None = None,
  759. distribution: str = "default",
  760. model: str = "anthropic/claude-sonnet-4.6",
  761. api_key: str | None = None,
  762. base_url: str = "https://openrouter.ai/api/v1",
  763. max_turns: int = 10,
  764. num_workers: int = 4,
  765. resume: bool = False,
  766. verbose: bool = False,
  767. list_distributions: bool = False,
  768. ephemeral_system_prompt: str | None = None,
  769. log_prefix_chars: int = 100,
  770. providers_allowed: str | None = None,
  771. providers_ignored: str | None = None,
  772. providers_order: str | None = None,
  773. provider_sort: str | None = None,
  774. reasoning_effort: str | None = None,
  775. reasoning_disabled: bool = False,
  776. prefill_messages_file: str | None = None,
  777. max_samples: int | None = None,
  778. ):
  779. """
  780. Run batch processing of agent prompts from a dataset.
  781. Args:
  782. dataset_file (str): Path to JSONL file with 'prompt' field in each entry
  783. batch_size (int): Number of prompts per batch
  784. run_name (str): Name for this run (used for output and checkpointing)
  785. distribution (str): Toolset distribution to use (default: "default")
  786. model (str): Model name to use (default: "claude-opus-4-20250514")
  787. api_key (str): API key for model authentication
  788. base_url (str): Base URL for model API
  789. max_turns (int): Maximum number of tool calling iterations per prompt (default: 10)
  790. num_workers (int): Number of parallel worker processes (default: 4)
  791. resume (bool): Resume from checkpoint if run was interrupted (default: False)
  792. verbose (bool): Enable verbose logging (default: False)
  793. list_distributions (bool): List available toolset distributions and exit
  794. ephemeral_system_prompt (str): System prompt used during agent execution but NOT saved to trajectories (optional)
  795. log_prefix_chars (int): Number of characters to show in log previews for tool calls/responses (default: 20)
  796. providers_allowed (str): Comma-separated list of OpenRouter providers to allow (e.g. "anthropic,openai")
  797. providers_ignored (str): Comma-separated list of OpenRouter providers to ignore (e.g. "together,deepinfra")
  798. providers_order (str): Comma-separated list of OpenRouter providers to try in order (e.g. "anthropic,openai,google")
  799. provider_sort (str): Sort providers by "price", "throughput", or "latency" (OpenRouter only)
  800. reasoning_effort (str): Reasoning effort: "none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra" (default: "medium")
  801. reasoning_disabled (bool): Completely disable reasoning/thinking tokens (default: False)
  802. prefill_messages_file (str): Path to JSON file containing prefill messages (list of {role, content} dicts)
  803. max_samples (int): Only process the first N samples from the dataset (optional, processes all if not set)
  804. Examples:
  805. # Basic usage
  806. python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=my_run
  807. # Resume interrupted run
  808. python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=my_run --resume
  809. # Use specific distribution
  810. python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=image_test --distribution=image_gen
  811. # With disabled reasoning
  812. python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=my_run \\
  813. --reasoning_disabled
  814. # With prefill messages from file
  815. python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=my_run \\
  816. --prefill_messages_file=configs/prefill_opus.json
  817. # List available distributions
  818. python batch_runner.py --list_distributions
  819. """
  820. if list_distributions:
  821. from toolset_distributions import print_distribution_info
  822. print("📊 Available Toolset Distributions")
  823. print("=" * 70)
  824. all_dists = list_distributions()
  825. for dist_name in sorted(all_dists.keys()):
  826. print_distribution_info(dist_name)
  827. print("\n💡 Usage:")
  828. print(" python batch_runner.py --dataset_file=data.jsonl --batch_size=10 \\")
  829. print(" --run_name=my_run --distribution=<name>")
  830. return
  831. for invalid, message in (
  832. (not dataset_file, "--dataset_file is required"),
  833. (not batch_size or batch_size < 1, "--batch_size must be a positive integer"),
  834. (not run_name, "--run_name is required"),
  835. ):
  836. if invalid:
  837. print(f"❌ Error: {message}")
  838. raise SystemExit(1)
  839. # --reasoning_disabled takes priority, then --reasoning_effort, then default (medium)
  840. reasoning_config = None
  841. if reasoning_disabled:
  842. reasoning_config = {"effort": "none"}
  843. print("🧠 Reasoning: DISABLED (effort=none)")
  844. elif reasoning_effort:
  845. valid_efforts = ["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"]
  846. if reasoning_effort not in valid_efforts:
  847. print(f"❌ Error: --reasoning_effort must be one of: {', '.join(valid_efforts)}")
  848. raise SystemExit(1)
  849. reasoning_config = {"enabled": True, "effort": reasoning_effort}
  850. print(f"🧠 Reasoning effort: {reasoning_effort}")
  851. prefill_messages = None
  852. if prefill_messages_file:
  853. try:
  854. with open(prefill_messages_file, 'r', encoding='utf-8') as f:
  855. prefill_messages = json.load(f)
  856. if not isinstance(prefill_messages, list):
  857. print("❌ Error: prefill_messages_file must contain a JSON array of messages")
  858. raise SystemExit(1)
  859. print(f"💬 Loaded {len(prefill_messages)} prefill messages from {prefill_messages_file}")
  860. except Exception as e:
  861. print(f"❌ Error loading prefill messages: {e}")
  862. raise SystemExit(1)
  863. try:
  864. runner = BatchRunner(
  865. dataset_file=dataset_file,
  866. batch_size=batch_size,
  867. run_name=run_name,
  868. distribution=distribution,
  869. max_iterations=max_turns,
  870. base_url=base_url,
  871. api_key=api_key,
  872. model=model,
  873. num_workers=num_workers,
  874. verbose=verbose,
  875. ephemeral_system_prompt=ephemeral_system_prompt,
  876. log_prefix_chars=log_prefix_chars,
  877. providers_allowed=_split_csv(providers_allowed),
  878. providers_ignored=_split_csv(providers_ignored),
  879. providers_order=_split_csv(providers_order),
  880. provider_sort=provider_sort,
  881. reasoning_config=reasoning_config,
  882. prefill_messages=prefill_messages,
  883. max_samples=max_samples,
  884. )
  885. runner.run(resume=resume)
  886. except Exception as e:
  887. print(f"\n❌ Fatal error: {e}")
  888. if verbose:
  889. traceback.print_exc()
  890. raise SystemExit(1)
  891. if __name__ == "__main__":
  892. fire.Fire(main)