mini_swe_runner.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. #!/usr/bin/env python3
  2. """SWE Runner with Hermes Trajectory Format
  3. Runs tool-calling agent tasks in Hermes-Agent's execution environments (local,
  4. docker, modal) and writes trajectories in Hermes format (from/value pairs with
  5. <tool_call>/<tool_response> XML), compatible with batch_runner.py and
  6. trajectory_compressor.py. Supports single tasks and JSONL batch mode.
  7. Usage:
  8. python mini_swe_runner.py --task "Create a hello world Python script" --env local
  9. python mini_swe_runner.py --task "List files in the working directory" --env docker --image python:3.11-slim
  10. python mini_swe_runner.py --prompts_file prompts.jsonl --output_file trajectories.jsonl --env docker
  11. """
  12. import importlib
  13. import json
  14. import logging
  15. import os
  16. import tempfile
  17. from datetime import datetime
  18. from typing import List, Dict, Any, Optional
  19. import fire
  20. from dotenv import load_dotenv
  21. from agent.tool_dispatch_helpers import make_tool_result_message
  22. from trajectory_compressor import _effective_temperature_for_model
  23. # Load environment variables
  24. load_dotenv()
  25. TERMINAL_TOOL_DEFINITION = {
  26. "type": "function",
  27. "function": {
  28. "name": "terminal",
  29. "description": """Execute bash commands in a sandboxed environment.
  30. **Environment:**
  31. - Isolated execution environment (local, Docker, or Modal cloud)
  32. - Filesystem persists between tool calls within the same task
  33. - Internet access available
  34. **Command Execution:**
  35. - Provide the command to execute via the 'command' parameter
  36. - Optional 'timeout' parameter in seconds (default: 60)
  37. **Examples:**
  38. - Run command: `{"command": "ls -la"}`
  39. - With timeout: `{"command": "long_task.sh", "timeout": 300}`
  40. **Best Practices:**
  41. - Use non-interactive commands (avoid vim, nano, interactive python)
  42. - Pipe to cat if output might be large
  43. - Install tools with apt-get or pip as needed
  44. **Completion:**
  45. - When task is complete, output: echo "MINI_SWE_AGENT_FINAL_OUTPUT" followed by your result
  46. """,
  47. "parameters": {
  48. "type": "object",
  49. "properties": {
  50. "command": {"type": "string", "description": "The bash command to execute"},
  51. "timeout": {"type": "integer", "description": "Command timeout in seconds (default: 60)"},
  52. },
  53. "required": ["command"],
  54. },
  55. },
  56. }
  57. SYSTEM_PROMPT = """You are an AI agent that can execute bash commands to complete tasks.
  58. When you need to run commands, use the 'terminal' tool with your bash command.
  59. **Important:**
  60. - When you have completed the task successfully, run: echo "MINI_SWE_AGENT_FINAL_OUTPUT" followed by a summary
  61. - Be concise and efficient in your approach
  62. - Install any needed tools with apt-get or pip
  63. - Avoid interactive commands (no vim, nano, less, etc.)
  64. Complete the user's task step by step."""
  65. HERMES_SYSTEM_PREFIX = (
  66. "You are a function calling AI model. You are provided with function signatures within <tools> </tools> XML tags. "
  67. "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting "
  68. "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug "
  69. "into functions. After calling & executing the functions, you will be provided with function results within "
  70. "<tool_response> </tool_response> XML tags. Here are the available tools:\n"
  71. )
  72. HERMES_SYSTEM_SUFFIX = (
  73. "For each function call return a JSON object, with the following pydantic model json schema for each:\n"
  74. "{'title': 'FunctionCall', 'type': 'object', 'properties': {'name': {'title': 'Name', 'type': 'string'}, "
  75. "'arguments': {'title': 'Arguments', 'type': 'object'}}, 'required': ['name', 'arguments']}\n"
  76. "Each function call should be enclosed within <tool_call> </tool_call> XML tags.\n"
  77. "Example:\n<tool_call>\n{'name': <function-name>,'arguments': <args-dict>}\n</tool_call>"
  78. )
  79. _OPENROUTER_URL = "https://openrouter.ai/api/v1"
  80. def create_environment(env_type: str = "local", image: str = "python:3.11-slim", cwd: str | None = None, timeout: int = 60, **kwargs):
  81. """Create a Hermes execution environment (``local`` ignores ``image``/``kwargs``).
  82. ``cwd=None`` means the host temp dir locally and the sandbox's own ``/tmp`` inside a container.
  83. """
  84. if env_type == "local":
  85. from tools.environments.local import LocalEnvironment
  86. return LocalEnvironment(cwd=cwd or tempfile.gettempdir(), timeout=timeout)
  87. if env_type not in ("docker", "modal"):
  88. raise ValueError(f"Unknown environment type: {env_type}. Use 'local', 'docker', or 'modal'")
  89. cwd = cwd or "/tmp" # container-side path, not the host temp dir # no-tmp: ok — container-side path, not the host temp dir
  90. module = importlib.import_module(f"tools.environments.{env_type}")
  91. return getattr(module, f"{env_type.capitalize()}Environment")(image=image, cwd=cwd, timeout=timeout, **kwargs)
  92. def _parse_json_args(raw: Any) -> Any:
  93. """Decode tool-call arguments; invalid JSON becomes ``{}``."""
  94. if not isinstance(raw, str):
  95. return raw
  96. try:
  97. return json.loads(raw)
  98. except json.JSONDecodeError:
  99. return {}
  100. def _gpt_content(msg: dict[str, Any], content: str) -> str:
  101. """Prefix ``content`` with a ``<think>`` block when the message carries reasoning."""
  102. return (f"<think>{msg['reasoning']}</think>" if msg.get("reasoning") else "") + content
  103. class MiniSWERunner:
  104. """Tool-calling agent loop over a Hermes execution environment, emitting Hermes trajectories."""
  105. def __init__(self, model: str = "anthropic/claude-sonnet-4.6", base_url: str | None = None, api_key: str | None = None,
  106. env_type: str = "local", image: str = "python:3.11-slim", cwd: str | None = None,
  107. max_iterations: int = 15, command_timeout: int = 60, verbose: bool = False):
  108. self.model, self.max_iterations, self.command_timeout, self.verbose = model, max_iterations, command_timeout, verbose
  109. self.env_type, self.image, self.cwd = env_type, image, cwd
  110. self.logger = logging.getLogger(__name__)
  111. self.client = self._init_client(base_url, api_key)
  112. self.env = None # created per-task
  113. self.tools = [TERMINAL_TOOL_DEFINITION]
  114. print("🤖 Mini-SWE Runner initialized")
  115. print(f" Model: {self.model}")
  116. print(f" Environment: {self.env_type}")
  117. if self.env_type != "local":
  118. print(f" Image: {self.image}")
  119. print(f" Max iterations: {self.max_iterations}")
  120. def _init_client(self, base_url: Optional[str], api_key: Optional[str]):
  121. """Explicit api_key/base_url -> direct OpenAI client; otherwise the provider router."""
  122. if api_key or base_url:
  123. from openai import OpenAI
  124. return OpenAI(base_url=base_url or _OPENROUTER_URL, api_key=api_key or os.getenv(
  125. "OPENROUTER_API_KEY", os.getenv("ANTHROPIC_API_KEY", os.getenv("OPENAI_API_KEY", ""))))
  126. from agent.auxiliary_client import resolve_provider_client
  127. client, _ = resolve_provider_client("openrouter", model=self.model)
  128. if client is None:
  129. client, _ = resolve_provider_client("auto", model=self.model)
  130. if client is None:
  131. from openai import OpenAI
  132. client = OpenAI(base_url=_OPENROUTER_URL, api_key=os.getenv("OPENROUTER_API_KEY", ""))
  133. return client
  134. def _create_env(self):
  135. print(f"🔧 Creating {self.env_type} environment...")
  136. self.env = create_environment(env_type=self.env_type, image=self.image, cwd=self.cwd, timeout=self.command_timeout)
  137. print("✅ Environment ready")
  138. def _cleanup_env(self):
  139. if self.env is not None:
  140. stop = getattr(self.env, 'cleanup', None) or getattr(self.env, 'stop', None)
  141. if stop:
  142. stop()
  143. self.env = None
  144. def _execute_command(self, command: str, timeout: int | None = None) -> dict[str, Any]:
  145. """Run ``command`` in the environment; returns ``{output, exit_code, error}``."""
  146. if self.env is None:
  147. self._create_env()
  148. try:
  149. result = self.env.execute(command, timeout=timeout or self.command_timeout)
  150. return {"output": result.get("output", ""), "exit_code": result.get("returncode", 0), "error": None}
  151. except Exception as e:
  152. return {"output": "", "exit_code": -1, "error": str(e)}
  153. def _format_tools_for_system_message(self) -> str:
  154. return json.dumps([
  155. {"name": t["function"]["name"], "description": t["function"].get("description", ""),
  156. "parameters": t["function"].get("parameters", {}), "required": None}
  157. for t in self.tools
  158. ], ensure_ascii=False)
  159. def _tool_response_turn(self, messages: list[dict[str, Any]], i: int) -> tuple:
  160. """Fold the tool messages following assistant turn ``i`` into one ``tool`` value.
  161. Returns ``(value_or_None, index_of_last_consumed_message)``.
  162. """
  163. tool_calls = messages[i]["tool_calls"]
  164. tool_responses = []
  165. j = i + 1
  166. while j < len(messages) and messages[j]["role"] == "tool":
  167. tool_msg = messages[j]
  168. tool_content = tool_msg["content"]
  169. try:
  170. if tool_content.strip().startswith(("{", "[")):
  171. tool_content = json.loads(tool_content)
  172. except (json.JSONDecodeError, AttributeError):
  173. pass
  174. k = len(tool_responses)
  175. body = json.dumps({"tool_call_id": tool_msg.get("tool_call_id", ""),
  176. "name": tool_calls[k]["function"]["name"] if k < len(tool_calls) else "unknown",
  177. "content": tool_content}, ensure_ascii=False)
  178. tool_responses.append(f"<tool_response>\n{body}\n</tool_response>")
  179. j += 1
  180. return ("\n".join(tool_responses), j - 1) if tool_responses else (None, i)
  181. def _convert_to_hermes_format(self, messages: list[dict[str, Any]], user_query: str) -> list[dict[str, Any]]:
  182. """Convert the OpenAI-style message list to the Hermes trajectory format used by batch_runner.py."""
  183. system_msg = HERMES_SYSTEM_PREFIX + f"<tools>\n{self._format_tools_for_system_message()}\n</tools>\n" + HERMES_SYSTEM_SUFFIX
  184. trajectory = [{"from": "system", "value": system_msg}, {"from": "human", "value": user_query}]
  185. i = 1 # first user message already added
  186. while i < len(messages):
  187. msg = messages[i]
  188. if msg["role"] == "user":
  189. trajectory.append({"from": "human", "value": msg["content"]})
  190. elif msg["role"] == "assistant" and not msg.get("tool_calls"):
  191. trajectory.append({"from": "gpt", "value": _gpt_content(msg, msg.get("content") or "")})
  192. elif msg["role"] == "assistant":
  193. content = (msg["content"] + "\n") if msg.get("content") else ""
  194. for tool_call in msg["tool_calls"]:
  195. if isinstance(tool_call, dict) and tool_call:
  196. tool_call_json = {"name": tool_call["function"]["name"], "arguments": _parse_json_args(tool_call["function"]["arguments"])}
  197. content += f"<tool_call>\n{json.dumps(tool_call_json, ensure_ascii=False)}\n</tool_call>\n"
  198. trajectory.append({"from": "gpt", "value": _gpt_content(msg, content).rstrip()})
  199. tool_value, i = self._tool_response_turn(messages, i)
  200. if tool_value is not None:
  201. trajectory.append({"from": "tool", "value": tool_value})
  202. i += 1
  203. return trajectory
  204. def _call_model(self, messages: list[dict[str, Any]]):
  205. """One chat completion with the ephemeral system prompt; returns the message or None on API error."""
  206. api_kwargs = {"model": self.model, "messages": [{"role": "system", "content": SYSTEM_PROMPT}] + messages,
  207. "tools": self.tools, "timeout": 300.0}
  208. # requested_temperature=None: only fixed model contracts (Kimi omit / Arcee 0.5) apply here.
  209. fixed_temperature = _effective_temperature_for_model(self.model, None, str(getattr(self.client, "base_url", "") or ""))
  210. if fixed_temperature is not None:
  211. api_kwargs["temperature"] = fixed_temperature
  212. try:
  213. return self.client.chat.completions.create(**api_kwargs).choices[0].message
  214. except Exception as e:
  215. self.logger.error("API call failed: %s", e)
  216. def _run_tool_calls(self, assistant_message, messages: list[dict[str, Any]]) -> bool:
  217. """Record the assistant turn, execute each terminal call, append results; True if the completion signal fired."""
  218. print(f"🔧 Tool calls: {len(assistant_message.tool_calls)}")
  219. messages.append({"role": "assistant", "content": assistant_message.content, "tool_calls": [
  220. {"id": tc.id, "type": tc.type, "function": {"name": tc.function.name, "arguments": tc.function.arguments}}
  221. for tc in assistant_message.tool_calls
  222. ]})
  223. completed = False
  224. for tc in assistant_message.tool_calls:
  225. args = _parse_json_args(tc.function.arguments)
  226. command = args.get("command", "echo 'No command provided'")
  227. print(f" 📞 terminal: {command[:60]}...")
  228. result = self._execute_command(command, args.get("timeout", self.command_timeout))
  229. if "MINI_SWE_AGENT_FINAL_OUTPUT" in result["output"]:
  230. print(" ✅ Task completion signal detected!")
  231. completed = True
  232. messages.append(make_tool_result_message(tc.function.name, json.dumps({"content": result}, ensure_ascii=False), tc.id))
  233. print(f" ✅ exit_code={result['exit_code']}, output={len(result['output'])} chars")
  234. return completed
  235. def run_task(self, task: str) -> dict[str, Any]:
  236. """Run one task; returns ``{conversations, completed, api_calls, metadata}``."""
  237. print(f"\n{'='*60}")
  238. print(f"📝 Task: {task[:80]}{'...' if len(task) > 80 else ''}")
  239. print(f"{'='*60}")
  240. self._create_env()
  241. messages = [{"role": "user", "content": task}]
  242. api_call_count = 0
  243. completed = False
  244. try:
  245. while api_call_count < self.max_iterations:
  246. api_call_count += 1
  247. print(f"\n🔄 API call #{api_call_count}/{self.max_iterations}")
  248. assistant_message = self._call_model(messages)
  249. if assistant_message is None:
  250. break
  251. if assistant_message.content:
  252. print(f"🤖 Assistant: {assistant_message.content[:100]}...")
  253. if not assistant_message.tool_calls:
  254. messages.append({"role": "assistant", "content": assistant_message.content or ""})
  255. completed = True
  256. print("🎉 Agent finished (no more tool calls)")
  257. break
  258. if self._run_tool_calls(assistant_message, messages):
  259. completed = True
  260. break
  261. if api_call_count >= self.max_iterations:
  262. print(f"⚠️ Reached max iterations ({self.max_iterations})")
  263. finally:
  264. self._cleanup_env()
  265. return {"conversations": self._convert_to_hermes_format(messages, task), "completed": completed, "api_calls": api_call_count,
  266. "metadata": {"model": self.model, "env_type": self.env_type, "timestamp": datetime.now().isoformat()}}
  267. def run_batch(self, prompts: list[str], output_file: str) -> list[dict[str, Any]]:
  268. """Run every prompt, appending each result to ``output_file`` as it finishes."""
  269. results = []
  270. print(f"\n📦 Running batch of {len(prompts)} tasks")
  271. print(f"📁 Output: {output_file}")
  272. with open(output_file, 'w', encoding='utf-8') as f:
  273. for i, prompt in enumerate(prompts, 1):
  274. print(f"\n{'='*60}")
  275. print(f"📋 Task {i}/{len(prompts)}")
  276. print(f"{'='*60}")
  277. try:
  278. result = self.run_task(prompt)
  279. print(f"✅ Task {i} completed (api_calls={result['api_calls']})")
  280. except Exception as e:
  281. self.logger.error("Error on task %s: %s", i, e)
  282. result = {"conversations": [], "completed": False, "api_calls": 0, "error": str(e),
  283. "metadata": {"timestamp": datetime.now().isoformat()}}
  284. results.append(result)
  285. f.write(json.dumps(result, ensure_ascii=False) + "\n")
  286. f.flush()
  287. print(f"\n✅ Batch complete! {len(results)} trajectories saved to {output_file}")
  288. return results
  289. def _load_prompts(prompts_file: str) -> list[str]:
  290. """One prompt per non-blank line: JSON ``{"prompt"|"task": ...}`` or raw text."""
  291. prompts = []
  292. with open(prompts_file, 'r', encoding='utf-8') as f:
  293. for line in f:
  294. line = line.strip()
  295. if not line:
  296. continue
  297. try:
  298. entry = json.loads(line)
  299. prompts.append(entry.get("prompt", entry.get("task", "")))
  300. except json.JSONDecodeError:
  301. prompts.append(line)
  302. return prompts
  303. def main(
  304. task: str | None = None,
  305. prompts_file: str | None = None,
  306. output_file: str = "swe-runner-test1.jsonl",
  307. model: str = "claude-sonnet-4-20250514",
  308. base_url: str | None = None,
  309. api_key: str | None = None,
  310. env: str = "local",
  311. image: str = "python:3.11-slim",
  312. cwd: str | None = None,
  313. max_iterations: int = 15,
  314. timeout: int = 60,
  315. verbose: bool = False,
  316. ):
  317. """
  318. Run SWE tasks with Hermes trajectory format output.
  319. Args:
  320. task: Single task to run (use this OR prompts_file)
  321. prompts_file: JSONL file with prompts (each line: {"prompt": "..."})
  322. output_file: Output JSONL file for trajectories
  323. model: Model name (default: claude-sonnet-4-20250514)
  324. base_url: API base URL (optional)
  325. api_key: API key (optional, uses env vars)
  326. env: Environment type - "local", "docker", or "modal"
  327. image: Docker/Modal image (default: python:3.11-slim)
  328. cwd: Working directory (default: host temp dir locally, /tmp inside a container)
  329. max_iterations: Maximum tool-calling iterations (default: 15)
  330. timeout: Command timeout in seconds (default: 60)
  331. verbose: Enable verbose logging
  332. """
  333. print("🚀 Mini-SWE Runner with Hermes Trajectory Format")
  334. print("=" * 60)
  335. # Configure root logging at the entry point (not in library __init__).
  336. logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO,
  337. format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%H:%M:%S')
  338. runner = MiniSWERunner(model=model, base_url=base_url, api_key=api_key, env_type=env, image=image, cwd=cwd,
  339. max_iterations=max_iterations, command_timeout=timeout, verbose=verbose)
  340. if task:
  341. result = runner.run_task(task)
  342. with open(output_file, 'w', encoding='utf-8') as f:
  343. f.write(json.dumps(result, ensure_ascii=False) + "\n")
  344. print(f"\n📁 Trajectory saved to: {output_file}")
  345. print(f"✅ Completed: {result['completed']}")
  346. print(f"📞 API calls: {result['api_calls']}")
  347. print(f"💬 Turns: {len(result['conversations'])}")
  348. elif prompts_file:
  349. prompts = _load_prompts(prompts_file)
  350. if not prompts:
  351. print(f"❌ No prompts found in {prompts_file}")
  352. return
  353. runner.run_batch(prompts, output_file)
  354. else:
  355. print("❌ Please provide either --task or --prompts_file")
  356. print(" Example: python mini_swe_runner.py --task 'Create a hello world script'")
  357. if __name__ == "__main__":
  358. fire.Fire(main)