run_batch.py 8.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243
  1. #!/usr/bin/env python3
  2. """
  3. run_batch.py — Run a workflow many times, varying parameters per run.
  4. Two modes:
  5. 1. --count N --randomize-seed
  6. Submit N runs, each with a fresh random seed. Use for quick variations.
  7. 2. --sweep '{"seed": [1,2,3], "steps": [20,30]}'
  8. Cartesian product of values. With cloud subscription, runs in parallel
  9. up to your tier's concurrent-job limit.
  10. Both modes write each run's outputs into output-dir/run_NNN/.
  11. Examples:
  12. python3 run_batch.py --workflow flux_dev.json \
  13. --args '{"prompt": "a cat"}' \
  14. --count 8 --randomize-seed \
  15. --output-dir ./outputs/cat-batch
  16. python3 run_batch.py --workflow sdxl.json \
  17. --args '{"prompt": "abstract"}' \
  18. --sweep '{"seed": [1,2,3], "steps": [20, 40]}' \
  19. --output-dir ./outputs/sweep
  20. """
  21. from __future__ import annotations
  22. import argparse
  23. import itertools
  24. import json
  25. import sys
  26. from concurrent.futures import ThreadPoolExecutor, as_completed
  27. from pathlib import Path
  28. sys.path.insert(0, str(Path(__file__).resolve().parent))
  29. from _common import (
  30. DEFAULT_LOCAL_HOST, ENV_API_KEY, coerce_seed, emit_json, log,
  31. looks_like_video_workflow, resolve_api_key, unwrap_workflow,
  32. )
  33. from run_workflow import (
  34. ComfyRunner, download_outputs, inject_params,
  35. )
  36. from extract_schema import extract_schema
  37. def expand_sweep(sweep: dict, base_args: dict, count: int, randomize_seed: bool) -> list[dict]:
  38. """Generate a list of args dicts for each run."""
  39. if sweep:
  40. # Cartesian product
  41. keys = list(sweep.keys())
  42. values = [sweep[k] if isinstance(sweep[k], list) else [sweep[k]] for k in keys]
  43. runs = []
  44. for combo in itertools.product(*values):
  45. ar = dict(base_args)
  46. for k, v in zip(keys, combo):
  47. ar[k] = v
  48. runs.append(ar)
  49. return runs
  50. # Count mode
  51. runs = []
  52. for _ in range(count):
  53. ar = dict(base_args)
  54. if randomize_seed:
  55. ar["seed"] = coerce_seed(None)
  56. runs.append(ar)
  57. return runs
  58. def execute_one(
  59. runner: ComfyRunner, workflow: dict, schema: dict, args: dict,
  60. *, output_dir: Path, timeout: int, ws: bool,
  61. ) -> dict:
  62. wf, warnings = inject_params(workflow, schema, args)
  63. sub = runner.submit(wf)
  64. if "_http_error" in sub:
  65. return {"status": "error", "error": "submission HTTP error",
  66. "details": sub.get("body"), "args": args}
  67. pid = sub.get("prompt_id")
  68. if not pid:
  69. return {"status": "error", "error": "no prompt_id", "response": sub, "args": args}
  70. if sub.get("node_errors"):
  71. return {"status": "error", "error": "validation failed",
  72. "node_errors": sub["node_errors"], "args": args}
  73. if ws:
  74. result = runner.monitor_ws(pid, timeout=timeout)
  75. else:
  76. result = runner.poll_status(pid, timeout=timeout)
  77. if result["status"] != "success":
  78. return {
  79. "status": result["status"],
  80. "prompt_id": pid,
  81. "details": result.get("data"),
  82. "args": args,
  83. }
  84. outputs = result.get("outputs") or runner.get_outputs(pid)
  85. downloaded = download_outputs(runner, outputs, output_dir, preserve_subfolder=False)
  86. return {
  87. "status": "success",
  88. "prompt_id": pid,
  89. "args": args,
  90. "outputs": downloaded,
  91. "warnings": warnings,
  92. }
  93. def main(argv: list[str] | None = None) -> int:
  94. p = argparse.ArgumentParser(
  95. description="Submit a workflow many times with varying parameters.",
  96. )
  97. p.add_argument("--workflow", required=True)
  98. p.add_argument("--args", default="{}", help="Base parameters JSON")
  99. p.add_argument("--count", type=int, default=0,
  100. help="Number of runs (use with --randomize-seed)")
  101. p.add_argument("--sweep", default="",
  102. help='JSON dict of param→list of values. Cartesian product. '
  103. 'e.g. \'{"seed":[1,2,3],"cfg":[5,8]}\'')
  104. p.add_argument("--randomize-seed", action="store_true",
  105. help="In --count mode, vary seed per run")
  106. p.add_argument("--host", default=DEFAULT_LOCAL_HOST)
  107. p.add_argument("--api-key", help=f"or set ${ENV_API_KEY}")
  108. p.add_argument("--partner-key")
  109. p.add_argument("--parallel", type=int, default=1,
  110. help="Concurrent submissions (cloud: up to your tier limit). "
  111. "Default 1 (sequential)")
  112. p.add_argument("--output-dir", default="./outputs/batch")
  113. p.add_argument("--timeout", type=int, default=0)
  114. p.add_argument("--ws", action="store_true")
  115. p.add_argument("--continue-on-error", action="store_true",
  116. help="Don't stop the batch when a run fails")
  117. args = p.parse_args(argv)
  118. if args.count <= 0 and not args.sweep:
  119. emit_json({"error": "Specify --count N or --sweep '{...}'"})
  120. return 1
  121. base_args = json.loads(args.args) if args.args.strip() else {}
  122. sweep = json.loads(args.sweep) if args.sweep.strip() else {}
  123. # Validate sweep shape
  124. if sweep:
  125. if not isinstance(sweep, dict):
  126. emit_json({"error": "--sweep must be a JSON object {param: [values]}"})
  127. return 1
  128. empty = [k for k, v in sweep.items() if isinstance(v, list) and len(v) == 0]
  129. if empty:
  130. emit_json({"error": f"--sweep parameters have empty value lists: {empty}"})
  131. return 1
  132. # If user passed BOTH --sweep and --count/--randomize-seed, --sweep wins
  133. if args.count or args.randomize_seed:
  134. log("--sweep set; ignoring --count / --randomize-seed (sweep defines the runs)")
  135. wf_path = Path(args.workflow).expanduser()
  136. if not wf_path.exists():
  137. emit_json({"error": f"Workflow not found: {args.workflow}"})
  138. return 1
  139. try:
  140. with wf_path.open(encoding="utf-8-sig") as f:
  141. workflow = unwrap_workflow(json.load(f))
  142. except (ValueError, json.JSONDecodeError) as e:
  143. emit_json({"error": str(e)})
  144. return 1
  145. schema = extract_schema(workflow)
  146. runs = expand_sweep(sweep, base_args, args.count, args.randomize_seed)
  147. log(f"Planned {len(runs)} run(s)")
  148. api_key = resolve_api_key(args.api_key)
  149. runner = ComfyRunner(host=args.host, api_key=api_key, partner_key=args.partner_key)
  150. ok, info = runner.check_server()
  151. if not ok:
  152. emit_json({"error": "Cannot reach server", "details": info, "host": args.host})
  153. return 1
  154. timeout = args.timeout
  155. if timeout <= 0:
  156. timeout = 900 if looks_like_video_workflow(workflow) else 300
  157. base_dir = Path(args.output_dir).expanduser()
  158. base_dir.mkdir(parents=True, exist_ok=True)
  159. results: list[dict] = []
  160. failures = 0
  161. if args.parallel > 1:
  162. with ThreadPoolExecutor(max_workers=args.parallel) as ex:
  163. future_to_idx = {}
  164. for i, ar in enumerate(runs):
  165. run_dir = base_dir / f"run_{i:04d}"
  166. fut = ex.submit(
  167. execute_one, runner, workflow, schema, ar,
  168. output_dir=run_dir, timeout=timeout, ws=args.ws,
  169. )
  170. future_to_idx[fut] = i
  171. for fut in as_completed(future_to_idx):
  172. i = future_to_idx[fut]
  173. try:
  174. r = fut.result()
  175. except Exception as e:
  176. r = {"status": "error", "error": str(e), "args": runs[i]}
  177. r["index"] = i
  178. results.append(r)
  179. if r["status"] != "success":
  180. failures += 1
  181. log(f" run {i} → {r['status']}: {r.get('error','?')}")
  182. if not args.continue_on_error:
  183. log(" --continue-on-error not set; aborting batch")
  184. break
  185. else:
  186. log(f" run {i} → success: {len(r.get('outputs', []))} files")
  187. else:
  188. for i, ar in enumerate(runs):
  189. run_dir = base_dir / f"run_{i:04d}"
  190. r = execute_one(runner, workflow, schema, ar,
  191. output_dir=run_dir, timeout=timeout, ws=args.ws)
  192. r["index"] = i
  193. results.append(r)
  194. if r["status"] != "success":
  195. failures += 1
  196. log(f" run {i} → {r['status']}: {r.get('error','?')}")
  197. if not args.continue_on_error:
  198. log(" --continue-on-error not set; aborting batch")
  199. break
  200. else:
  201. log(f" run {i} → success: {len(r.get('outputs', []))} files")
  202. results.sort(key=lambda x: x.get("index", 0))
  203. emit_json({
  204. "status": "success" if failures == 0 else "partial",
  205. "total": len(runs),
  206. "completed": sum(1 for r in results if r["status"] == "success"),
  207. "failed": failures,
  208. "output_dir": str(base_dir),
  209. "results": results,
  210. })
  211. return 0 if failures == 0 else 1
  212. if __name__ == "__main__":
  213. sys.exit(main())