toolset_distributions.py 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106
  1. #!/usr/bin/env python3
  2. """Toolset distributions for batch data-generation runs.
  3. A distribution maps toolset names to the % chance each is enabled for a prompt
  4. (sampled independently, so several toolsets can be active at once). A key may
  5. be a "+"-grouped compound ("browser+search") that rolls once for all members.
  6. """
  7. from typing import Any, Dict, List, Optional
  8. import random
  9. from toolsets import validate_toolset
  10. def _dist(description: str, **toolsets: int) -> dict[str, object]:
  11. return {"description": description, "toolsets": toolsets}
  12. DISTRIBUTIONS = {
  13. "default": _dist("All available tools, all the time", web=100, vision=100, image_gen=100, terminal=100, file=100, browser=100),
  14. "image_gen": _dist("Heavy focus on image generation with vision and web support", image_gen=90, vision=90, web=55, terminal=45),
  15. "research": _dist("Web research with vision analysis and reasoning", web=90, browser=70, vision=50, terminal=10),
  16. "science": _dist("Scientific research with web, terminal, file, and browser capabilities",
  17. web=94, terminal=94, file=94, vision=65, browser=50, image_gen=15),
  18. "development": _dist("Terminal, file tools, and reasoning with occasional web lookup", terminal=80, file=80, web=30, vision=10),
  19. "safe": _dist("All tools except terminal for safety", web=80, browser=70, vision=60, image_gen=60),
  20. "balanced": _dist("Equal probability of all toolsets", web=50, vision=50, image_gen=50, terminal=50, file=50, browser=50),
  21. "minimal": _dist("Only web tools for basic research", web=100),
  22. "terminal_only": _dist("Terminal and file tools for code execution tasks", terminal=100, file=100),
  23. "terminal_web": _dist("Terminal and file tools with web search for documentation lookup", terminal=100, file=100, web=100),
  24. "creative": _dist("Image generation and vision analysis focus", image_gen=90, vision=90, web=30),
  25. "reasoning": _dist("Heavy research/reasoning distribution with minimal other tools", web=90, file=60, terminal=20),
  26. "browser_use": _dist("Full browser-based web interaction with search, vision, and page control", browser=100, web=80, vision=70),
  27. "browser_only": _dist("Only browser automation tools for pure web interaction tasks", browser=100),
  28. # browser-use-tasks.jsonl: one grouped roll keeps web_search (for finding URLs) coupled to browser
  29. # at the original 97% now that `browser` no longer bundles it (#64503).
  30. "browser_tasks": _dist(
  31. "Browser-focused distribution with web_search for finding URLs (Google blocks direct browser searches)",
  32. **{"browser+search": 97}, vision=12, terminal=15,
  33. ),
  34. # nous-terminal-tasks.jsonl
  35. "terminal_tasks": _dist("Terminal-focused distribution with high terminal/file availability, occasional other tools",
  36. terminal=97, file=97, web=97, browser=75, vision=50, image_gen=10),
  37. # mixed-browser-terminal-tasks.jsonl
  38. "mixed_tasks": _dist("Mixed distribution with high browser, terminal, and file availability for complex tasks",
  39. browser=92, terminal=92, file=92, web=35, vision=15, image_gen=15),
  40. }
  41. def get_distribution(name: str) -> Optional[dict[str, Any]]:
  42. """Distribution definition (description + toolsets), or None if unknown."""
  43. return DISTRIBUTIONS.get(name)
  44. def list_distributions() -> dict[str, dict]:
  45. return DISTRIBUTIONS.copy()
  46. def validate_distribution(distribution_name: str) -> bool:
  47. return distribution_name in DISTRIBUTIONS
  48. def _entry_members(entry: str) -> list[str]:
  49. """Toolsets named by a distribution entry: a bare name or a "+"-grouped compound."""
  50. return [name.strip() for name in entry.split("+")]
  51. def sample_toolsets_from_distribution(distribution_name: str) -> list[str]:
  52. """Sample toolset names, each entry included independently with its % probability.
  53. An entry may be a single toolset or a "+"-grouped compound like
  54. "browser+search": one roll selects (or skips) every member together, so
  55. co-occurrence guarantees survive that independent rolls would break
  56. (two independent 97% rolls co-occur only ~94% of the time).
  57. Falls back to the highest-probability entry when nothing was rolled.
  58. Raises ValueError for an unknown distribution.
  59. """
  60. dist = get_distribution(distribution_name)
  61. if not dist:
  62. raise ValueError(f"Unknown distribution: {distribution_name}")
  63. selected_toolsets = []
  64. for entry, probability in dist["toolsets"].items():
  65. members = _entry_members(entry)
  66. invalid = [name for name in members if not validate_toolset(name)]
  67. if invalid:
  68. print(f"⚠️ Warning: Toolset '{'+'.join(invalid)}' in distribution '{distribution_name}' is not valid")
  69. elif random.random() * 100 < probability:
  70. selected_toolsets.extend(members)
  71. if not selected_toolsets and dist["toolsets"]:
  72. highest_prob_entry = max(dist["toolsets"].items(), key=lambda x: x[1])[0]
  73. members = _entry_members(highest_prob_entry)
  74. if all(validate_toolset(name) for name in members):
  75. selected_toolsets.extend(members)
  76. return selected_toolsets
  77. def print_distribution_info(distribution_name: str) -> None:
  78. """Print a distribution's description and toolset probabilities (highest first)."""
  79. dist = get_distribution(distribution_name)
  80. if not dist:
  81. print(f"❌ Unknown distribution: {distribution_name}")
  82. return
  83. print(f"\n📊 Distribution: {distribution_name}")
  84. print(f" Description: {dist['description']}")
  85. print(" Toolsets:")
  86. for toolset, prob in sorted(dist["toolsets"].items(), key=lambda x: x[1], reverse=True):
  87. print(f" • {toolset:15} : {prob:3}% chance")