trajectory_compressor.py 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868
  1. #!/usr/bin/env python3
  2. """Trajectory Compressor — post-process agent trajectories into a token budget.
  3. Strategy: protect the head (system, human, first gpt, first tool) and the last N
  4. turns; from the middle, summarize only as many turns as needed (never splitting a
  5. <tool_call>/<tool_response> pair) and replace them with one human summary turn.
  6. Usage:
  7. python trajectory_compressor.py --input=data/my_run # directory
  8. python trajectory_compressor.py --input=data/trajectories.jsonl --sample_percent=15
  9. python trajectory_compressor.py --input=data/trajectories.jsonl --output=out.jsonl --target_max_tokens=16000
  10. """
  11. import json
  12. import os
  13. import random
  14. import shutil
  15. import tempfile
  16. import time
  17. import hermes_yaml as yaml
  18. import logging
  19. import asyncio
  20. from pathlib import Path
  21. from typing import Any, Callable, Dict, List, Optional, Tuple
  22. from dataclasses import asdict, dataclass, field
  23. from datetime import datetime
  24. from utils import base_url_host_matches, base_url_hostname
  25. import fire
  26. from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn, TimeElapsedColumn, TimeRemainingColumn
  27. from rich.console import Console
  28. from hermes_constants import OPENROUTER_BASE_URL, get_hermes_home
  29. from agent.compression_marker import elide_middle
  30. from agent.retry_utils import jittered_backoff
  31. from hermes_cli.env_loader import load_hermes_dotenv
  32. # Load .env from HERMES_HOME first, then project root as a dev fallback.
  33. load_hermes_dotenv(hermes_home=get_hermes_home(), project_env=Path(__file__).parent / ".env")
  34. def _response_finish_reason(response: Any) -> str:
  35. """Lowercased ``choices[0].finish_reason`` of a dict/object response, ``""`` if absent.
  36. Local copy of ``agent.context_compressor._response_finish_reason``: this
  37. standalone CLI deliberately avoids importing the heavy context compressor.
  38. """
  39. try:
  40. choices = (response.get("choices") if isinstance(response, dict) else getattr(response, "choices", None)) or []
  41. first = choices[0] if choices else None
  42. reason = first.get("finish_reason") if isinstance(first, dict) else getattr(first, "finish_reason", None)
  43. return str(reason).strip().lower() if reason else ""
  44. except Exception:
  45. return ""
  46. def _effective_temperature_for_model(model: str, requested_temperature: Optional[float], base_url: Optional[str] = None) -> Optional[float]:
  47. """Apply fixed model temperature contracts to direct client calls.
  48. Returns ``None`` when the model manages temperature server-side (Kimi);
  49. callers must omit the ``temperature`` kwarg entirely in that case.
  50. Shared with ``mini_swe_runner`` (which passes ``requested_temperature=None``).
  51. """
  52. try:
  53. from agent.auxiliary_client import _fixed_temperature_for_model, OMIT_TEMPERATURE
  54. except Exception:
  55. return requested_temperature
  56. fixed_temperature = _fixed_temperature_for_model(model, base_url)
  57. if fixed_temperature is OMIT_TEMPERATURE:
  58. return None # caller must omit temperature
  59. return requested_temperature if fixed_temperature is None else fixed_temperature
  60. def _load_jsonl(path: Path, on_error: Optional[Callable[[int, json.JSONDecodeError], None]] = None, start: int = 0) -> list[tuple[int, Any]]:
  61. """Return ``(line_num, entry)`` for each non-blank line; bad lines go to ``on_error``."""
  62. entries = []
  63. with open(path, 'r', encoding='utf-8') as f:
  64. for line_num, line in enumerate(f, start):
  65. if not line.strip():
  66. continue
  67. try:
  68. entries.append((line_num, json.loads(line)))
  69. except json.JSONDecodeError as e:
  70. if on_error is not None:
  71. on_error(line_num, e)
  72. return entries
  73. def _write_jsonl(path: Path, entries) -> None:
  74. with open(path, 'w', encoding='utf-8') as f:
  75. f.writelines(json.dumps(entry, ensure_ascii=False) + '\n' for entry in entries)
  76. # YAML section -> keys; "yaml_key:attr" when the config attribute name differs.
  77. _YAML_SECTIONS: dict[str, tuple[str, ...]] = {
  78. "tokenizer": ("name:tokenizer_name", "trust_remote_code"),
  79. "compression": ("target_max_tokens", "summary_target_tokens"),
  80. "protected_turns": ("first_system:protect_first_system", "first_human:protect_first_human",
  81. "first_gpt:protect_first_gpt", "first_tool:protect_first_tool", "last_n_turns:protect_last_n_turns"),
  82. "summarization": ("model:summarization_model", "base_url", "api_key_env", "temperature", "max_retries", "retry_delay"),
  83. "output": ("add_summary_notice", "summary_notice_text", "output_suffix"),
  84. "processing": ("num_workers", "max_concurrent_requests", "skip_under_target", "save_over_limit"),
  85. "metrics": ("enabled:metrics_enabled", "per_trajectory:metrics_per_trajectory", "output_file:metrics_output_file"),
  86. }
  87. @dataclass
  88. class CompressionConfig:
  89. """Configuration for trajectory compression (tokenizer / targets / protected turns / summarizer / output / processing / metrics)."""
  90. tokenizer_name: str = "moonshotai/Kimi-K2-Thinking"
  91. trust_remote_code: bool = True
  92. target_max_tokens: int = 15250
  93. summary_target_tokens: int = 750
  94. protect_first_system: bool = True
  95. protect_first_human: bool = True
  96. protect_first_gpt: bool = True
  97. protect_first_tool: bool = True
  98. protect_last_n_turns: int = 4
  99. summarization_model: str = "google/gemini-3-flash-preview"
  100. base_url: str = OPENROUTER_BASE_URL
  101. api_key_env: str = "OPENROUTER_API_KEY"
  102. temperature: float = 0.3
  103. max_retries: int = 3
  104. retry_delay: int = 2
  105. add_summary_notice: bool = True
  106. summary_notice_text: str = "\n\nSome of your previous tool responses may be summarized to preserve context."
  107. output_suffix: str = "_compressed"
  108. num_workers: int = 4
  109. max_concurrent_requests: int = 50 # Max concurrent API calls for summarization
  110. skip_under_target: bool = True
  111. save_over_limit: bool = True
  112. per_trajectory_timeout: int = 300 # seconds (default: 5 min)
  113. metrics_enabled: bool = True
  114. metrics_per_trajectory: bool = True
  115. metrics_output_file: str = "compression_metrics.json"
  116. @classmethod
  117. def from_yaml(cls, yaml_path: str) -> "CompressionConfig":
  118. """Load configuration from YAML file (missing keys keep the defaults)."""
  119. with open(yaml_path, 'r', encoding="utf-8") as f:
  120. data = yaml.safe_load(f) or {}
  121. config = cls()
  122. for section, keys in _YAML_SECTIONS.items():
  123. for key in keys if section in data else ():
  124. yaml_key, _, attr = key.partition(":")
  125. attr = attr or yaml_key
  126. value = data[section].get(yaml_key, getattr(config, attr))
  127. if attr == "base_url":
  128. value = value or config.base_url # ``base_url: null`` keeps the default
  129. setattr(config, attr, value)
  130. return config
  131. @dataclass
  132. class TrajectoryMetrics:
  133. """Metrics for a single trajectory compression."""
  134. original_tokens: int = 0
  135. compressed_tokens: int = 0
  136. tokens_saved: int = 0
  137. compression_ratio: float = 1.0
  138. original_turns: int = 0
  139. compressed_turns: int = 0
  140. turns_removed: int = 0
  141. turns_compressed_start_idx: int = -1
  142. turns_compressed_end_idx: int = -1
  143. turns_in_compressed_region: int = 0
  144. was_compressed: bool = False
  145. still_over_limit: bool = False
  146. skipped_under_target: bool = False
  147. summarization_api_calls: int = 0
  148. summarization_errors: int = 0
  149. def to_dict(self) -> dict[str, Any]:
  150. d = asdict(self)
  151. d["compression_ratio"] = round(self.compression_ratio, 4)
  152. region = {"start_idx": d.pop("turns_compressed_start_idx"), "end_idx": d.pop("turns_compressed_end_idx"),
  153. "turns_count": d.pop("turns_in_compressed_region")}
  154. items = list(d.items())
  155. items.insert(7, ("compression_region", region)) # after turns_removed: historical key order
  156. return dict(items)
  157. def _mean(values, default):
  158. return sum(values) / len(values) if values else default
  159. @dataclass
  160. class AggregateMetrics:
  161. """Aggregate metrics across all trajectories."""
  162. total_trajectories: int = 0
  163. trajectories_compressed: int = 0
  164. trajectories_skipped_under_target: int = 0
  165. trajectories_still_over_limit: int = 0
  166. trajectories_failed: int = 0
  167. total_tokens_before: int = 0
  168. total_tokens_after: int = 0
  169. total_tokens_saved: int = 0
  170. total_turns_before: int = 0
  171. total_turns_after: int = 0
  172. total_turns_removed: int = 0
  173. total_summarization_calls: int = 0
  174. total_summarization_errors: int = 0
  175. compression_ratios: list[float] = field(default_factory=list)
  176. tokens_saved_list: list[int] = field(default_factory=list)
  177. turns_removed_list: list[int] = field(default_factory=list)
  178. processing_start_time: str = ""
  179. processing_end_time: str = ""
  180. processing_duration_seconds: float = 0.0
  181. def add_trajectory_metrics(self, metrics: TrajectoryMetrics):
  182. """Add a trajectory's metrics to the aggregate."""
  183. self.total_trajectories += 1
  184. self.total_tokens_before += metrics.original_tokens
  185. self.total_tokens_after += metrics.compressed_tokens
  186. self.total_tokens_saved += metrics.tokens_saved
  187. self.total_turns_before += metrics.original_turns
  188. self.total_turns_after += metrics.compressed_turns
  189. self.total_turns_removed += metrics.turns_removed
  190. self.total_summarization_calls += metrics.summarization_api_calls
  191. self.total_summarization_errors += metrics.summarization_errors
  192. if metrics.was_compressed:
  193. self.trajectories_compressed += 1
  194. self.compression_ratios.append(metrics.compression_ratio)
  195. self.tokens_saved_list.append(metrics.tokens_saved)
  196. self.turns_removed_list.append(metrics.turns_removed)
  197. self.trajectories_skipped_under_target += bool(metrics.skipped_under_target)
  198. self.trajectories_still_over_limit += bool(metrics.still_over_limit)
  199. def to_dict(self) -> dict[str, Any]:
  200. return {
  201. "summary": {"total_trajectories": self.total_trajectories, "trajectories_compressed": self.trajectories_compressed,
  202. "trajectories_skipped_under_target": self.trajectories_skipped_under_target,
  203. "trajectories_still_over_limit": self.trajectories_still_over_limit, "trajectories_failed": self.trajectories_failed,
  204. "compression_rate": round(self.trajectories_compressed / max(self.total_trajectories, 1), 4)},
  205. "tokens": {"total_before": self.total_tokens_before, "total_after": self.total_tokens_after, "total_saved": self.total_tokens_saved,
  206. "overall_compression_ratio": round(self.total_tokens_after / max(self.total_tokens_before, 1), 4)},
  207. "turns": {"total_before": self.total_turns_before, "total_after": self.total_turns_after, "total_removed": self.total_turns_removed},
  208. "averages": {"avg_compression_ratio": round(_mean(self.compression_ratios, 1.0), 4),
  209. "avg_tokens_saved_per_compressed": round(_mean(self.tokens_saved_list, 0), 1),
  210. "avg_turns_removed_per_compressed": round(_mean(self.turns_removed_list, 0), 2)},
  211. "summarization": {"total_api_calls": self.total_summarization_calls, "total_errors": self.total_summarization_errors,
  212. "success_rate": round(1 - (self.total_summarization_errors / max(self.total_summarization_calls, 1)), 4)},
  213. "processing": {"start_time": self.processing_start_time, "end_time": self.processing_end_time,
  214. "duration_seconds": round(self.processing_duration_seconds, 2)},
  215. }
  216. # Ordered (hostname, provider) table for _detect_provider (codex is matched separately).
  217. _PROVIDER_HOSTS: tuple[tuple[str, str], ...] = (
  218. ("openrouter.ai", "openrouter"), ("nousresearch.com", "nous"), ("z.ai", "zai"), ("moonshot.ai", "kimi-coding"),
  219. ("moonshot.cn", "kimi-coding"), ("api.kimi.com", "kimi-coding"), ("arcee.ai", "arcee"), ("minimaxi.com", "minimax-cn"),
  220. ("minimax.io", "minimax"),
  221. )
  222. _SUMMARY_FALLBACK = "[CONTEXT SUMMARY]: [Summary generation failed - previous turns contained tool calls and responses that have been compressed to save context space.]"
  223. _STATUS_FMT = "[dim]✅ {compressed} compressed | ⏭️ {skipped} skipped | ⏱️ {timeouts} timeout | 🔄 {api_calls} API calls | ⚡ {in_flight} in-flight[/dim]"
  224. @dataclass
  225. class _RunProgress:
  226. """Shared counters + rich progress handles for one directory run."""
  227. progress: Any
  228. main_task: Any
  229. status_task: Any
  230. lock: asyncio.Lock
  231. semaphore: asyncio.Semaphore
  232. compressed: int = 0
  233. skipped: int = 0
  234. api_calls: int = 0
  235. in_flight: int = 0
  236. timeouts: int = 0
  237. def finish(self, update_status: bool = True) -> None:
  238. """Retire one in-flight entry and advance the bar (caller holds ``lock``)."""
  239. self.in_flight -= 1
  240. self.progress.advance(self.main_task)
  241. if update_status:
  242. self.progress.update(self.status_task, description=_STATUS_FMT.format(
  243. compressed=self.compressed, skipped=self.skipped, timeouts=self.timeouts,
  244. api_calls=self.api_calls, in_flight=self.in_flight))
  245. class TrajectoryCompressor:
  246. """Compresses agent trajectories to fit within a target token budget.
  247. Keeps protected head/tail turns, summarizes only as much of the middle as
  248. needed into one human summary turn, and keeps the remaining middle intact.
  249. """
  250. def __init__(self, config: CompressionConfig):
  251. self.config = config
  252. self.aggregate_metrics = AggregateMetrics()
  253. self._init_tokenizer()
  254. self._init_summarizer()
  255. self.logger = logging.getLogger(__name__)
  256. def _init_tokenizer(self):
  257. """Initialize HuggingFace tokenizer for token counting."""
  258. try:
  259. from transformers import AutoTokenizer
  260. self.tokenizer = AutoTokenizer.from_pretrained(self.config.tokenizer_name, trust_remote_code=self.config.trust_remote_code)
  261. print(f"✅ Loaded tokenizer: {self.config.tokenizer_name}")
  262. except Exception as e:
  263. raise RuntimeError(f"Failed to load tokenizer '{self.config.tokenizer_name}': {e}")
  264. def _init_summarizer(self):
  265. """Route summarization through call_llm for known providers, else a raw client."""
  266. provider = self._detect_provider()
  267. self._use_call_llm = bool(provider)
  268. if provider:
  269. self._llm_provider = provider
  270. from agent.auxiliary_client import resolve_provider_client
  271. client, _ = resolve_provider_client(provider, model=self.config.summarization_model)
  272. if client is None:
  273. raise RuntimeError(f"Provider '{provider}' is not configured. Check your API key or run: hermes setup")
  274. self.client = self.async_client = None # Not used directly
  275. else:
  276. # Custom endpoint — use config's raw base_url + api_key_env
  277. api_key = os.getenv(self.config.api_key_env)
  278. if not api_key:
  279. raise RuntimeError(f"Missing API key. Set {self.config.api_key_env} environment variable.")
  280. from openai import OpenAI
  281. from agent.auxiliary_client import _to_openai_base_url
  282. self.client = OpenAI(api_key=api_key, base_url=_to_openai_base_url(self.config.base_url))
  283. # AsyncOpenAI is created lazily in _get_async_client() so it binds to the current event
  284. # loop — each process_directory() runs its own asyncio.run(); a shared client would hit
  285. # "Event loop is closed".
  286. self.async_client = None
  287. self._async_client_api_key = api_key
  288. print(f"✅ Initialized summarizer client: {self.config.summarization_model}")
  289. print(f" Max concurrent requests: {self.config.max_concurrent_requests}")
  290. def _get_async_client(self):
  291. """Return a fresh AsyncOpenAI client bound to the running event loop."""
  292. from openai import AsyncOpenAI
  293. from agent.auxiliary_client import _to_openai_base_url
  294. self.async_client = AsyncOpenAI(api_key=self._async_client_api_key, base_url=_to_openai_base_url(self.config.base_url))
  295. return self.async_client
  296. def _detect_provider(self) -> str:
  297. """Provider name for the configured base_url, or ``""`` when unknown."""
  298. url = self.config.base_url or ""
  299. if base_url_hostname(url) == "chatgpt.com" and "/backend-api/codex" in url.lower():
  300. return "codex"
  301. return next((provider for host, provider in _PROVIDER_HOSTS if base_url_host_matches(url, host)), "")
  302. def count_tokens(self, text: str) -> int:
  303. """Token count via the configured tokenizer; falls back to len//4."""
  304. if not text:
  305. return 0
  306. try:
  307. return len(self.tokenizer.encode(text))
  308. except Exception:
  309. return len(text) // 4
  310. def count_trajectory_tokens(self, trajectory: list[dict[str, str]]) -> int:
  311. return sum(self.count_turn_tokens(trajectory))
  312. def count_turn_tokens(self, trajectory: list[dict[str, str]]) -> list[int]:
  313. return [self.count_tokens(turn.get("value", "")) for turn in trajectory]
  314. def _find_protected_indices(self, trajectory: list[dict[str, str]]) -> tuple[set, int, int]:
  315. """Return ``(protected_set, compressible_start, compressible_end)``."""
  316. n = len(trajectory)
  317. first_seen: dict[str, int] = {}
  318. for i, turn in enumerate(trajectory):
  319. first_seen.setdefault(turn.get("from", ""), i)
  320. protected = {first_seen[role] for role in ("system", "human", "gpt", "tool")
  321. if getattr(self.config, f"protect_first_{role}") and role in first_seen}
  322. protected.update(range(max(0, n - self.config.protect_last_n_turns), n))
  323. # Compressible region: after the last protected head turn, before the first tail turn.
  324. head_protected = [i for i in protected if i < n // 2]
  325. tail_protected = [i for i in protected if i >= n // 2]
  326. return protected, max(head_protected) + 1 if head_protected else 0, min(tail_protected) if tail_protected else n
  327. @staticmethod
  328. def _snap_boundary(trajectory: list[dict[str, str]], idx: int, min_idx: int, max_idx: int) -> int:
  329. """Move a boundary onto the nearest turn boundary within ``[min_idx, max_idx]`` that does not
  330. split a gpt <tool_call>/tool <tool_response> pair.
  331. A ``tool`` turn always directly follows the ``gpt`` turn it answers, so a boundary landing *on*
  332. a tool turn cuts the pair; only the end of the trajectory or a non-``tool`` turn is clean.
  333. Forward is preferred (folds an orphaned ``tool`` turn into the region that holds its ``gpt``
  334. turn); backward only when nothing clean lies ahead.
  335. """
  336. def clean(i: int) -> bool:
  337. return i >= len(trajectory) or trajectory[i].get("from") != "tool"
  338. forward = idx
  339. while forward < max_idx and not clean(forward):
  340. forward += 1
  341. if clean(forward):
  342. return forward
  343. backward = idx
  344. while backward > min_idx and not clean(backward):
  345. backward -= 1
  346. return backward
  347. def _extract_turn_content_for_summary(self, trajectory: list[dict[str, str]], start: int, end: int) -> str:
  348. """Format turns ``[start, end)`` for the summarization prompt (long values truncated)."""
  349. parts = []
  350. for i in range(start, end):
  351. turn = trajectory[i]
  352. value = turn.get("value", "")
  353. if len(value) > 3000:
  354. value = elide_middle(value, 1500, 500)
  355. parts.append(f"[Turn {i} - {turn.get('from', 'unknown').upper()}]:\n{value}")
  356. return "\n\n".join(parts)
  357. def _summary_prompt(self, content: str) -> str:
  358. return f"""Summarize the following agent conversation turns concisely. This summary will replace these turns in the conversation history.
  359. Write the summary from a neutral perspective describing what the assistant did and learned. Include:
  360. 1. What actions the assistant took (tool calls, searches, file operations)
  361. 2. Key information or results obtained
  362. 3. Any important decisions or findings
  363. 4. Relevant data, file names, values, or outputs
  364. Keep the summary factual and informative. Target approximately {self.config.summary_target_tokens} tokens.
  365. ---
  366. TURNS TO SUMMARIZE:
  367. {content}
  368. ---
  369. Write only the summary, starting with "[CONTEXT SUMMARY]:" prefix."""
  370. def _summary_request(self, prompt: str) -> tuple[Optional[float], dict[str, Any]]:
  371. """Return ``(temperature, create-kwargs)``; temperature None means omit it."""
  372. cfg = self.config
  373. temperature = _effective_temperature_for_model(cfg.summarization_model, cfg.temperature, cfg.base_url)
  374. kwargs = {"model": cfg.summarization_model, "messages": [{"role": "user", "content": prompt}],
  375. "max_tokens": cfg.summary_target_tokens * 2}
  376. if not getattr(self, '_use_call_llm', False) and temperature is not None:
  377. kwargs["temperature"] = temperature
  378. return temperature, kwargs
  379. def _finish_summary(self, response: Any) -> str:
  380. """Extract the summary text with the ``[CONTEXT SUMMARY]:`` prefix exactly once; a ``length`` stop is a failure."""
  381. if _response_finish_reason(response) == "length":
  382. # Storing a truncated summary silently corrupts the trajectory's memory, so raise and
  383. # let the retry/backoff loop handle it.
  384. raise RuntimeError("trajectory summarization hit the output token cap (finish_reason=length); summary is incomplete")
  385. content = response.choices[0].message.content
  386. text = (content if isinstance(content, str) else str(content) if content else "").strip()
  387. if text.startswith("[CONTEXT SUMMARY]:"):
  388. return text
  389. return "[CONTEXT SUMMARY]:" if not text else f"[CONTEXT SUMMARY]: {text}"
  390. def _summary_attempt_failed(self, metrics: TrajectoryMetrics, attempt: int, exc: Exception) -> Optional[float]:
  391. """Record a failed attempt; return the backoff delay, or None on the last attempt."""
  392. metrics.summarization_errors += 1
  393. self.logger.warning("Summarization attempt %d failed: %s", attempt + 1, exc)
  394. if attempt < self.config.max_retries - 1:
  395. return jittered_backoff(attempt + 1, base_delay=self.config.retry_delay, max_delay=30.0)
  396. return None
  397. def _generate_summary(self, content: str, metrics: TrajectoryMetrics) -> str:
  398. """Summarize ``content`` with retries; returns a fallback summary after the last failure."""
  399. prompt = self._summary_prompt(content)
  400. for attempt in range(self.config.max_retries):
  401. try:
  402. metrics.summarization_api_calls += 1
  403. temperature, kwargs = self._summary_request(prompt)
  404. if getattr(self, '_use_call_llm', False):
  405. from agent.auxiliary_client import call_llm
  406. response = call_llm(provider=self._llm_provider, temperature=temperature, **kwargs)
  407. else:
  408. response = self.client.chat.completions.create(**kwargs)
  409. return self._finish_summary(response)
  410. except Exception as e:
  411. delay = self._summary_attempt_failed(metrics, attempt, e)
  412. if delay is None:
  413. return _SUMMARY_FALLBACK
  414. time.sleep(delay)
  415. async def _generate_summary_async(self, content: str, metrics: TrajectoryMetrics) -> str:
  416. """Async twin of ``_generate_summary``."""
  417. prompt = self._summary_prompt(content)
  418. for attempt in range(self.config.max_retries):
  419. try:
  420. metrics.summarization_api_calls += 1
  421. temperature, kwargs = self._summary_request(prompt)
  422. if getattr(self, '_use_call_llm', False):
  423. from agent.auxiliary_client import async_call_llm
  424. response = await async_call_llm(provider=self._llm_provider, temperature=temperature, **kwargs)
  425. else:
  426. response = await self._get_async_client().chat.completions.create(**kwargs)
  427. return self._finish_summary(response)
  428. except Exception as e:
  429. delay = self._summary_attempt_failed(metrics, attempt, e)
  430. if delay is None:
  431. return _SUMMARY_FALLBACK
  432. await asyncio.sleep(delay)
  433. def _plan_compression(self, trajectory: list[dict[str, str]], metrics: TrajectoryMetrics) -> Optional[tuple[int, int]]:
  434. """Choose the ``[start, until)`` region to summarize, or None if nothing can be.
  435. Fills the pre-compression metrics either way. Accumulates turns from the
  436. start of the compressible middle until the savings cover the overage plus
  437. the summary itself, then snaps both boundaries off ``tool`` turns.
  438. """
  439. cfg = self.config
  440. turn_tokens = self.count_turn_tokens(trajectory)
  441. total_tokens = sum(turn_tokens)
  442. metrics.original_turns = metrics.compressed_turns = len(trajectory)
  443. metrics.original_tokens = metrics.compressed_tokens = total_tokens
  444. if total_tokens <= cfg.target_max_tokens:
  445. metrics.skipped_under_target = True
  446. return None
  447. metrics.still_over_limit = True
  448. _, start, end = self._find_protected_indices(trajectory)
  449. # Never *start* on an orphaned <tool_response> whose <tool_call> is in the protected head.
  450. start = self._snap_boundary(trajectory, start, start, end)
  451. if start >= end:
  452. return None
  453. # Replacing N turns with one summary saves sum(N) - summary_target_tokens.
  454. target_tokens_to_compress = total_tokens - cfg.target_max_tokens + cfg.summary_target_tokens
  455. accumulated = 0
  456. until = start
  457. for i in range(start, end):
  458. accumulated += turn_tokens[i]
  459. until = i + 1
  460. if accumulated >= target_tokens_to_compress:
  461. break
  462. if accumulated < target_tokens_to_compress and until < end:
  463. until = end
  464. # The remainder is kept verbatim, so a tail boundary on a tool turn would orphan a marker.
  465. until = self._snap_boundary(trajectory, until, start, end)
  466. # A region no larger than the summary replacing it cannot shrink the trajectory.
  467. if until <= start or sum(turn_tokens[start:until]) <= cfg.summary_target_tokens:
  468. return None
  469. metrics.turns_compressed_start_idx, metrics.turns_compressed_end_idx = start, until
  470. metrics.turns_in_compressed_region = until - start
  471. return start, until
  472. def _assemble_compressed(self, trajectory: list[dict[str, str]], start: int, until: int, summary: str,
  473. metrics: TrajectoryMetrics) -> list[dict[str, str]]:
  474. """Head (with summary notice on system) + summary human turn + verbatim tail; finalize metrics."""
  475. compressed = []
  476. for turn in trajectory[:start]:
  477. turn = turn.copy()
  478. if turn.get("from") == "system" and self.config.add_summary_notice:
  479. turn["value"] = turn["value"] + self.config.summary_notice_text
  480. compressed.append(turn)
  481. compressed.append({"from": "human", "value": summary})
  482. compressed.extend(turn.copy() for turn in trajectory[until:])
  483. metrics.compressed_turns = len(compressed)
  484. metrics.compressed_tokens = self.count_trajectory_tokens(compressed)
  485. metrics.turns_removed = metrics.original_turns - metrics.compressed_turns
  486. metrics.tokens_saved = metrics.original_tokens - metrics.compressed_tokens
  487. metrics.compression_ratio = metrics.compressed_tokens / max(metrics.original_tokens, 1)
  488. metrics.was_compressed = True
  489. metrics.still_over_limit = metrics.compressed_tokens > self.config.target_max_tokens
  490. return compressed
  491. def compress_trajectory(self, trajectory: list[dict[str, str]]) -> tuple[list[dict[str, str]], TrajectoryMetrics]:
  492. """Compress one trajectory into the target budget; returns ``(trajectory, metrics)``."""
  493. metrics = TrajectoryMetrics()
  494. region = self._plan_compression(trajectory, metrics)
  495. if region is None:
  496. return trajectory, metrics
  497. summary = self._generate_summary(self._extract_turn_content_for_summary(trajectory, *region), metrics)
  498. return self._assemble_compressed(trajectory, *region, summary, metrics), metrics
  499. async def compress_trajectory_async(self, trajectory: list[dict[str, str]]) -> tuple[list[dict[str, str]], TrajectoryMetrics]:
  500. """Async twin of ``compress_trajectory``."""
  501. metrics = TrajectoryMetrics()
  502. region = self._plan_compression(trajectory, metrics)
  503. if region is None:
  504. return trajectory, metrics
  505. summary = await self._generate_summary_async(self._extract_turn_content_for_summary(trajectory, *region), metrics)
  506. return self._assemble_compressed(trajectory, *region, summary, metrics), metrics
  507. async def process_entry_async(self, entry: dict[str, Any]) -> tuple[dict[str, Any], TrajectoryMetrics]:
  508. """Compress one JSONL entry's ``conversations``; attach metrics when compressed."""
  509. if not isinstance(entry, dict) or "conversations" not in entry:
  510. return entry, TrajectoryMetrics()
  511. compressed_trajectory, metrics = await self.compress_trajectory_async(entry["conversations"])
  512. result = dict(entry, conversations=compressed_trajectory)
  513. if self.config.metrics_per_trajectory and metrics.was_compressed:
  514. result["compression_metrics"] = metrics.to_dict()
  515. return result, metrics
  516. def process_directory(self, input_dir: Path, output_dir: Path):
  517. """Compress every ``*.jsonl`` in ``input_dir`` into ``output_dir`` (async, parallel API calls)."""
  518. asyncio.run(self._process_directory_async(input_dir, output_dir))
  519. async def _process_one(self, run: _RunProgress, file_path: Path, entry_idx: int, entry: dict) -> Optional[tuple[dict[str, Any], TrajectoryMetrics]]:
  520. """Process one entry under the semaphore/timeout; None means dropped (timed out)."""
  521. async with run.semaphore:
  522. async with run.lock:
  523. run.in_flight += 1
  524. try:
  525. processed_entry, metrics = await asyncio.wait_for(self.process_entry_async(entry), timeout=self.config.per_trajectory_timeout)
  526. async with run.lock:
  527. self.aggregate_metrics.add_trajectory_metrics(metrics)
  528. if metrics.was_compressed:
  529. run.compressed += 1
  530. run.api_calls += metrics.summarization_api_calls
  531. run.skipped += bool(metrics.skipped_under_target)
  532. run.finish()
  533. return processed_entry, metrics
  534. except TimeoutError:
  535. self.logger.warning("Timeout processing entry from %s:%s (>%ss)", file_path, entry_idx, self.config.per_trajectory_timeout)
  536. async with run.lock:
  537. self.aggregate_metrics.trajectories_failed += 1
  538. run.timeouts += 1
  539. run.finish()
  540. return None
  541. except Exception as e:
  542. self.logger.error("Error processing entry from %s:%s: %s", file_path, entry_idx, e)
  543. async with run.lock:
  544. self.aggregate_metrics.trajectories_failed += 1
  545. run.finish(update_status=False)
  546. return entry, TrajectoryMetrics() # keep the original on error
  547. async def _process_directory_async(self, input_dir: Path, output_dir: Path):
  548. console = Console()
  549. self.aggregate_metrics.processing_start_time = datetime.now().isoformat()
  550. start_time = time.time()
  551. jsonl_files = sorted(input_dir.glob("*.jsonl"))
  552. if not jsonl_files:
  553. self.logger.warning("No JSONL files found in %s", input_dir)
  554. return
  555. console.print("\n[dim]Loading all entries...[/dim]")
  556. all_entries = [] # List of (file_path, entry_idx, entry)
  557. for file_path in jsonl_files:
  558. def _warn(line_num, e, file_path=file_path):
  559. self.logger.warning("Skipping invalid JSON at %s:%s: %s", file_path, line_num, e)
  560. all_entries.extend((file_path, idx, entry) for idx, entry in _load_jsonl(file_path, _warn))
  561. total_entries = len(all_entries)
  562. console.print(f"\n{'='*60}")
  563. console.print(f"📂 Input: {input_dir}")
  564. console.print(f"📂 Output: {output_dir}")
  565. console.print(f"📄 Files to process: {len(jsonl_files)}")
  566. console.print(f"📊 Total trajectories: {total_entries:,}")
  567. console.print(f"🎯 Target max tokens: {self.config.target_max_tokens:,}")
  568. console.print(f"📝 Summary target tokens: {self.config.summary_target_tokens}")
  569. console.print(f"⚡ Max concurrent API calls: {self.config.max_concurrent_requests}")
  570. console.print(f"{'='*60}\n")
  571. with Progress(
  572. SpinnerColumn(), TextColumn("[progress.description]{task.description}"), BarColumn(), TaskProgressColumn(),
  573. TextColumn("•"), TimeElapsedColumn(), TextColumn("•"), TimeRemainingColumn(),
  574. console=console, refresh_per_second=10, # Higher refresh for async
  575. ) as progress:
  576. run = _RunProgress(
  577. progress, progress.add_task(f"[cyan]Compressing {total_entries:,} trajectories", total=total_entries),
  578. progress.add_task("[dim]Starting...[/dim]", total=None),
  579. asyncio.Lock(), asyncio.Semaphore(self.config.max_concurrent_requests),
  580. )
  581. outcomes = await asyncio.gather(*(self._process_one(run, *item) for item in all_entries))
  582. progress.remove_task(run.status_task)
  583. # Write results preserving original order; timed-out entries are dropped.
  584. console.print("\n[dim]Writing output files...[/dim]")
  585. output_dir.mkdir(parents=True, exist_ok=True)
  586. results = {f: [] for f in jsonl_files}
  587. for (file_path, _, _), outcome in zip(all_entries, outcomes):
  588. if outcome is not None:
  589. results[file_path].append(outcome[0])
  590. for file_path in jsonl_files:
  591. _write_jsonl(output_dir / file_path.name, results[file_path])
  592. self.aggregate_metrics.processing_end_time = datetime.now().isoformat()
  593. self.aggregate_metrics.processing_duration_seconds = time.time() - start_time
  594. self._print_summary()
  595. if self.config.metrics_enabled:
  596. metrics_path = output_dir / self.config.metrics_output_file
  597. with open(metrics_path, 'w', encoding="utf-8") as f: # noqa: ASYNC230 -- small local write; a local open() is non-blocking in practice
  598. json.dump(self.aggregate_metrics.to_dict(), f, indent=2)
  599. console.print(f"\n💾 Metrics saved to {metrics_path}")
  600. def _print_summary(self):
  601. """Print comprehensive compression summary statistics."""
  602. m = self.aggregate_metrics.to_dict()
  603. s, t, u, a, z, p = m['summary'], m['tokens'], m['turns'], m['averages'], m['summarization'], m['processing']
  604. total, compressed = s['total_trajectories'], s['trajectories_compressed']
  605. pct = lambda n: (n / max(total, 1)) * 100
  606. duration = p['duration_seconds']
  607. time_str = f"{duration/60:.1f} minutes" if duration > 60 else f"{duration:.1f} seconds"
  608. sections = [
  609. ("📁 TRAJECTORIES", 54, [
  610. f"║{'':4}Total Processed: {total:>10,}{' '*32}║",
  611. f"║{'':4}├─ Compressed: {compressed:>10,} ({pct(compressed):>5.1f}%){' '*18}║",
  612. f"║{'':4}├─ Skipped (under limit):{s['trajectories_skipped_under_target']:>9,} ({pct(s['trajectories_skipped_under_target']):>5.1f}%){' '*18}║",
  613. f"║{'':4}├─ Still over limit: {s['trajectories_still_over_limit']:>10,} ({pct(s['trajectories_still_over_limit']):>5.1f}%){' '*18}║",
  614. f"║{'':4}└─ Failed: {s['trajectories_failed']:>10,}{' '*32}║",
  615. ]),
  616. ("🔢 TOKENS", 60, [
  617. f"║{'':4}Before Compression: {t['total_before']:>15,} tokens{' '*21}║",
  618. f"║{'':4}After Compression: {t['total_after']:>15,} tokens{' '*21}║",
  619. f"║{'':4}Total Saved: {t['total_saved']:>15,} tokens{' '*21}║",
  620. f"║{'':4}Overall Compression: {t['overall_compression_ratio']:>14.1%}{' '*28}║",
  621. ] + ([f"║{'':4}Space Savings: {(t['total_saved'] / t['total_before']) * 100:>14.1f}%{' '*28}║"] if t['total_before'] > 0 else [])),
  622. ("💬 CONVERSATION TURNS", 48, [
  623. f"║{'':4}Before Compression: {u['total_before']:>15,} turns{' '*22}║",
  624. f"║{'':4}After Compression: {u['total_after']:>15,} turns{' '*22}║",
  625. f"║{'':4}Total Removed: {u['total_removed']:>15,} turns{' '*22}║",
  626. ]),
  627. ("📈 AVERAGES (Compressed Trajectories Only)", 27, [
  628. f"║{'':4}Avg Compression Ratio: {a['avg_compression_ratio']:>14.1%}{' '*28}║",
  629. f"║{'':4}Avg Tokens Saved: {a['avg_tokens_saved_per_compressed']:>14,.0f}{' '*28}║",
  630. f"║{'':4}Avg Turns Removed: {a['avg_turns_removed_per_compressed']:>14.1f}{' '*28}║",
  631. ] if compressed > 0 else [f"║{'':4}No trajectories were compressed{' '*38}║"]),
  632. ("🤖 SUMMARIZATION API", 49, [
  633. f"║{'':4}API Calls Made: {z['total_api_calls']:>15,}{' '*27}║",
  634. f"║{'':4}Errors: {z['total_errors']:>15,}{' '*27}║",
  635. f"║{'':4}Success Rate: {z['success_rate']:>14.1%}{' '*28}║",
  636. ]),
  637. ("⏱️ PROCESSING TIME", 51, [
  638. f"║{'':4}Duration: {time_str:>20}{' '*22}║",
  639. f"║{'':4}Throughput: {total / max(duration, 0.001):>15.1f} traj/sec{' '*18}║",
  640. f"║{'':4}Started: {p['start_time'][:19]:>20}{' '*22}║",
  641. f"║{'':4}Finished: {p['end_time'][:19]:>20}{' '*22}║",
  642. ]),
  643. ]
  644. print("\n")
  645. print(f"╔{'═'*70}╗")
  646. print(f"║{'TRAJECTORY COMPRESSION REPORT':^70}║")
  647. for title, pad, rows in sections:
  648. print(f"╠{'═'*70}╣")
  649. print(f"║{'':2}{title}{' '*pad}║")
  650. print(f"║{'─'*70}║")
  651. for row in rows:
  652. print(row)
  653. print(f"╚{'═'*70}╝")
  654. ratios = self.aggregate_metrics.compression_ratios
  655. if ratios:
  656. saved = self.aggregate_metrics.tokens_saved_list
  657. print("\n📊 Distribution Summary:")
  658. print(f" Compression ratios: min={min(ratios):.2%}, max={max(ratios):.2%}, median={sorted(ratios)[len(ratios)//2]:.2%}")
  659. print(f" Tokens saved: min={min(saved):,}, max={max(saved):,}, median={sorted(saved)[len(saved)//2]:,}")
  660. # ---------------------------------------------------------------------------
  661. # CLI
  662. # ---------------------------------------------------------------------------
  663. def _load_cli_config(config: str, target_max_tokens: Optional[int], tokenizer: Optional[str]) -> CompressionConfig:
  664. """Load the YAML config (defaults if missing) and apply CLI overrides."""
  665. if Path(config).exists():
  666. print(f"📋 Loading config from {config}")
  667. compression_config = CompressionConfig.from_yaml(config)
  668. else:
  669. print(f"⚠️ Config not found at {config}, using defaults")
  670. compression_config = CompressionConfig()
  671. if target_max_tokens:
  672. compression_config.target_max_tokens = target_max_tokens
  673. if tokenizer:
  674. compression_config.tokenizer_name = tokenizer
  675. return compression_config
  676. def _print_dry_run(icon: str, target: Any, output_path: Path) -> None:
  677. print("\n🔍 DRY RUN MODE - analyzing without writing")
  678. print(f"{icon} Would process: {target}")
  679. print(f"{icon} Would output to: {output_path}")
  680. def _sample(entries: list, sample_percent: float) -> list:
  681. return random.sample(entries, min(max(1, int(len(entries) * sample_percent / 100)), len(entries)))
  682. def _run_file_mode(input_path: Path, output: Optional[str], compression_config: CompressionConfig, sample_percent: Optional[float], seed: int, dry_run: bool) -> None:
  683. """Single-file input: (sample,) compress via a temp directory, merge into one output file."""
  684. print("📄 Input mode: Single JSONL file")
  685. output_path = Path(output) if output else input_path.parent / (input_path.stem + compression_config.output_suffix + ".jsonl")
  686. entries = [entry for _, entry in _load_jsonl(input_path, lambda n, e: print(f"⚠️ Skipping invalid JSON at line {n}: {e}"), start=1)]
  687. total_entries = len(entries)
  688. print(f" Loaded {total_entries:,} trajectories from {input_path.name}")
  689. if sample_percent is not None:
  690. random.seed(seed)
  691. entries = random.sample(entries, max(1, int(total_entries * sample_percent / 100)))
  692. print(f" Sampled {len(entries):,} trajectories ({sample_percent}% of {total_entries:,})")
  693. if dry_run:
  694. _print_dry_run("📄", f"{len(entries):,} trajectories", output_path)
  695. return
  696. with tempfile.TemporaryDirectory() as temp_dir:
  697. temp_input_dir, temp_output_dir = Path(temp_dir) / "input", Path(temp_dir) / "output"
  698. temp_input_dir.mkdir()
  699. _write_jsonl(temp_input_dir / "trajectories.jsonl", entries)
  700. TrajectoryCompressor(compression_config).process_directory(temp_input_dir, temp_output_dir)
  701. output_path.parent.mkdir(parents=True, exist_ok=True)
  702. with open(output_path, 'w', encoding='utf-8') as out_f:
  703. for jsonl_file in sorted(temp_output_dir.glob("*.jsonl")):
  704. with open(jsonl_file, 'r', encoding='utf-8') as in_f:
  705. shutil.copyfileobj(in_f, out_f)
  706. metrics_file = temp_output_dir / compression_config.metrics_output_file
  707. if metrics_file.exists():
  708. metrics_output = output_path.parent / (output_path.stem + "_metrics.json")
  709. shutil.copy(metrics_file, metrics_output)
  710. print(f"💾 Metrics saved to {metrics_output}")
  711. print("\n✅ Compression complete!")
  712. print(f"📄 Output: {output_path}")
  713. def _run_dir_mode(input_path: Path, output: Optional[str], compression_config: CompressionConfig, sample_percent: Optional[float], seed: int, dry_run: bool) -> None:
  714. """Directory input: compress in place, or per-file sample into a temp dir first."""
  715. print("📁 Input mode: Directory of JSONL files")
  716. output_path = Path(output) if output else input_path.parent / (input_path.name + compression_config.output_suffix)
  717. if sample_percent is None:
  718. if dry_run:
  719. _print_dry_run("📁", input_path, output_path)
  720. return
  721. TrajectoryCompressor(compression_config).process_directory(input_path, output_path)
  722. else:
  723. print(f"\n⚠️ Sampling from directory: will sample {sample_percent}% from each file")
  724. with tempfile.TemporaryDirectory() as temp_dir:
  725. temp_input_dir = Path(temp_dir) / "input"
  726. temp_input_dir.mkdir()
  727. random.seed(seed)
  728. total_original = total_sampled = 0
  729. for jsonl_file in sorted(input_path.glob("*.jsonl")):
  730. entries = [entry for _, entry in _load_jsonl(jsonl_file)]
  731. sampled_entries = _sample(entries, sample_percent)
  732. total_original += len(entries)
  733. total_sampled += len(sampled_entries)
  734. _write_jsonl(temp_input_dir / jsonl_file.name, sampled_entries)
  735. print(f" Sampled {total_sampled:,} from {total_original:,} total trajectories")
  736. if dry_run:
  737. _print_dry_run("📁", temp_input_dir, output_path)
  738. return
  739. TrajectoryCompressor(compression_config).process_directory(temp_input_dir, output_path)
  740. print("\n✅ Compression complete!")
  741. def main(input: str, output: str | None = None, config: str = "configs/trajectory_compression.yaml", target_max_tokens: int | None = None,
  742. tokenizer: str | None = None, sample_percent: float | None = None, seed: int = 42, dry_run: bool = False):
  743. """
  744. Compress agent trajectories to fit within a target token budget.
  745. Supports both single JSONL files and directories containing multiple JSONL files.
  746. Optionally sample a percentage of trajectories before compression.
  747. Args:
  748. input: Path to JSONL file or directory containing JSONL files
  749. output: Output path (file for file input, directory for dir input)
  750. Default: adds "_compressed" suffix to input name
  751. config: Path to YAML configuration file
  752. target_max_tokens: Override target token count from config
  753. tokenizer: Override tokenizer name from config
  754. sample_percent: Sample this percentage of trajectories (1-100) before compression
  755. seed: Random seed for sampling reproducibility (default: 42)
  756. dry_run: Analyze without compressing (just show what would happen)
  757. """
  758. print("🗜️ Trajectory Compressor")
  759. print("=" * 60)
  760. compression_config = _load_cli_config(config, target_max_tokens, tokenizer)
  761. if sample_percent is not None:
  762. if sample_percent <= 0 or sample_percent > 100:
  763. print(f"❌ sample_percent must be between 1 and 100, got {sample_percent}")
  764. return
  765. print(f"🎲 Will sample {sample_percent}% of trajectories (seed={seed})")
  766. input_path = Path(input)
  767. if not input_path.exists():
  768. print(f"❌ Input not found: {input}")
  769. return
  770. run_mode = _run_file_mode if input_path.is_file() else _run_dir_mode
  771. run_mode(input_path, output, compression_config, sample_percent, seed, dry_run)
  772. if __name__ == "__main__":
  773. fire.Fire(main)