prompt_size.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240
  1. """Prompt-size diagnostic: ``hermes prompt-size``.
  2. Builds a real inspection agent (so the numbers match what ships on the wire) but never makes a
  3. network call: dummy credentials force ``AIAgent.__init__`` down the direct-construction path, then
  4. ``build_system_prompt_parts`` / ``agent.tools`` are inspected offline.
  5. """
  6. from __future__ import annotations
  7. import json
  8. import re
  9. from pathlib import Path
  10. from typing import Any, Dict, List, Optional, Tuple
  11. _SKILLS_BLOCK_RE = re.compile(r"<available_skills>.*?</available_skills>", re.DOTALL)
  12. # A rendered skill entry is ``- name: desc`` (or ``- name``) indented four spaces under its category
  13. # header, or two when it is a one-skill category's only row; headers never start with ``- ``.
  14. _SKILL_LINE_PREFIX = "- "
  15. # Posture-demoted categories render all visible skill names on one shared line.
  16. _NAMES_ONLY_LINE_RE = re.compile(r"^ .+ \[names only\]: (?P<names>.+)$")
  17. # Cap the human-readable "Skills by size" table; ``--json`` always has them all.
  18. _SKILLS_TABLE_LIMIT = 20
  19. def _bytes(s: str) -> int:
  20. return len(s.encode("utf-8"))
  21. def _size(text: str) -> dict[str, int]:
  22. return {"chars": len(text), "bytes": _bytes(text)}
  23. def _fmt_kb(n: int) -> str:
  24. return f"{n / 1024:.1f} KB"
  25. def _tool_name(tool: Any) -> str:
  26. """Callable name of a tool schema (OpenAI ``function`` shape)."""
  27. if not isinstance(tool, dict):
  28. return ""
  29. fn = tool.get("function")
  30. return str(fn["name"]) if isinstance(fn, dict) and fn.get("name") else str(tool.get("name", ""))
  31. def _build_inspection_agent(platform: str) -> Any:
  32. """Offline AIAgent for prompt inspection: dummy ``api_key`` + ``base_url`` force the
  33. direct-construction path (no provider auto-detection, no network); toolsets resolve the way
  34. the gateway does so the breakdown matches a real session.
  35. """
  36. from run_agent import AIAgent
  37. from hermes_cli.config import load_config
  38. from hermes_cli.tools_config import _get_platform_tools
  39. from agent.skill_utils import parse_config_string_list
  40. cfg = load_config()
  41. model_cfg = cfg.get("model", {}) if isinstance(cfg.get("model"), dict) else {}
  42. agent_cfg = cfg.get("agent") or {}
  43. return AIAgent(
  44. model=model_cfg.get("default") or model_cfg.get("model") or "",
  45. api_key="inspect-only", base_url="https://openrouter.ai/api/v1", quiet_mode=True, save_trajectories=False,
  46. platform=platform, enabled_toolsets=sorted(_get_platform_tools(cfg, platform)),
  47. disabled_toolsets=parse_config_string_list(agent_cfg.get("disabled_toolsets")) or None,
  48. )
  49. def _skill_md_paths_by_name() -> dict[str, Path]:
  50. """Map each installed skill's frontmatter ``name`` AND directory name to its ``SKILL.md``.
  51. Local skills win over external dirs (``get_all_skills_dirs`` yields local first), matching
  52. the index's own precedence.
  53. """
  54. from agent.skill_utils import get_all_skills_dirs, iter_skill_index_files, parse_frontmatter
  55. mapping: dict[str, Path] = {}
  56. for skills_dir in get_all_skills_dirs():
  57. if not skills_dir.exists():
  58. continue
  59. for skill_file in iter_skill_index_files(skills_dir, "SKILL.md"):
  60. dir_name = skill_file.parent.name
  61. try:
  62. frontmatter, _ = parse_frontmatter(
  63. skill_file.read_text(encoding="utf-8-sig")
  64. )
  65. frontmatter_name = str(frontmatter.get("name") or dir_name)
  66. except Exception:
  67. frontmatter_name = dir_name
  68. mapping.setdefault(frontmatter_name, skill_file) # first (local) occurrence wins
  69. mapping.setdefault(dir_name, skill_file)
  70. return mapping
  71. def _compute_skills_breakdown(skills_block: str) -> list[dict[str, Any]]:
  72. """Per-skill byte breakdown parsed from the rendered ``<available_skills>``.
  73. ``index_line_bytes`` is the skill's attributed always-on index cost. For a compact
  74. ``[names only]`` line each name keeps its own bytes plus an even share of the shared prefix
  75. and separators.
  76. """
  77. name_to_path = _skill_md_paths_by_name()
  78. entries: list[dict[str, Any]] = []
  79. def append_entry(name: str, **index_fields: int) -> None: # kwarg order == output key order
  80. path = name_to_path.get(name)
  81. md_bytes: Optional[int] = None
  82. try:
  83. md_bytes = path.stat().st_size if path is not None else None
  84. except OSError:
  85. pass
  86. entries.append({"name": name, **index_fields, "skill_md_bytes": md_bytes, "path": str(path) if path is not None else ""})
  87. for line in skills_block.splitlines():
  88. line_bytes = _bytes(line)
  89. if (compact_match := _NAMES_ONLY_LINE_RE.match(line)) is not None:
  90. names = [n.strip() for n in compact_match.group("names").split(",") if n.strip()]
  91. name_bytes = [_bytes(name) for name in names]
  92. shared_base, shared_remainder = divmod(line_bytes - sum(name_bytes), len(names)) if names else (0, 0)
  93. for index, name in enumerate(names):
  94. shared = shared_base + (1 if index < shared_remainder else 0)
  95. append_entry(name, index_line_bytes=name_bytes[index] + shared, index_line_total_bytes=line_bytes,
  96. index_line_shared_bytes=shared, index_line_skill_count=len(names))
  97. elif (row := line.lstrip()).startswith(_SKILL_LINE_PREFIX):
  98. # Partition on ``": "`` (not ``:``) so namespaced names like ``codex:rescue`` stay intact.
  99. name = row[len(_SKILL_LINE_PREFIX):].partition(": ")[0].strip()
  100. if name:
  101. append_entry(name, index_line_bytes=line_bytes, index_line_total_bytes=line_bytes,
  102. index_line_shared_bytes=0, index_line_skill_count=1)
  103. entries.sort(key=lambda e: (-(e["skill_md_bytes"] or 0), e["name"]))
  104. return entries
  105. def _compute_toolsets_breakdown(tools: list[Any]) -> list[dict[str, Any]]:
  106. """Per-toolset schema-byte breakdown, largest-first (tie-broken by name). Each tool is
  107. attributed to its single canonical toolset so ``json_bytes`` sums to the grand total.
  108. """
  109. from tools.registry import registry
  110. tool_to_toolset = registry.get_tool_to_toolset_map()
  111. groups: dict[str, dict[str, Any]] = {}
  112. for tool in tools:
  113. toolset = tool_to_toolset.get(_tool_name(tool)) or "(unknown)"
  114. group = groups.setdefault(toolset, {"toolset": toolset, "tool_count": 0, "json_bytes": 0})
  115. group["tool_count"] += 1
  116. group["json_bytes"] += _bytes(json.dumps(tool, ensure_ascii=False))
  117. return sorted(groups.values(), key=lambda g: (-g["json_bytes"], g["toolset"]))
  118. def compute_prompt_breakdown(platform: str = "cli") -> dict[str, Any]:
  119. """Prompt-size measurements for a fresh session: ``system_prompt``, ``skills_index``,
  120. ``memory``, ``user_profile``, ``tools``, ``sections`` (the three prompt tiers), and the
  121. largest-first ``skills_breakdown`` / ``toolsets_breakdown`` ("what should I disable?").
  122. """
  123. from agent.system_prompt import build_system_prompt, build_system_prompt_parts
  124. agent = _build_inspection_agent(platform)
  125. parts = build_system_prompt_parts(agent)
  126. full = build_system_prompt(agent)
  127. stable, context, volatile = (parts.get(k, "") for k in ("stable", "context", "volatile"))
  128. # The skills index lives in the volatile tier (moved from stable so skill edits don't
  129. # invalidate the cached identity prefix); fall back to stable for older layouts.
  130. skills_match = _SKILLS_BLOCK_RE.search(volatile) or _SKILLS_BLOCK_RE.search(stable)
  131. skills_index = skills_match.group(0) if skills_match else ""
  132. # Memory + user profile are joined into ``volatile``; re-derive them from the store so the
  133. # numbers stay attributable.
  134. memory_block = user_block = ""
  135. store = getattr(agent, "_memory_store", None)
  136. if store is not None:
  137. try:
  138. if getattr(agent, "_memory_enabled", True):
  139. memory_block = store.format_for_system_prompt("memory") or ""
  140. if getattr(agent, "_user_profile_enabled", True):
  141. user_block = store.format_for_system_prompt("user") or ""
  142. except Exception:
  143. pass
  144. tools = getattr(agent, "tools", None) or []
  145. sections: list[tuple[str, int, int]] = [
  146. (label, len(text), _bytes(text))
  147. for label, text in (("stable (identity/guidance/skills)", stable), ("context (AGENTS.md/cwd files)", context),
  148. ("volatile (memory/profile/timestamp)", volatile))
  149. ]
  150. return {
  151. "platform": platform,
  152. "model": getattr(agent, "model", "") or "",
  153. "system_prompt": _size(full),
  154. "skills_index": _size(skills_index),
  155. "memory": _size(memory_block),
  156. "user_profile": _size(user_block),
  157. "tools": {"count": len(tools), "json_bytes": _bytes(json.dumps(tools, ensure_ascii=False))},
  158. "sections": sections,
  159. "skills_breakdown": _compute_skills_breakdown(skills_index),
  160. "toolsets_breakdown": _compute_toolsets_breakdown(tools),
  161. }
  162. def render_breakdown(data: dict[str, Any]) -> str:
  163. """Render the breakdown as plain text suitable for a terminal."""
  164. sp = data["system_prompt"]
  165. tools = data["tools"]
  166. lines: list[str] = [
  167. f"Prompt-size breakdown (platform={data['platform']}, model={data['model'] or 'unset'})", "",
  168. f" System prompt total : {sp['bytes']:>8,} B ({_fmt_kb(sp['bytes'])}, {sp['chars']:,} chars)", "",
  169. " Major blocks:",
  170. ]
  171. for label, key in (("skills index", "skills_index"), ("memory", "memory"), ("user profile", "user_profile")):
  172. byts = data[key]["bytes"]
  173. lines.append(f" {label:<19}: {byts:>8,} B ({_fmt_kb(byts)})")
  174. lines += ["", " Prompt tiers:"] + [f" {label:<36}: {byts:>8,} B ({_fmt_kb(byts)})" for label, _chars, byts in data["sections"]]
  175. lines += ["", f" Tool schemas : {tools['json_bytes']:>8,} B ({_fmt_kb(tools['json_bytes'])}, {tools['count']} tools)"]
  176. if toolsets := data.get("toolsets_breakdown") or []:
  177. lines += ["", " Toolsets by size (tool-schema JSON, largest first):", f" {'toolset':<22} {'tools':>5} {'schema':>10}"]
  178. lines += [f" {ts['toolset']:<22} {ts['tool_count']:>5} {ts['json_bytes']:>8,} B ({_fmt_kb(ts['json_bytes'])})" for ts in toolsets]
  179. # Per-skill cost — index line (always shipped) vs SKILL.md (read on load).
  180. if skills := data.get("skills_breakdown") or []:
  181. lines += ["", " Skills by size (SKILL.md on-disk = read cost; index cost = attributed always-on bytes, largest first):",
  182. f" {'skill':<28} {'SKILL.md':>10} {'index cost':>10}"]
  183. shown = skills[:_SKILLS_TABLE_LIMIT]
  184. for sk in shown:
  185. md = sk["skill_md_bytes"]
  186. md_str = f"{md:>8,} B" if md is not None else f"{'n/a':>10}"
  187. name = sk["name"] if len(sk["name"]) <= 28 else sk["name"][:27] + "…"
  188. lines.append(f" {name:<28} {md_str} {sk['index_line_bytes']:>8,} B")
  189. if (remaining := len(skills) - len(shown)) > 0:
  190. lines.append(f" … and {remaining} more (use --json for the full list)")
  191. return "\n".join(lines)
  192. def cmd_prompt_size(args: Any) -> None:
  193. """Entry point for ``hermes prompt-size``."""
  194. try:
  195. data = compute_prompt_breakdown(getattr(args, "platform", "cli") or "cli")
  196. except Exception as e:
  197. print(f"Could not compute prompt-size breakdown: {e}")
  198. return
  199. print(json.dumps(data, ensure_ascii=False, indent=2) if getattr(args, "json", False) else render_breakdown(data))