tool_search_catalog.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336
  1. """Deferred-tool catalog for tool search: BM25 retrieval over deferrable tool
  2. defs plus the budgeted, byte-stable catalog listing embedded in the bridge."""
  3. from __future__ import annotations
  4. import functools
  5. import math
  6. import re
  7. import threading
  8. from collections import Counter
  9. from dataclasses import dataclass, field
  10. from typing import Any, Dict, List, Optional, Tuple
  11. import snowballstemmer
  12. # Reserved bridge names: a user/plugin/MCP tool may not take them (registry override
  13. # protection rejects such registrations).
  14. TOOL_SEARCH_NAME = "tool_search"
  15. TOOL_DESCRIBE_NAME = "tool_describe"
  16. TOOL_CALL_NAME = "tool_call"
  17. BRIDGE_TOOL_NAMES = frozenset({TOOL_SEARCH_NAME, TOOL_DESCRIBE_NAME, TOOL_CALL_NAME})
  18. # Chars-per-token rule of thumb; 4.0 slightly underestimates (fewer false activations).
  19. CHARS_PER_TOKEN = 4.0
  20. @dataclass
  21. class CatalogEntry:
  22. """One deferrable tool, in a form the bridge tools can search and serve."""
  23. name: str
  24. description: str
  25. schema: dict[str, Any] # the full {"type":"function", "function": {...}} entry
  26. source: str # "mcp" | "plugin" | "other"
  27. source_name: str # toolset name, e.g. "mcp-github" or "kanban"
  28. _tokens: list[str] = field(default_factory=list) # pre-tokenized for BM25
  29. _TOKEN_RE = re.compile(r"[A-Za-z0-9]+")
  30. _thread_local = threading.local()
  31. @functools.lru_cache(maxsize=16384)
  32. def _stem(token: str) -> str:
  33. """Stem one token, memoized across stateless catalog rebuilds. Snowball stemmers carry
  34. mutable parsing state and bridge dispatch runs on parallel tool-call threads, so the
  35. stemmer is one-per-thread, created lazily."""
  36. if getattr(_thread_local, "stemmer", None) is None:
  37. _thread_local.stemmer = snowballstemmer.stemmer("english")
  38. return _thread_local.stemmer.stemWord(token)
  39. def _tokenize(text: str) -> list[str]:
  40. """Lowercase alphanumeric tokens, Snowball-stemmed (English); shared by the index and
  41. query paths so "issues" matches ``create_issue``."""
  42. return [_stem(token.lower()) for token in _TOKEN_RE.findall(text)] if text else []
  43. def _fn(td: dict[str, Any]) -> dict[str, Any]:
  44. """The ``function`` block of a tool-def (``{}`` when absent/None)."""
  45. return td.get("function") or {}
  46. def _registry_entry(name: str) -> Any:
  47. """Registry entry for ``name``; None when unregistered OR the registry raises (lookup
  48. failures must never fail a bridge call). Lazy import: tests patch the registry."""
  49. try:
  50. from tools.registry import registry
  51. return registry.get_entry(name)
  52. except Exception:
  53. return None
  54. def _registry_toolset(name: str) -> Optional[str]:
  55. """Toolset of a registered tool; None when unregistered or malformed (no str toolset)."""
  56. toolset = getattr(_registry_entry(name), "toolset", None)
  57. return toolset if isinstance(toolset, str) else None
  58. def _entry_search_text(td: dict[str, Any], source_label: str = "") -> str:
  59. """Search-text blob: split name words + source label + description + top-level parameter
  60. names (schema bodies are noise with no recall gain). The ``mcp__`` prefix is dropped — it
  61. is in every MCP document, so its IDF is ~0. The source label lets a service-name query
  62. ("linear") reach a tool whose own name omits the vendor."""
  63. fn = _fn(td)
  64. name = fn.get("name", "")
  65. name = name.removeprefix("mcp__")
  66. name_words = re.sub(r"[_.:-]", " ", name)
  67. extra = source_label if source_label and source_label not in name_words.split() else ""
  68. param_names = " ".join(((fn.get("parameters") or {}).get("properties") or {}).keys())
  69. return f"{name_words} {extra} {fn.get('description', '') or ''} {param_names}"
  70. def _classify_source(name: str) -> tuple[str, str]:
  71. """Return (source_kind, source_name) for a registered tool name."""
  72. toolset = _registry_toolset(name)
  73. if toolset is None:
  74. return ("other", "")
  75. return ("mcp" if toolset.startswith("mcp-") else "plugin", toolset)
  76. def build_catalog(tool_defs: list[dict[str, Any]]) -> list[CatalogEntry]:
  77. """Build the deferred-tool catalog from the deferrable subset of tool-defs."""
  78. catalog: list[CatalogEntry] = []
  79. for td in tool_defs:
  80. fn = _fn(td)
  81. name = fn.get("name", "")
  82. if not name:
  83. continue
  84. source, source_name = _classify_source(name)
  85. # Index the human-facing label ("linear", not "mcp-linear").
  86. source_label = _listing_group_label(source_name) if source_name else ""
  87. catalog.append(CatalogEntry(
  88. name=name, description=fn.get("description", "") or "", schema=td, source=source,
  89. source_name=source_name, _tokens=_tokenize(_entry_search_text(td, source_label))))
  90. return catalog
  91. def _bm25_score(query_tokens: list[str], doc_tokens: list[str], doc_lengths: list[int],
  92. avg_dl: float, doc_freq: dict[str, int], n_docs: int, k1: float = 1.5,
  93. b: float = 0.75) -> float:
  94. """Standard BM25 for one query against one document (inlined; the catalog is bounded —
  95. typically < 500 tools — so a dependency is not worth it)."""
  96. score = 0.0
  97. dl = len(doc_tokens)
  98. doc_tf = Counter(doc_tokens)
  99. for q in query_tokens:
  100. df, tf = doc_freq.get(q, 0), doc_tf.get(q, 0)
  101. if df and tf:
  102. idf = math.log(1 + (n_docs - df + 0.5) / (df + 0.5))
  103. score += idf * tf * (k1 + 1) / (tf + k1 * (1 - b + b * dl / max(avg_dl, 1.0)))
  104. return score
  105. _CorpusStats = tuple[list[int], float, dict[str, int], int] # doc_lengths, avg_dl, df, n_docs
  106. def _corpus_stats(catalog: list[CatalogEntry]) -> _CorpusStats:
  107. """Compute the BM25 statistics shared by every query over a catalog."""
  108. doc_lengths = [len(entry._tokens) for entry in catalog]
  109. avg_dl = sum(doc_lengths) / max(len(doc_lengths), 1)
  110. doc_freq = Counter(tok for entry in catalog for tok in set(entry._tokens))
  111. return doc_lengths, avg_dl, dict(doc_freq), len(catalog)
  112. def _gate_token(query_tokens: list[str], doc_freq: dict[str, int], n_docs: int) -> str:
  113. """The query token with the highest IDF: the word that names the intent. ``send``,
  114. ``read``, ``create`` sit in dozens of tool documents and separate nothing; ``gmail``,
  115. ``github``, ``incident`` sit in a few and separate everything. A document without this
  116. token answered a different question, however many common tokens it shares."""
  117. def _idf(token: str) -> float:
  118. df = doc_freq.get(token, 0)
  119. return math.log(1 + (n_docs - df + 0.5) / (df + 0.5))
  120. return max(query_tokens, key=_idf)
  121. # Relevance floor. The rarest-token gate stops queries whose intent word no tool carries; it
  122. # does not stop a long hunt whose every word exists SOMEWHERE in the catalog while no single
  123. # tool carries more than one of them (observed: "run shell command execute code python" -> a
  124. # workflow-rerun tool sharing only "run"; the model read "results exist" as "it is in here"
  125. # and re-searched 216 times). A document must also match MIN_QUERY_TERM_COVERAGE of the
  126. # query's ANSWERABLE terms (present in at least one document) before it is offered. Coverage
  127. # only engages from MIN_ANSWERABLE_TERMS_FOR_COVERAGE terms up: short queries ("list issues")
  128. # legitimately differ from a tool by a word, and are where a coverage rule costs real recall.
  129. MIN_QUERY_TERM_COVERAGE = 0.5
  130. MIN_ANSWERABLE_TERMS_FOR_COVERAGE = 4
  131. def _required_term_coverage(answerable_term_count: int) -> int:
  132. """How many of a query's ANSWERABLE unique terms a document must match to be offered:
  133. one below the engagement floor, else at least half, rounded up."""
  134. if answerable_term_count < MIN_ANSWERABLE_TERMS_FOR_COVERAGE:
  135. return 1
  136. return math.ceil(answerable_term_count * MIN_QUERY_TERM_COVERAGE)
  137. def search_catalog(catalog: list[CatalogEntry], query: str, limit: int = 5, *,
  138. corpus_stats: Optional[_CorpusStats] = None) -> list[CatalogEntry]:
  139. """Top-``limit`` catalog entries for ``query`` by BM25 (exact name match ranks first).
  140. Admission is by the query's rarest token (:func:`_gate_token`), not by ``score > 0``:
  141. BM25 is additive over the tokens a document shares with the query, so on a large catalog
  142. ``score > 0`` admits one-token matches and fills every slot with them (measured: "send
  143. gmail email" returned 5 incident tools that only shared ``email``). A token no document
  144. carries admits nothing; the caller's empty-group hint tells the model to retry without it.
  145. Long queries additionally need :func:`_required_term_coverage` of their answerable terms."""
  146. query_tokens = _tokenize(query) if catalog and limit > 0 else []
  147. if not query_tokens:
  148. return []
  149. corpus_stats = corpus_stats or _corpus_stats(catalog)
  150. doc_freq = corpus_stats[2]
  151. gate = _gate_token(query_tokens, doc_freq, corpus_stats[3])
  152. answerable = {t for t in query_tokens if doc_freq.get(t, 0) > 0}
  153. required_terms = _required_term_coverage(len(answerable))
  154. exact_name = query.strip().lower()
  155. def _admitted(entry: CatalogEntry) -> bool:
  156. """Carries the intent word AND enough of the answerable terms (exact name is exempt)."""
  157. if entry.name.lower() == exact_name:
  158. return True
  159. tokens = set(entry._tokens)
  160. return gate in tokens and sum(1 for t in answerable if t in tokens) >= required_terms
  161. scored = [
  162. (float("inf") if entry.name.lower() == exact_name
  163. else _bm25_score(query_tokens, entry._tokens, *corpus_stats), entry)
  164. for entry in catalog if _admitted(entry)]
  165. scored.sort(key=lambda x: x[0], reverse=True)
  166. return [e for _, e in scored[:limit]]
  167. # Sentence end: ., !, ? followed by whitespace/EOS, not inside e.g./i.e./etc.
  168. _SENTENCE_END_RE = re.compile(r"(?<!\be\.g)(?<!\bi\.e)(?<!\betc)[.!?](?=\s|$)")
  169. # A catalog line is its tool's first sentence: built-in tools keep that sentence within this cap
  170. # and lead with the key capability (tests/tools/test_tool_search.py pins it), so no line clips.
  171. CATALOG_SUMMARY_MAX_CHARS = 80
  172. def _short_desc(description: str, max_chars: int = CATALOG_SUMMARY_MAX_CHARS) -> str:
  173. """First sentence of a tool description, clipped to ``max_chars`` on a word boundary.
  174. ``e.g.``/``i.e.``/``etc.`` do not end a sentence; whitespace normalization and the regex
  175. search stay linear-time on hostile input."""
  176. text = " ".join((description or "").split())
  177. m = _SENTENCE_END_RE.search(text)
  178. text = text[:m.end()] if m else text
  179. if len(text) <= max_chars:
  180. return text
  181. clipped = text[:max_chars]
  182. clipped = clipped.rsplit(" ", 1)[0] if " " in clipped else clipped
  183. return clipped.rstrip(",;: ") + "…"
  184. def _listing_group_label(source_name: str) -> str:
  185. """Human-facing group heading for a toolset, e.g. ``mcp-github`` -> ``github``."""
  186. label = source_name or "other"
  187. return label.removeprefix("mcp-")
  188. def hidden_declared_sources() -> list[dict[str, Any]]:
  189. """Return deterministic summaries for declared MCP servers hidden by their check."""
  190. from hermes_platform import declaration
  191. from tools.mcp_liveness import unavailable_details
  192. from tools.registry import registry
  193. grouped: dict[str, list[Any]] = {}
  194. for entry in registry.get_all_entries():
  195. if entry.toolset.startswith("mcp-"):
  196. grouped.setdefault(entry.toolset[4:], []).append(entry)
  197. rows: list[dict[str, Any]] = []
  198. for server_name in sorted(grouped):
  199. if declaration.lookup(server_name) is None:
  200. continue
  201. entries = grouped[server_name]
  202. if any(entry.check_fn is None or bool(entry.check_fn()) for entry in entries):
  203. continue
  204. details = unavailable_details(server_name)
  205. if details is None:
  206. continue
  207. _decl, _current, sentence = details
  208. rows.append({"name": server_name, "tool_count": len(entries), "unavailable": sentence})
  209. return rows
  210. def build_catalog_listing_with_form(
  211. deferrable: list[dict[str, Any]], *, max_tokens: int = 4000) -> tuple[Optional[str], str]:
  212. """Render the deferred-catalog manifest: ``- name: short desc`` lines grouped per source.
  213. Returns ``(text, form)``; form is ``"full"``, ``"names"``, ``"mixed"`` (oversized servers
  214. collapsed to a name + count line), ``"groups"`` (every server summarized) or ``"none"``
  215. (over budget even summarized -> text is None). Ordering is deterministic (sorted groups
  216. and tools) so the block is byte-stable — the request prefix stays cacheable. Degradation
  217. is PER SERVER, largest first: one huge server must not cost a small one its listing."""
  218. groups: dict[str, list[tuple[str, str]]] = {}
  219. for td in deferrable:
  220. fn = _fn(td)
  221. name = fn.get("name", "")
  222. if name:
  223. # _classify_source gives ("other", "") when unregistered; the label of "" is "other".
  224. label = _listing_group_label(_classify_source(name)[1])
  225. groups.setdefault(label, []).append((name, _short_desc(fn.get("description", ""))))
  226. unavailable = hidden_declared_sources()
  227. if not groups and not unavailable:
  228. return None, "none"
  229. def render_group(label: str, mode: str) -> str:
  230. """Render one server's block. mode: 'full' | 'names' | 'summary'."""
  231. tools = sorted(groups[label])
  232. if mode == "summary":
  233. return (f"{label} ({len(tools)} tools — names not listed; "
  234. f"discover via `{TOOL_SEARCH_NAME}`)")
  235. if mode == "full":
  236. lines = [f"- {name}: {desc}" if desc else f"- {name}" for name, desc in tools]
  237. else:
  238. lines = [", ".join(name for name, _ in tools)] if len(tools) > 1 else [f"- {tools[0][0]}"]
  239. # A one-tool group's "<label> tools (1):" heading says nothing its single line does not.
  240. return "\n".join(lines if len(tools) == 1 else [f"{label} tools ({len(tools)}):", *lines])
  241. header = ("Deferred tool catalog (call schemas via "
  242. f"`{TOOL_DESCRIBE_NAME}`, invoke via `{TOOL_CALL_NAME}`):")
  243. def assemble_if_fits(modes: dict[str, str]) -> Optional[str]:
  244. # Headless one-tool lines go first, straight under the catalog header: placed between
  245. # groups they would read as members of the group above them.
  246. singles = [render_group(label, modes[label]) for label in sorted(groups)
  247. if len(groups[label]) == 1 and modes[label] != "summary"]
  248. available_blocks = {label: render_group(label, modes[label]) for label in groups
  249. if len(groups[label]) > 1 or modes[label] == "summary"}
  250. unavailable_blocks = {
  251. row["name"]: (
  252. f"{row['name']} ({row['tool_count']} tools unavailable: {row['unavailable']})"
  253. if row.get("tool_count") is not None
  254. else f"{row['name']} (tools unavailable: {row['unavailable']})"
  255. )
  256. for row in unavailable
  257. }
  258. blocks = [
  259. available_blocks[label] if label in available_blocks else unavailable_blocks[label]
  260. for label in sorted(available_blocks | unavailable_blocks)
  261. ]
  262. text = "\n".join([header] + singles + blocks)
  263. return text if math.ceil(len(text) / CHARS_PER_TOKEN) <= max_tokens else None
  264. for mode in ("full", "names"): # 1. everything full; 2. everything names-only
  265. text = assemble_if_fits({lbl: mode for lbl in groups})
  266. if text is not None:
  267. return text, mode
  268. # 3. Collapse the LARGEST rendered groups first (deterministic: size then label); the servers
  269. # left uncollapsed get their full lines back when those fit, else names-only.
  270. summarized: set[str] = set()
  271. for lbl in sorted(groups, key=lambda lbl: (-len(render_group(lbl, "names")), lbl)):
  272. summarized.add(lbl)
  273. for rest in ("full", "names"):
  274. text = assemble_if_fits({g: "summary" if g in summarized else rest for g in groups})
  275. if text is not None:
  276. return text, "groups" if len(summarized) == len(groups) else "mixed"
  277. return None, "none"