shared_metrics_connectors.py 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124
  1. """Connector work in the shared metrics: the search-to-execute chain and the batch and search rows.
  2. Each entry of a connector ``tool_call`` batch dispatches with its own call id (``<id>#<position>``,
  3. ``tools/connectors/dispatch.py``), so entries count as their own calls; the wrapper's post under the
  4. bare id is dropped once an entry ran. Connector slugs, tool names and query text never leave this
  5. module: rows carry only buckets and closed enums.
  6. """
  7. from __future__ import annotations
  8. import json
  9. from dataclasses import dataclass, field
  10. from typing import Any, Iterable
  11. from . import shared_metrics_contract as contract
  12. from .shared_metrics_contract import CONNECTOR_NAME_PREFIX
  13. # Described names are kept per conversation to answer "was this tool described before it ran".
  14. _MAX_DESCRIBED = 256
  15. def is_connector_name(name: Any) -> bool:
  16. return isinstance(name, str) and name.startswith(CONNECTOR_NAME_PREFIX)
  17. def _call_entries(args: Any) -> list[dict[str, Any]]:
  18. """The ``calls[]`` entries of a ``tool_call`` invocation; [] when the arguments do not parse."""
  19. if not isinstance(args, dict):
  20. return []
  21. from tools.tool_search_validation import normalize_tool_call_entries
  22. entries, err = normalize_tool_call_entries(args)
  23. return [] if err else entries
  24. def is_connector_batch(event: dict[str, Any]) -> bool:
  25. """A ``tool_call`` wrapper whose entries are all connector calls (each entry is counted itself)."""
  26. if event.get("tool_name") != "tool_call":
  27. return False
  28. entries = _call_entries(event.get("args"))
  29. return bool(entries) and all(is_connector_name(entry.get("name")) for entry in entries)
  30. def is_batch_entry(identity: tuple[str, str, str], wrapper: tuple[str, str, str]) -> bool:
  31. """Whether ``identity`` is an entry (``<id>#<n>``) of the batch whose wrapper has identity ``wrapper``."""
  32. return bool(wrapper[2]) and identity[2].startswith(f"{wrapper[2]}#") and identity[:2] == wrapper[:2]
  33. def batch_entry_ran(identities: Iterable[tuple[str, str, str]], wrapper: tuple[str, str, str]) -> bool:
  34. """Whether an entry of the batch whose wrapper has identity ``wrapper`` was already counted."""
  35. return any(is_batch_entry(identity, wrapper) for identity in identities)
  36. def _described_names(args: Any) -> list[str]:
  37. """``tool_describe`` names as its dispatcher reads them: a bare string is one name, items stripped."""
  38. names = args.get("names") if isinstance(args, dict) else None
  39. names = [names] if isinstance(names, str) else names if isinstance(names, list) else []
  40. return [str(name or "").strip() for name in names]
  41. @dataclass
  42. class ConnectorSessionState:
  43. """Per-conversation chain state; the caller holds the session lock. Compression hands it to the
  44. rotated-to segment so the chain spans the whole conversation."""
  45. searches_since_execute: int = 0
  46. described: set[str] = field(default_factory=set)
  47. def observe(self, event: dict[str, Any]) -> dict[str, str] | None:
  48. """Update the chain after a counted call; the ``hermes.connector_chain.count`` fields for an entry."""
  49. name = event.get("tool_name")
  50. if name == "tool_search":
  51. self.searches_since_execute += 1
  52. return None
  53. if name == "tool_describe":
  54. for described in _described_names(event.get("args")):
  55. if is_connector_name(described) and len(self.described) < _MAX_DESCRIBED:
  56. self.described.add(described)
  57. return None
  58. if not is_connector_name(name):
  59. return None
  60. fields = {
  61. "described_first": "yes" if name in self.described else "no",
  62. "outcome": contract.tool_outcome(event),
  63. "searches_before_bucket": contract.count_bucket(self.searches_since_execute),
  64. }
  65. self.searches_since_execute = 0
  66. return fields
  67. # ---- row builders ----------------------------------------------------------------------------
  68. def tool_batch_fields(route: dict[str, str], tool_calls: Iterable[Any]) -> dict[str, str] | None:
  69. """One model reply's tool calls: how many logical calls (``tool_call`` entries counted one by
  70. one) and how many of them were connector calls. None for a reply with no calls."""
  71. total = connectors = 0
  72. for tc in tool_calls:
  73. fn = getattr(tc, "function", None)
  74. name = getattr(fn, "name", None)
  75. names = [name]
  76. if name == "tool_call":
  77. entries = _call_entries(_parsed_arguments(getattr(fn, "arguments", None)))
  78. names = [entry.get("name") for entry in entries] or [name]
  79. total += len(names)
  80. connectors += sum(1 for entry_name in names if is_connector_name(entry_name))
  81. if not total:
  82. return None
  83. share = "none" if not connectors else "all" if connectors == total else "some"
  84. return {**route, "connector_share": share, "size_bucket": contract.count_bucket(total)}
  85. def _parsed_arguments(raw: Any) -> Any:
  86. if isinstance(raw, dict):
  87. return raw
  88. try:
  89. return json.loads(raw) if isinstance(raw, str) else None
  90. except ValueError:
  91. return None
  92. def tool_search_fields(*, scope: str, hits: int, outcome: str) -> dict[str, str] | None:
  93. if scope not in contract.TOOL_SEARCH_SCOPES or outcome not in contract.TOOL_SEARCH_OUTCOMES:
  94. return None
  95. return {"hits_bucket": contract.count_bucket(hits), "outcome": outcome, "scope": scope}