shared_metrics_events.py 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164
  1. """Call-site API for process-level shared-metrics facts.
  2. Setup completion, slash commands, compression, model switches, fallbacks and extension installs
  3. happen outside any single tool or model call, so their producers call these functions directly.
  4. Every function takes RAW values (normalization lives in shared_metrics_fields), is a no-op unless
  5. shared metrics are enabled, and never raises into the caller.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import threading
  10. from typing import Any, Callable
  11. from . import shared_metrics_contract as contract
  12. from . import shared_metrics_fields as fields_
  13. logger = logging.getLogger(__name__)
  14. def _emit(mark: str, build: Callable[..., dict[str, str] | None], **raw: Any) -> None:
  15. try:
  16. from .relay_shared_metrics import enabled, record_process_mark
  17. if not enabled():
  18. return
  19. data = build(**raw)
  20. if data is not None:
  21. record_process_mark(mark, data)
  22. except Exception:
  23. logger.debug("Shared-metrics %s not recorded", mark, exc_info=True)
  24. def emit_saved(marks: list[tuple[str, dict[str, str]]]) -> int:
  25. """``_emit`` for facts recovered from a file: how many rows are settled (saved, rejected, or
  26. collection off), so the caller deletes the file only then. Blocks on the store: no hot paths."""
  27. try:
  28. from .relay_shared_metrics import record_process_marks_saved
  29. return record_process_marks_saved(marks) if marks else 0
  30. except Exception:
  31. logger.debug("Shared-metrics recovered rows not recorded", exc_info=True)
  32. return 0
  33. def record_setup_completed(*, surface: str, provider: str | None) -> None:
  34. _emit(contract.SETUP_COMPLETED_MARK, fields_.setup_completed_fields, surface=surface, provider=provider)
  35. def record_slash_command(*, command: str, surface: str) -> None:
  36. _emit(contract.SLASH_COMMAND_MARK, fields_.slash_command_fields, command=command, surface=surface)
  37. def record_tool_search(*, scope: str, hits: int, outcome: str) -> None:
  38. from .shared_metrics_connectors import tool_search_fields
  39. _emit(contract.TOOL_SEARCH_MARK, tool_search_fields, scope=scope, hits=hits, outcome=outcome)
  40. def record_compression(
  41. *, trigger: str, outcome: str, tokens_before: int | None, context_length: int | None,
  42. failure_class: str | None = None,
  43. ) -> None:
  44. _emit(
  45. contract.COMPRESSION_MARK, fields_.compression_fields, trigger=trigger, outcome=outcome,
  46. tokens_before=tokens_before, context_length=context_length, failure_class=failure_class,
  47. )
  48. # Per-thread, not on the compressor or the agent: abort paths restore the compressor snapshot (seed
  49. # included) before they emit, and a stalled attempt's worker can unwind after its stall-fallback retry
  50. # began on another thread. An attempt begins and emits on one thread (the pool worker or the caller).
  51. _compression_attempt = threading.local()
  52. # Attempts that ended before anything could fail: another path held the lock, the transcript had
  53. # nothing summarizable (no LLM call was made), the user stopped it, or a newer attempt replaced it.
  54. # Counting these as ``failed`` made a session that is simply all protected tail read as broken.
  55. _SKIPPED_COMPRESSION_CLASSES = contract.COMPRESSION_SKIP_CLASSES
  56. def begin_compression_attempt(trigger: str, tokens_before: Any) -> None:
  57. _compression_attempt.pending = (trigger, tokens_before)
  58. def discard_compression_attempt() -> None:
  59. """Clear this thread's pending attempt without recording a shared metric."""
  60. _compression_attempt.pending = None
  61. def finish_compression_attempt(
  62. commit_status: str, failure_class: str | None, context_length: Any, agent: Any = None,
  63. ) -> None:
  64. """Count this thread's pending compression attempt once; a committed one broke the prompt cache."""
  65. pending, _compression_attempt.pending = getattr(_compression_attempt, "pending", None), None
  66. if not pending:
  67. return
  68. # Classify the stored value, so the outcome and the recorded reason always agree.
  69. committed = commit_status == "committed"
  70. reason = fields_.compression_failure_class("success" if committed else "failed", failure_class)
  71. outcome = "success" if committed else "skipped" if reason in _SKIPPED_COMPRESSION_CLASSES else "failed"
  72. record_compression(
  73. trigger=pending[0], outcome=outcome, tokens_before=pending[1], context_length=context_length,
  74. failure_class=reason,
  75. )
  76. if outcome == "success" and agent is not None:
  77. from .shared_metrics_efficiency import record_cache_break
  78. record_cache_break(agent, "compression")
  79. def record_gateway_slash_command(event: Any) -> None:
  80. """Count a user-typed gateway slash command once: a queued busy-path event re-enters
  81. ``_handle_message`` when drained, so the event carries the mark."""
  82. command = event.get_command()
  83. if not command or getattr(event, "_slash_command_counted", False):
  84. return
  85. event._slash_command_counted = True
  86. record_slash_command(command=command, surface="gateway")
  87. def record_model_switch(
  88. *, from_provider: str | None, to_provider: str | None, surface: str, from_model: str | None = None,
  89. session_id: str | None = None,
  90. ) -> None:
  91. """``from_model`` also counts the switch as friction against the model the user left;
  92. ``session_id`` (the switched session) also counts how many turns that model served first."""
  93. _emit(
  94. contract.MODEL_SWITCH_MARK, fields_.model_switch_fields,
  95. from_provider=from_provider, to_provider=to_provider, surface=surface,
  96. )
  97. if from_model:
  98. from .shared_metrics_model import record_model_friction
  99. record_model_friction("switch_away", provider=from_provider, model=from_model)
  100. if session_id:
  101. try:
  102. from .relay_shared_metrics import record_model_switch_after
  103. record_model_switch_after(str(session_id))
  104. except Exception:
  105. logger.debug("Shared-metrics model_switch_after not recorded", exc_info=True)
  106. def record_fallback(*, from_provider: str | None, to_provider: str | None, reason: Any) -> None:
  107. _emit(
  108. contract.FALLBACK_MARK, fields_.fallback_fields,
  109. from_provider=from_provider, to_provider=to_provider, reason=reason,
  110. )
  111. def record_extension_install(
  112. *, kind: str, source: str, name: str | None, outcome: str,
  113. # ---- iuf c1 ----
  114. failure_class: str | None = None, registry: str | None = None, error: BaseException | None = None,
  115. # ---- end iuf c1 ----
  116. ) -> None:
  117. """``failure_class`` names why a failed install stopped (the kind's closed set); a raised
  118. ``error`` is classified by its tagged class or type instead, inside the metrics guard.
  119. ``registry`` is the skills-hub adapter id that resolved/served a skill."""
  120. _emit(
  121. contract.EXTENSION_INSTALL_MARK, fields_.extension_install_fields,
  122. kind=kind, source=source, name=name, outcome=outcome,
  123. failure_class=failure_class, registry=registry, error=error,
  124. )