shared_metrics_busy.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. """Is the agent making people wait? Busy-chat arrivals and the day's peak of chats running at once.
  2. ``hermes.gateway.busy_arrival.count``: one row per human message that arrives while its chat's turn
  3. is still running (the adapter's busy path, or the runner's running-turn check at FIFO admission for a
  4. message the adapter saw as idle because an earlier queued turn now runs outside its session guard),
  5. recorded once its fate is known: it reached the agent (its own turn started, or it was steered into
  6. the running one), it was folded into another busy message, or it never ran (cancelled by /stop or
  7. /new, refused before its turn, deleted with its chat). Commands, internal events, plain-text approval
  8. answers and plugin-consumed input are not counted. The arrival rides the event object itself
  9. (``_busy_arrival``), so the merge, admission and settle seams need no lookup; the durable FIFO's
  10. admission id links it to the turn that later runs the restored copy. It belongs to the profile that
  11. runs the chat's turns (the event's routing identity), never one inferred from the chat id.
  12. ``gateway_peak_busy_chats_bucket`` (on ``hermes.engagement.day.count``): the most distinct chats
  13. (session keys) that had a turn running at the same moment during the UTC day, per owning profile.
  14. Tracked in memory from the runner's running-turn set at every turn boundary and folded into the
  15. profile's engagement day state, which reports it when the day closes. No chat id leaves the process.
  16. This module is imported by the contract at import time: stdlib only at module level.
  17. """
  18. from __future__ import annotations
  19. import asyncio
  20. import logging
  21. import threading
  22. import time
  23. from collections import OrderedDict
  24. from datetime import UTC, datetime
  25. from pathlib import Path
  26. from typing import Any, Callable
  27. logger = logging.getLogger(__name__)
  28. BUSY_ARRIVAL_METRIC = "hermes.gateway.busy_arrival.count"
  29. BUSY_ARRIVAL_DISPOSITIONS = frozenset({"queued", "steered", "interrupted", "merged", "dropped", "other"})
  30. BUSY_WAIT_BUCKETS = frozenset({"lt_5s", "5s_to_30s", "30s_to_2m", "2m_to_10m", "gte_10m", "never"})
  31. BUSY_PEAK_FIELD = "gateway_peak_busy_chats_bucket"
  32. # Terminal FIFO outcomes of a row that reached the agent; any other (``cancelled``, ``rejected``, a
  33. # deleted row) never ran.
  34. _REACHED_OUTCOMES = frozenset({"completed", "interrupted", "failed", "steered", "steering"})
  35. _WAIT_THRESHOLDS = ((5.0, "lt_5s"), (30.0, "5s_to_30s"), (120.0, "30s_to_2m"), (600.0, "2m_to_10m"))
  36. def wait_bucket(seconds: float | None) -> str:
  37. if seconds is None:
  38. return "never"
  39. seconds = max(0.0, seconds)
  40. return next((label for limit, label in _WAIT_THRESHOLDS if seconds < limit), "gte_10m")
  41. def busy_arrival_fields(*, platform: Any, disposition: str, seconds: float | None) -> dict[str, str]:
  42. from .shared_metrics_gateway import _platform
  43. return {
  44. "disposition": disposition if disposition in BUSY_ARRIVAL_DISPOSITIONS else "other",
  45. "platform": _platform(platform),
  46. "wait_bucket": wait_bucket(seconds),
  47. }
  48. # ---- busy arrivals ----
  49. class _Arrival:
  50. __slots__ = ("admission_id", "admitting", "arrived", "disposition", "done", "home", "merged", "parked", "platform")
  51. def __init__(self, platform: Any, home: str | None) -> None:
  52. self.arrived = time.monotonic()
  53. self.platform, self.home = platform, home
  54. self.disposition = "queued" # admitted to the FIFO unless the busy resolution says otherwise
  55. self.admitting = False # reached the runner's admission: from here on, never running = dropped
  56. self.admission_id: str | None = None
  57. self.merged: list[_Arrival] = []
  58. self.done = False
  59. self.parked = False # its row stays queued behind a paused FIFO: not dropped, owed later
  60. # FIFO admission id -> arrival whose row is still queued. Touched only on the gateway loop.
  61. _queued: OrderedDict[str, _Arrival] = OrderedDict()
  62. _QUEUED_MAX = 4096
  63. def _tag(event: Any) -> _Arrival | None:
  64. tag = getattr(event, "_busy_arrival", None)
  65. return tag if isinstance(tag, _Arrival) and not tag.done else None
  66. def stamp_busy_arrival(adapter: Any, event: Any) -> None:
  67. """A message reached its chat while the chat is busy: the adapter's busy dispatch
  68. (``_handle_message_while_active``, re-entered for the same event when a burst ends), or the
  69. runner admitting it to the FIFO while the chat's turn runs. Stamped once. Its home is the
  70. routing identity's runtime profile (pinned at ingress), whose collection setting decides."""
  71. if getattr(event, "_busy_arrival", None) is not None or getattr(event, "internal", False) or event.get_command():
  72. return
  73. try:
  74. from gateway.session_identity import identity_of
  75. from .shared_metrics_gateway import _source_platform
  76. identity = identity_of(event.source)
  77. home = str(identity.runtime_home) if identity is not None else None
  78. event._busy_arrival = _Arrival(_source_platform(adapter, getattr(event.source, "chat_id", None)), home)
  79. except Exception:
  80. logger.debug("Busy arrival not stamped", exc_info=True)
  81. def note_busy_merged(head: Any, event: Any) -> None:
  82. """``event`` was folded into the held burst ``head``: its fate is the head's."""
  83. tag, head_tag = _tag(event), _tag(head)
  84. if tag is not None and head_tag is not None:
  85. tag.done = True
  86. head_tag.merged.append(tag)
  87. def note_busy_outcome(event: Any, outcome: Any) -> None:
  88. """The busy resolution for a follow-up (``_BusySteerOutcome``): steered into the running turn
  89. (steer or redirect) reaches the agent now; a hard interrupt ends the turn so it runs next."""
  90. tag = _tag(event)
  91. if tag is None:
  92. return
  93. _bind_home(tag)
  94. if outcome.steered or outcome.redirected:
  95. _resolve(tag, "steered", time.monotonic())
  96. elif outcome.effective_mode == "interrupt":
  97. tag.disposition = "interrupted"
  98. def note_busy_admitting(event: Any) -> None:
  99. """The runner is admitting the message to the routed profile's FIFO (in that profile's scope)."""
  100. tag = _tag(event)
  101. if tag is not None:
  102. tag.admitting = True
  103. _bind_home(tag)
  104. def _bind_home(tag: _Arrival) -> None:
  105. from hermes_constants import get_hermes_home
  106. tag.home = str(get_hermes_home())
  107. def note_busy_admitted(event: Any, admission_id: str, status: str, outcome: Any = None) -> None:
  108. """The row is committed; ``status`` (and a terminal row's ``outcome``) as read right after. A
  109. drain that already claimed it ran it; a row already cancelled (/stop, /new, a pre-claim refusal)
  110. or deleted with its chat (``outcome`` None) never did."""
  111. tag = _tag(event)
  112. if tag is None:
  113. return
  114. if status == "queued":
  115. tag.admission_id = str(admission_id)
  116. _queued[tag.admission_id] = tag
  117. while len(_queued) > _QUEUED_MAX:
  118. _queued.popitem(last=False)[1].done = True
  119. elif status in ("started", "unknown") or (status == "terminal" and outcome in _REACHED_OUTCOMES):
  120. _resolve(tag, tag.disposition, time.monotonic()) # ``unknown``: it started, then its turn was lost
  121. elif status == "terminal":
  122. _resolve(tag, "dropped", None)
  123. def busy_row_settled(admission_id: Any, outcome: str) -> None:
  124. """A queued row left the FIFO without a turn of its own (``SessionAuthority._settle_cancelled``):
  125. cancelled (/stop, /new, a refused head, an edit) never ran; a steered transfer reached the agent."""
  126. tag = _queued.pop(str(admission_id), None)
  127. if tag is None or tag.done:
  128. return
  129. if outcome == "steered":
  130. _resolve(tag, "steered", time.monotonic())
  131. else:
  132. _resolve(tag, "dropped", None)
  133. def note_busy_parked(event: Any) -> None:
  134. """The handler is answering with a pause notice (``runtime_draining``, ``session_busy``,
  135. ``unknown_execution``) while the message's row is still queued: it is saved, not dropped. It
  136. counts only if this process later runs that row (``busy_turn_started``); a restart owes it to
  137. the next process, which has no record of the arrival."""
  138. tag = _tag(event)
  139. if tag is not None:
  140. tag.parked = True
  141. def busy_turn_started(admission_id: Any) -> None:
  142. """The FIFO claimed this row and is running it: a queued busy arrival reached the agent."""
  143. tag = _queued.pop(str(admission_id), None)
  144. if tag is not None and not tag.done:
  145. _resolve(tag, tag.disposition, time.monotonic())
  146. def settle_busy_arrival(event: Any, *, discarded: bool = False) -> None:
  147. """The busy handler for ``event`` returned (or its held burst was dropped by /stop, /new):
  148. an arrival that reached admission and never ran was dropped. One stopped by a restart is not
  149. counted: its committed row outlives this process. A parked one stays owed (``note_busy_parked``)."""
  150. tag = _tag(event)
  151. if tag is None or (tag.parked and not discarded):
  152. return
  153. if tag.admission_id is not None:
  154. _queued.pop(tag.admission_id, None)
  155. if not discarded and _cancelling():
  156. tag.done = True
  157. elif discarded or tag.admitting:
  158. _resolve(tag, "dropped", None)
  159. else:
  160. tag.done = True # answered before admission (approval reply, plugin consume, refused sender)
  161. def _cancelling() -> bool:
  162. try:
  163. task = asyncio.current_task()
  164. except RuntimeError: # no running loop
  165. return False
  166. return task is not None and task.cancelling() > 0
  167. def _resolve(tag: _Arrival, disposition: str, reached: float | None) -> None:
  168. rows = [(tag, disposition)] + [(merged, "merged") for merged in tag.merged]
  169. for arrival, kind in rows:
  170. arrival.done = True
  171. seconds = None if reached is None else reached - arrival.arrived
  172. from .shared_metrics_gateway import _in_home, _submit
  173. _submit(_in_home, tag.home, _emit_busy_arrival, arrival.platform, kind, seconds)
  174. def _emit_busy_arrival(platform: Any, disposition: str, seconds: float | None) -> None:
  175. from .shared_metrics_events import _emit
  176. _emit(BUSY_ARRIVAL_METRIC, busy_arrival_fields, platform=platform, disposition=disposition, seconds=seconds)
  177. # ---- peak of chats with a running turn, per owning profile and UTC day ----
  178. _peak_lock = threading.Lock()
  179. _running: dict[str, str] = {} # session key -> owning profile home (resolved)
  180. _peaks: dict[str, dict[str, int]] = {} # home -> UTC day -> most chats running at once that day
  181. _PEAK_DAYS = 8 # a closing day's peak is kept until the profile's next interaction saves it
  182. _resolved: dict[str, str] = {}
  183. _now: Callable[[], float] = time.time # test / live-probe seam
  184. def _day(epoch: float) -> str:
  185. return datetime.fromtimestamp(epoch, tz=UTC).date().isoformat()
  186. def _home_key(home: Any) -> str:
  187. raw = str(home)
  188. if raw not in _resolved:
  189. try:
  190. _resolved[raw] = str(Path(raw).resolve())
  191. except OSError:
  192. _resolved[raw] = raw
  193. return _resolved[raw]
  194. def note_running_chat_owner(runner: Any, session_key: str, source: Any) -> None:
  195. """``session_key`` is being claimed outside its profile's scope (the startup auto-resume loop):
  196. count it toward the profile that owns the session, resolved the way its turn will resolve it."""
  197. try:
  198. home = _home_key(runner._resolve_profile_home_for_source(source))
  199. with _peak_lock:
  200. _running[str(session_key)] = home
  201. except Exception:
  202. logger.debug("Running-chat owner not resolved", exc_info=True)
  203. def observe_running_chats(session_keys: Any) -> None:
  204. """The runner's running-turn set at a turn boundary (claim or release). A newly running chat
  205. belongs to the profile whose scope claimed it."""
  206. try:
  207. keys = {str(key) for key in session_keys}
  208. home = None
  209. with _peak_lock:
  210. for key in [k for k in _running if k not in keys]:
  211. del _running[key]
  212. for key in keys - set(_running):
  213. if home is None:
  214. from hermes_constants import get_hermes_home
  215. home = _home_key(get_hermes_home())
  216. _running[key] = home
  217. counts: dict[str, int] = {}
  218. for owner in _running.values():
  219. counts[owner] = counts.get(owner, 0) + 1
  220. today = _day(_now())
  221. for owner, count in counts.items():
  222. days = _peaks.setdefault(owner, {})
  223. days[today] = max(days.get(today, 0), count)
  224. for stale in sorted(days)[:-_PEAK_DAYS]:
  225. del days[stale]
  226. except Exception:
  227. logger.debug("Running-chat peak not updated", exc_info=True)
  228. def day_peak(home: Any, day: str) -> int:
  229. """Most chats of ``home``'s profile running at once on UTC ``day``, as seen by this process.
  230. An earlier day keeps its own peak (the day closing after midnight reads it); chats still running
  231. when the day turned also count toward the new one."""
  232. owner = _home_key(home)
  233. with _peak_lock:
  234. running = sum(1 for h in _running.values() if h == owner)
  235. days = dict(_peaks.get(owner, {}))
  236. peak = days.get(day, 0)
  237. return max(peak, running) if not days or day >= max(days) else peak