dialogue_usage.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200
  1. """Opt-in, payload-free observations of actual HTTP model attempts.
  2. Passes request/response bytes unchanged. No retries, usage estimates, or network
  3. calls are introduced. Unsupported transports remain explicitly incomplete.
  4. """
  5. from contextvars import ContextVar
  6. from contextlib import contextmanager
  7. from functools import wraps
  8. import json
  9. import time
  10. import uuid
  11. _scope = ContextVar('yaovi_dialogue_usage_scope', default=None)
  12. _installed = False
  13. _default = None
  14. _emit = None
  15. @contextmanager
  16. def scope(**values):
  17. token = _scope.set({**(_default or {}), **(_scope.get() or {}), **values})
  18. try:
  19. yield
  20. finally:
  21. _scope.reset(token)
  22. class Attempt:
  23. def __init__(self, request):
  24. self.context = dict(_scope.get() or _default or {})
  25. self.call_id = str(uuid.uuid4())
  26. self.done = False
  27. self.buffer = b''
  28. self.usage = {}
  29. self.model = 'unknown'
  30. self.status_code = None
  31. self.provider = self.context.get('provider', 'unknown')
  32. self.sse = False
  33. self.incomplete = False
  34. self.terminal_received = False
  35. # Only read an already buffered model request; do not touch streams.
  36. try:
  37. payload = json.loads(request.content)
  38. self.model = str(payload.get('model', 'unknown'))
  39. except Exception:
  40. pass
  41. self.event('started')
  42. def event(self, status):
  43. u = self.usage
  44. i = u.get('prompt_tokens', u.get('input_tokens'))
  45. o = u.get('completion_tokens', u.get('output_tokens'))
  46. valid = lambda n: isinstance(n, int) and not isinstance(n, bool) and n >= 0
  47. obs = dict(version=1, eventId=str(uuid.uuid4()), callId=self.call_id,
  48. runId=self.context.get('rootRunId', 'unknown'), role=self.context.get('role', 'unknown'),
  49. attempt=1, provider=self.provider, model=self.model, status=status,
  50. timestampMs=int(time.time() * 1000), source='provider' if valid(i) and valid(o) else 'unknown',
  51. inputTokens=i if valid(i) else None, outputTokens=o if valid(o) else None,
  52. inputIncludesCache=True, outputIncludesReasoning=True)
  53. if self.context.get('parentCallId'):
  54. obs['parentCallId'] = self.context['parentCallId']
  55. read = (u.get('prompt_tokens_details') or u.get('input_tokens_details') or {}).get('cached_tokens')
  56. reasoning = (u.get('completion_tokens_details') or u.get('output_tokens_details') or {}).get('reasoning_tokens')
  57. obs.update(cacheReadTokens=read if valid(read) else None, reasoningTokens=reasoning if valid(reasoning) else None)
  58. # Anthropic input excludes separately billed cache buckets.
  59. if 'cache_read_input_tokens' in u or 'cache_creation_input_tokens' in u:
  60. obs.update(inputIncludesCache=False, cacheReadTokens=u.get('cache_read_input_tokens'), cacheWriteTokens=u.get('cache_creation_input_tokens'))
  61. if self.incomplete:
  62. obs.update(inputTokens=None, outputTokens=None, source='unknown')
  63. try:
  64. _emit('usage.observation', observation=obs)
  65. except Exception:
  66. pass # Observability cannot change execution behavior.
  67. def parse(self, data):
  68. if data == b'[DONE]':
  69. self.terminal_received = True
  70. return
  71. try:
  72. value = json.loads(data)
  73. if value.get('type') in ('message_stop', 'response.completed'):
  74. self.terminal_received = True
  75. for item in [value, value.get('response', {}), value.get('message', {})]:
  76. if isinstance(item, dict) and isinstance(item.get('usage'), dict):
  77. self.usage.update(item['usage'])
  78. if self.model == 'unknown' and isinstance(item, dict) and item.get('model'):
  79. self.model = str(item['model'])
  80. except Exception:
  81. pass
  82. def feed(self, data):
  83. self.buffer += data
  84. if self.sse:
  85. while b'\n' in self.buffer:
  86. line, self.buffer = self.buffer.split(b'\n', 1)
  87. if line.startswith(b'data:'):
  88. self.parse(line[5:].strip())
  89. if len(self.buffer) > 2 * 1024 * 1024:
  90. self.buffer = b''
  91. self.incomplete = True
  92. def finish(self, status='completed'):
  93. if self.done:
  94. return
  95. self.done = True
  96. if self.buffer:
  97. self.parse(self.buffer[5:].strip() if self.buffer.startswith(b'data:') else self.buffer)
  98. self.event('failed' if self.status_code and self.status_code >= 400 else 'completed' if self.terminal_received else status)
  99. def install(context, emit, agent_class=None):
  100. """Install only with an explicit isolated observation request; return restore."""
  101. global _installed, _default, _emit
  102. if not context:
  103. return lambda: None
  104. if _installed:
  105. raise RuntimeError('Usage observer already installed')
  106. import httpx
  107. _installed, _default, _emit = True, dict(context), emit
  108. sync_send, async_send = httpx.Client.send, httpx.AsyncClient.send
  109. def observed(request):
  110. return request.method == 'POST' and any(request.url.path.rstrip('/').endswith(x) for x in ('/chat/completions', '/responses', '/messages'))
  111. class SyncStream(httpx.SyncByteStream):
  112. def __init__(self, stream, attempt): self.stream, self.attempt = stream, attempt
  113. def __iter__(self):
  114. try:
  115. for data in self.stream:
  116. self.attempt.feed(data)
  117. yield data
  118. self.attempt.finish()
  119. except BaseException:
  120. self.attempt.finish('failed')
  121. raise
  122. def close(self):
  123. try: self.stream.close()
  124. finally: self.attempt.finish('cancelled')
  125. class AsyncStream(httpx.AsyncByteStream):
  126. def __init__(self, stream, attempt): self.stream, self.attempt = stream, attempt
  127. async def __aiter__(self):
  128. try:
  129. async for data in self.stream:
  130. self.attempt.feed(data)
  131. yield data
  132. self.attempt.finish()
  133. except BaseException:
  134. self.attempt.finish('failed')
  135. raise
  136. async def aclose(self):
  137. try: await self.stream.aclose()
  138. finally: self.attempt.finish('cancelled')
  139. def attach(response, attempt, asynchronous=False):
  140. attempt.status_code = response.status_code
  141. attempt.sse = 'text/event-stream' in response.headers.get('content-type', '')
  142. if response.is_stream_consumed:
  143. attempt.feed(response.content)
  144. attempt.finish()
  145. else:
  146. response.stream = (AsyncStream if asynchronous else SyncStream)(response.stream, attempt)
  147. return response
  148. @wraps(sync_send)
  149. def send(self, request, **kwargs):
  150. if not observed(request): return sync_send(self, request, **kwargs)
  151. attempt = Attempt(request)
  152. try: return attach(sync_send(self, request, **kwargs), attempt)
  153. except BaseException:
  154. attempt.finish('failed')
  155. raise
  156. @wraps(async_send)
  157. async def asend(self, request, **kwargs):
  158. if not observed(request): return await async_send(self, request, **kwargs)
  159. attempt = Attempt(request)
  160. try: return attach(await async_send(self, request, **kwargs), attempt, True)
  161. except BaseException:
  162. attempt.finish('failed')
  163. raise
  164. httpx.Client.send, httpx.AsyncClient.send = send, asend
  165. original_run = getattr(agent_class, 'run_conversation', None)
  166. if original_run:
  167. @wraps(original_run)
  168. def run(self, *args, **kwargs):
  169. role = 'child' if getattr(self, 'platform', '') == 'subagent' else context.get('role', 'foreground')
  170. with scope(role=role):
  171. return original_run(self, *args, **kwargs)
  172. agent_class.run_conversation = run
  173. from agent import auxiliary_client
  174. original_aux, original_async_aux = auxiliary_client.call_llm, auxiliary_client.async_call_llm
  175. @wraps(original_aux)
  176. def aux(*args, **kwargs):
  177. with scope(role='auxiliary'): return original_aux(*args, **kwargs)
  178. @wraps(original_async_aux)
  179. async def async_aux(*args, **kwargs):
  180. with scope(role='auxiliary'): return await original_async_aux(*args, **kwargs)
  181. auxiliary_client.call_llm, auxiliary_client.async_call_llm = aux, async_aux
  182. def restore():
  183. global _installed, _default, _emit
  184. httpx.Client.send, httpx.AsyncClient.send = sync_send, async_send
  185. auxiliary_client.call_llm, auxiliary_client.async_call_llm = original_aux, original_async_aux
  186. if original_run: agent_class.run_conversation = original_run
  187. _installed, _default, _emit = False, None, None
  188. return restore