local_prompt_progress.py 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123
  1. """Prompt-reading progress from llama-server.
  2. llama-server reads a long prompt for minutes before it writes the first token. Asked with
  3. ``return_progress``, it streams a progress chunk after each batch it reads. Hermes uses those
  4. chunks as signs of life: the stall timers stay quiet while the server works, and the router
  5. notices a client hang-up within seconds (it only checks the client socket when it writes).
  6. Compression also leaves the time the server spends reading the prompt out of its total time
  7. limit (``conversation_compression_deadline``).
  8. """
  9. from __future__ import annotations
  10. import contextlib
  11. import threading
  12. import time
  13. from typing import Any, Callable, Optional
  14. _reading = threading.local()
  15. _CONTENT_FIELDS = ("content", "tool_calls", "reasoning_content", "reasoning", "refusal")
  16. def with_prompt_progress(kwargs: dict[str, Any]) -> dict[str, Any]:
  17. """Copy of chat-completions *kwargs* that asks llama-server for prompt progress chunks."""
  18. extra_body = kwargs.get("extra_body")
  19. merged = dict(extra_body) if isinstance(extra_body, dict) else {}
  20. merged.setdefault("return_progress", True)
  21. return {**kwargs, "extra_body": merged}
  22. def prompt_progress(chunk: Any) -> Optional[dict]:
  23. """The ``prompt_progress`` payload of a progress-only chunk, else None.
  24. A progress chunk carries an empty assistant delta. A chunk with text, tool calls, reasoning or a
  25. finish reason is a real chunk even if a server attaches progress to it.
  26. """
  27. extra = getattr(chunk, "model_extra", None)
  28. progress = extra.get("prompt_progress") if isinstance(extra, dict) else None
  29. if progress is None and isinstance(chunk, dict):
  30. progress = chunk.get("prompt_progress")
  31. if not isinstance(progress, dict):
  32. return None
  33. choices = getattr(chunk, "choices", None)
  34. if choices is None and isinstance(chunk, dict):
  35. choices = chunk.get("choices")
  36. for choice in choices or []:
  37. finish = choice.get("finish_reason") if isinstance(choice, dict) else getattr(choice, "finish_reason", None)
  38. delta = choice.get("delta") if isinstance(choice, dict) else getattr(choice, "delta", None)
  39. if finish:
  40. return None
  41. for field in _CONTENT_FIELDS:
  42. value = delta.get(field) if isinstance(delta, dict) else getattr(delta, field, None)
  43. if value:
  44. return None
  45. return progress
  46. def _count(progress: dict, key: str) -> Optional[int]:
  47. try:
  48. return int(progress[key])
  49. except (KeyError, TypeError, ValueError):
  50. return None
  51. def prompt_progress_percent(progress: dict) -> Optional[int]:
  52. """Share of the prompt the server has read, 0-100, or None without a usable total."""
  53. try:
  54. total = int(progress.get("total") or 0)
  55. processed = int(progress.get("processed") or 0)
  56. except (TypeError, ValueError):
  57. return None
  58. if total <= 0:
  59. return None
  60. return max(0, min(100, round(processed / total * 100)))
  61. @contextlib.contextmanager
  62. def prompt_reading_hook(hook: Optional[Callable[[float], None]]):
  63. """Install ``hook(seconds)`` for this thread; it hears each stretch of prompt reading."""
  64. previous = getattr(_reading, "hook", None)
  65. _reading.hook = hook
  66. try:
  67. yield
  68. finally:
  69. _reading.hook = previous
  70. def current_prompt_reading_hook() -> Optional[Callable[[float], None]]:
  71. """This thread's prompt-reading hook (carried into provider daemon threads)."""
  72. return getattr(_reading, "hook", None)
  73. class PromptReadingClock:
  74. """Time llama-server spends reading the prompt.
  75. Counted per progress chunk, and only when the server reports more tokens read than before: a
  76. server that keeps reporting without reading anything gets no extra time. Each stretch is the
  77. smaller of the wall time since the previous chunk and the server's own reading time, so time
  78. spent queued or loading the model before the reading starts doesn't count.
  79. """
  80. def __init__(self, started: float):
  81. self._mark = started
  82. self._processed = -1
  83. self._server_ms = 0
  84. self.seconds = 0.0
  85. def tick(self, progress: dict) -> bool:
  86. """Count the reading up to this progress chunk; False when it reports nothing new read."""
  87. now = time.monotonic()
  88. wall, self._mark = now - self._mark, now
  89. processed, server_ms = _count(progress, "processed"), _count(progress, "time_ms")
  90. if processed is None or processed <= self._processed:
  91. return False
  92. server = wall if server_ms is None else (server_ms - self._server_ms) / 1000
  93. self._processed, self._server_ms = processed, self._server_ms if server_ms is None else server_ms
  94. spent = max(0.0, min(wall, server))
  95. self.seconds += spent
  96. hook = current_prompt_reading_hook()
  97. if hook is not None:
  98. hook(spent)
  99. return True