load_progress.py 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156
  1. """Live model-load progress from the managed llama-server router.
  2. Children emit per-tensor progress ({stages, current, value}) which the router relays ONLY over its
  3. /models/sse stream — GET /models carries just the coarse status. The watcher starts on first call,
  4. reconnects with backoff (the router bounces on download/eject), and never raises into callers: no
  5. router, no state file, or no SSE support (older engines) all read as "nothing loading".
  6. """
  7. from __future__ import annotations
  8. from contextlib import suppress
  9. import json
  10. import logging
  11. import threading
  12. import time
  13. import urllib.request
  14. logger = logging.getLogger(__name__)
  15. _TEXT_STAGE_SHARE = 0.85 # composite range share for the text model
  16. _RECONNECT_DELAY_S = 3.0
  17. _STALE_ENTRY_TTL_S = 120.0 # a loading entry with no events this long is dead
  18. _LOAD_EVENTS = ("status_change", "model_status")
  19. _lock = threading.Lock()
  20. _watcher: threading.Thread | None = None
  21. _snapshot: dict[str, dict] = {}
  22. def _pct(x: float) -> int:
  23. return max(0, min(100, round(x * 100)))
  24. def _composite_percent(stages: list[str], current: str, value: float) -> int:
  25. """Map (stage, in-stage value) onto one 0-100 range, text-heavy."""
  26. if not stages or current not in stages or len(stages) == 1:
  27. return _pct(value)
  28. extras = [s for s in stages if s != "text_model"]
  29. extra_share = (1.0 - _TEXT_STAGE_SHARE) / len(extras) if extras else 0.0
  30. offset = 0.0
  31. for stage in stages:
  32. share = _TEXT_STAGE_SHARE if stage == "text_model" else extra_share
  33. if stage == current:
  34. return _pct(offset + share * value)
  35. offset += share
  36. return _pct(value)
  37. def _endpoint() -> tuple[str, str] | None:
  38. """(base_root, api_key) of the managed router via the ownership-guarded reader, or None."""
  39. from hermes_cli.local_runtime.endpoint import managed_root
  40. return managed_root()
  41. def _apply_event(model: str, event: str, data: dict) -> None:
  42. with _lock:
  43. status = str(data.get("status", ""))
  44. if event in _LOAD_EVENTS and status == "loading":
  45. progress = data.get("progress") or {}
  46. stages = [str(s) for s in (progress.get("stages") or [])]
  47. current = str(progress.get("current", ""))
  48. value = progress.get("value")
  49. entry = _snapshot.setdefault(model, {"stage": "", "value": 0.0, "percent": 0, "ts": 0.0})
  50. entry["ts"] = time.monotonic()
  51. if current and isinstance(value, (int, float)):
  52. entry["stage"] = current
  53. entry["value"] = float(value)
  54. entry["percent"] = _composite_percent(stages, current, float(value))
  55. elif event in (*_LOAD_EVENTS, "model_remove") and status != "loading":
  56. # Any terminal status (loaded/unloaded/failed) ends the load.
  57. _snapshot.pop(model, None)
  58. def _clear_snapshot() -> None:
  59. with _lock:
  60. _snapshot.clear()
  61. def _watch() -> None:
  62. while True:
  63. endpoint = _endpoint()
  64. if endpoint is None:
  65. _clear_snapshot()
  66. time.sleep(_RECONNECT_DELAY_S)
  67. continue
  68. base, key = endpoint
  69. try:
  70. req = urllib.request.Request(f"{base}/models/sse", headers={
  71. "Authorization": f"Bearer {key}", "Accept": "text/event-stream"})
  72. with urllib.request.urlopen(req, timeout=60) as r:
  73. buf = b""
  74. while True:
  75. chunk = r.read1(4096) if hasattr(r, "read1") else r.read(4096)
  76. if not chunk:
  77. break
  78. buf += chunk
  79. while b"\n" in buf:
  80. line, buf = buf.split(b"\n", 1)
  81. text = line.decode("utf-8", "replace").strip()
  82. if not text.startswith("data:"):
  83. continue
  84. with suppress(json.JSONDecodeError, TypeError):
  85. msg = json.loads(text[5:].strip())
  86. _apply_event(str(msg.get("model", "")), str(msg.get("event", "")),
  87. msg.get("data") or {})
  88. except Exception as exc:
  89. logger.debug("load-progress SSE reconnecting: %s", exc)
  90. # Stream ended (router bounce, timeout, error): loading entries from the dead connection
  91. # are unverifiable — drop rather than freeze.
  92. _clear_snapshot()
  93. time.sleep(_RECONNECT_DELAY_S)
  94. def _ensure_watcher() -> None:
  95. global _watcher
  96. with _lock:
  97. if _watcher is None or not _watcher.is_alive():
  98. _watcher = threading.Thread(target=_watch, daemon=True, name="llamacpp-load-progress")
  99. _watcher.start()
  100. def get_loading_progress() -> dict[str, dict]:
  101. """{model_id: {"stage", "value", "percent"}} for models loading right now. Empty when
  102. nothing is loading (or nothing is knowable)."""
  103. _ensure_watcher()
  104. now = time.monotonic()
  105. with _lock:
  106. return {m: {"stage": e["stage"], "value": e["value"], "percent": e["percent"]}
  107. for m, e in _snapshot.items() if now - e["ts"] < _STALE_ENTRY_TTL_S}
  108. def get_prefill_progress(model: str) -> dict | None:
  109. """{"processed": tokens} while the managed server is prompt-processing for ``model``, or None
  110. (idle, decoding, unreachable, or foreign server).
  111. /slots exposes ``n_prompt_tokens_processed`` but no total, so callers supply the denominator.
  112. The busiest processing slot wins: a parallel small request freezes its counter during decode
  113. while a live prefill keeps climbing. One HTTP call per poll; every failure reads as "no
  114. prefill" — garnish, never load-bearing.
  115. """
  116. ep = _endpoint()
  117. if ep is None:
  118. return None
  119. try:
  120. from hermes_cli.local_runtime.endpoint import child_query, managed_get_json
  121. slots = managed_get_json(*ep, f"/slots?{child_query(model)}", timeout_s=2)
  122. except Exception:
  123. return None
  124. best = 0
  125. for slot in slots if isinstance(slots, list) else []:
  126. with suppress(TypeError, ValueError):
  127. if slot.get("is_processing"):
  128. best = max(best, int(slot.get("n_prompt_tokens_processed") or 0))
  129. return {"processed": best} if best > 0 else None