model_metadata_llamacpp.py 2.4 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364
  1. """llama.cpp context-window probes (sibling of ``model_metadata``)."""
  2. from __future__ import annotations
  3. from itertools import pairwise
  4. from typing import Optional
  5. from urllib.parse import quote
  6. def _json_object(resp) -> dict:
  7. """The response body when it is a JSON object, else an empty dict."""
  8. try:
  9. body = resp.json()
  10. except ValueError:
  11. return {}
  12. return body if isinstance(body, dict) else {}
  13. def llamacpp_context(client, server_url: str, model: str) -> Optional[int]:
  14. """llama.cpp's RUNTIME n_ctx: /props for a loaded model, else the ``--ctx-size`` the router will
  15. launch it with (its /models entry), so a lazily-loaded model doesn't fall to a family catch-all.
  16. The /props read passes autoload=false: the router loads any model a request names, and a context
  17. probe must not undo an Eject or an idle unload."""
  18. import httpx
  19. from agent.model_metadata import _positive_int
  20. for props_path in (f"/props?model={quote(model)}&autoload=false", "/props"):
  21. try:
  22. resp = client.get(f"{server_url}{props_path}")
  23. except httpx.HTTPError:
  24. return None
  25. if resp.status_code != 200:
  26. continue
  27. settings = _json_object(resp).get("default_generation_settings")
  28. n_ctx = _positive_int(settings.get("n_ctx")) if isinstance(settings, dict) else None
  29. if n_ctx is not None:
  30. return n_ctx
  31. return router_launch_ctx(client, server_url, model)
  32. def router_launch_ctx(client, server_url: str, model: str) -> Optional[int]:
  33. """``--ctx-size`` from the router's launch args for *model* (GET /models), or None."""
  34. import httpx
  35. from agent.model_metadata import _model_id_matches
  36. try:
  37. resp = client.get(f"{server_url}/models")
  38. except httpx.HTTPError:
  39. return None
  40. if resp.status_code != 200:
  41. return None
  42. entries = _json_object(resp).get("data")
  43. for entry in entries if isinstance(entries, list) else []:
  44. if not isinstance(entry, dict) or not _model_id_matches(str(entry.get("id", "")), model):
  45. continue
  46. status = entry.get("status")
  47. raw_args = status.get("args") if isinstance(status, dict) else None
  48. args = [str(a) for a in raw_args] if isinstance(raw_args, list) else []
  49. for flag, value in pairwise(args):
  50. if flag in ("--ctx-size", "-c") and value.isdigit() and int(value) > 0:
  51. return int(value)
  52. return None