devices.py 4.0 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788
  1. """Ask the installed llama.cpp build which accelerators it sees, in a disposable process.
  2. ggml's device type is the engine's own integrated-vs-discrete verdict (Vulkan reads it from
  3. VkPhysicalDeviceType), and llama.cpp places layers on discrete devices whenever one exists, so the
  4. budget can follow the same rule. Loading a GPU driver can crash or hang the caller, hence a child
  5. with a deadline that imports only the standard library (``-I``). Run directly:
  6. ``python -I devices.py <engine dir> <backend>`` prints the device list as JSON.
  7. """
  8. from __future__ import annotations
  9. import ctypes
  10. import json
  11. import os
  12. from pathlib import Path
  13. import subprocess
  14. import sys
  15. # enum ggml_backend_dev_type (ggml-backend.h): CPU, GPU, IGPU, ACCEL.
  16. GGML_DEVICE_GPU = 1
  17. GGML_DEVICE_IGPU = 2
  18. def probe_devices(engine_dir: Path, backend: str) -> list[dict]:
  19. """GPU/iGPU devices the ``backend`` library in ``engine_dir`` registers; [] on any failure."""
  20. from hermes_cli._subprocess_compat import windows_hide_flags
  21. try:
  22. out = subprocess.run(
  23. [sys.executable, "-I", str(Path(__file__).resolve()), str(engine_dir), backend],
  24. stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8",
  25. errors="replace", timeout=20, creationflags=windows_hide_flags(), check=False)
  26. devices = json.loads(out.stdout) if out.returncode == 0 else []
  27. except (OSError, ValueError, subprocess.TimeoutExpired):
  28. return []
  29. if not isinstance(devices, list):
  30. return []
  31. return [d for d in devices if isinstance(d, dict)
  32. and d.get("type") in (GGML_DEVICE_GPU, GGML_DEVICE_IGPU)
  33. and isinstance(d.get("total"), int) and d["total"] > 0
  34. and isinstance(d.get("description"), str)]
  35. def _library(directory: Path, name: str) -> Path:
  36. return directory / (f"{name}.dll" if os.name == "nt" else f"lib{name}.so")
  37. def _read_devices(directory: Path, backend: str) -> list[dict]:
  38. dll_directory = os.add_dll_directory(str(directory)) if os.name == "nt" else None
  39. try:
  40. base = ctypes.CDLL(str(_library(directory, "ggml-base")), mode=ctypes.RTLD_GLOBAL)
  41. core = ctypes.CDLL(str(_library(directory, "ggml")), mode=ctypes.RTLD_GLOBAL)
  42. core.ggml_backend_load.argtypes = [ctypes.c_char_p]
  43. core.ggml_backend_load.restype = ctypes.c_void_p
  44. registry = core.ggml_backend_load(os.fsencode(_library(directory, f"ggml-{backend}")))
  45. if not registry:
  46. return []
  47. signatures = {
  48. "ggml_backend_reg_dev_count": ([ctypes.c_void_p], ctypes.c_size_t),
  49. "ggml_backend_reg_dev_get": ([ctypes.c_void_p, ctypes.c_size_t], ctypes.c_void_p),
  50. "ggml_backend_dev_description": ([ctypes.c_void_p], ctypes.c_char_p),
  51. "ggml_backend_dev_type": ([ctypes.c_void_p], ctypes.c_int),
  52. "ggml_backend_dev_memory": ([ctypes.c_void_p, ctypes.POINTER(ctypes.c_size_t),
  53. ctypes.POINTER(ctypes.c_size_t)], None),
  54. }
  55. for name, (arguments, result) in signatures.items():
  56. function = getattr(base, name)
  57. function.argtypes, function.restype = arguments, result
  58. devices = []
  59. for index in range(min(base.ggml_backend_reg_dev_count(registry), 64)):
  60. device = base.ggml_backend_reg_dev_get(registry, index)
  61. if not device:
  62. continue
  63. free, total = ctypes.c_size_t(), ctypes.c_size_t()
  64. base.ggml_backend_dev_memory(device, ctypes.byref(free), ctypes.byref(total))
  65. devices.append({
  66. "description": base.ggml_backend_dev_description(device).decode("utf-8", errors="replace"),
  67. "type": base.ggml_backend_dev_type(device),
  68. "free": min(free.value, total.value), "total": total.value,
  69. })
  70. return devices
  71. finally:
  72. if dll_directory is not None:
  73. dll_directory.close()
  74. if __name__ == "__main__":
  75. print(json.dumps(_read_devices(Path(sys.argv[1]), sys.argv[2])))