hardware.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591
  1. """Live hardware budget probe.
  2. Budget-source rule: discrete cards may trust the device query (measured honest within rounding);
  3. unified-memory devices must budget from OS free physical memory minus headroom — their device
  4. queries have been observed off by 3x in both directions. Every probe here must work under a
  5. stripped PATH — gateway and service sessions don't inherit the interactive environment.
  6. """
  7. from __future__ import annotations
  8. from contextlib import suppress
  9. import csv
  10. import logging
  11. import os
  12. import re
  13. import shutil
  14. import subprocess
  15. import sys
  16. import time
  17. from dataclasses import replace
  18. from pathlib import Path
  19. from hermes_cli.local_runtime.devices import GGML_DEVICE_GPU
  20. from hermes_cli.local_runtime.estimator import HardwareBudget
  21. from hermes_platform.host.facts import gpu_class
  22. logger = logging.getLogger(__name__)
  23. _GIB = 1 << 30
  24. # Reserve carved off the card before any grant: the desktop's own co-residents (compositor,
  25. # browser, Electron) measure ~2-2.5 GiB, and a window granted into that space demotes silently
  26. # under WDDM. 7% covers big cards; the 2 GiB floor is what a 512 MiB floor failed to cover (a
  27. # 221K grant measured 31.9/32.6 GiB with the desktop running — 'fits' by the math, demoted in
  28. # reality). Small cards give up window to this; spill mode is their path to big models.
  29. _MARGIN_FLOOR = 2 << 30
  30. _MARGIN_FRACTION = 0.09
  31. # Room left on top of what other programs hold when a model launches: a browser tab or a chat app
  32. # can take another GiB between launch and the next request.
  33. _LAUNCH_HEADROOM = 1 << 30
  34. # UMA headroom: on unified-memory machines the model shares physical memory with the OS and every
  35. # app, so budget from RAM minus this fraction.
  36. _UMA_HEADROOM_FRACTION = 0.20
  37. # llama-server's prompt cache: its own default size, and the smallest worth keeping. The cache holds
  38. # conversations the server switched away from (a background job, an earlier chat); without it each
  39. # switch re-reads the whole conversation.
  40. _PROMPT_CACHE_MAX = 8 << 30
  41. _PROMPT_CACHE_MIN = 512 << 20
  42. # Engine-fallback gates for the unified-pool quirk — BOTH must hold, and no discrete card can
  43. # meet either: (1) the allocator's pool exceeds the smi report by well past rounding/ECC slack
  44. # (discrete cards agree within ~2%; carve-out disagreement runs to whole multiples), and (2) the
  45. # pool is system-RAM-sized. The driver's INTEGRATED attribute, when readable, bypasses both gates
  46. # in whichever direction it points.
  47. _POOL_DISAGREEMENT_FACTOR = 1.5
  48. _POOL_RAM_FRACTION = 0.75
  49. # cuDeviceGetAttribute enum: device is integrated with host memory.
  50. _CU_DEVICE_ATTRIBUTE_INTEGRATED = 18
  51. _NVIDIA_VENDOR_ID = 0x10DE
  52. # Engines whose devices the ggml probe reports; the CPU, CUDA and Metal builds answer elsewhere.
  53. _GPU_ENGINE_BACKENDS = ("vulkan", "hip")
  54. # One probe per process once a device answers (silicon doesn't change); a miss retries after this
  55. # long so a runtime installed mid-session gets picked up by the engine fallback.
  56. _POOL_NEGATIVE_TTL_S = 60.0
  57. _pool_probe_cache: tuple[float, tuple[int, bool | None] | None] | None = None
  58. # ' CUDA0: NVIDIA Example Device (1234-core Example GPU) (46464 MiB, 46284 MiB free)'
  59. # — greedy .* pins the LAST parenthesized group, so device names with parentheses parse.
  60. _DEVICE_LINE_RE = re.compile(r"CUDA\d+:.*\((\d+)\s*MiB,\s*\d+\s*MiB free\)\s*$")
  61. def _stdout(*argv: str) -> str:
  62. return subprocess.run(
  63. list(argv), capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=5,
  64. check=False,
  65. ).stdout
  66. _LINUX_RAM_KEYS = frozenset({"MemTotal", "MemAvailable", "MemFree"})
  67. def _linux_meminfo_text() -> str | None:
  68. """Raw /proc/meminfo, or None when procfs cannot be read."""
  69. try:
  70. return Path("/proc/meminfo").read_text(encoding="utf-8")
  71. except OSError:
  72. return None
  73. def _linux_ram_from_meminfo(text: str) -> tuple[int, int] | None:
  74. """(total, available) from meminfo text, or None if it cannot be trusted.
  75. MemAvailable includes reclaimable page cache, which ``getconf _AVPHYS_PAGES``
  76. counts as used. Kernels that omit MemAvailable fall back to MemFree. Zero is
  77. a real available value — a missing-field check must not treat it as absent.
  78. """
  79. fields: dict[str, int] = {}
  80. try:
  81. for line in text.splitlines():
  82. name, separator, value = line.partition(":")
  83. if not separator or name not in _LINUX_RAM_KEYS:
  84. continue
  85. parts = value.split()
  86. if len(parts) != 2 or parts[1] != "kB":
  87. continue
  88. fields[name] = int(parts[0]) * 1024
  89. total = fields.get("MemTotal")
  90. # Key presence, not truthiness: MemAvailable 0 must not fall through to MemFree.
  91. if "MemAvailable" in fields:
  92. available = fields["MemAvailable"]
  93. else:
  94. available = fields.get("MemFree")
  95. if (
  96. total is None
  97. or total <= 0
  98. or available is None
  99. or not 0 <= available <= total
  100. ):
  101. return None
  102. return total, available
  103. except (AttributeError, TypeError, ValueError):
  104. return None
  105. def _windows_memory_status():
  106. """GlobalMemoryStatusEx, or None off Windows."""
  107. import ctypes
  108. class MEMORYSTATUSEX(ctypes.Structure):
  109. _fields_ = ([("dwLength", ctypes.c_ulong), ("dwMemoryLoad", ctypes.c_ulong)]
  110. + [(name, ctypes.c_ulonglong) for name in (
  111. "ullTotalPhys", "ullAvailPhys", "ullTotalPageFile", "ullAvailPageFile",
  112. "ullTotalVirtual", "ullAvailVirtual", "ullAvailExtendedVirtual")])
  113. try:
  114. stat = MEMORYSTATUSEX()
  115. stat.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
  116. if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)):
  117. return None
  118. return stat
  119. except (AttributeError, OSError):
  120. return None
  121. def _ram_bytes() -> tuple[int, int]:
  122. """(total, available) physical memory, cross-platform stdlib."""
  123. stat = _windows_memory_status()
  124. if stat is not None:
  125. return stat.ullTotalPhys, stat.ullAvailPhys
  126. try:
  127. if sys.platform == "darwin":
  128. # macOS getconf has no _PHYS_PAGES/_AVPHYS_PAGES (exit 64) — the POSIX branch would
  129. # return (0, 0) and every model would read unavailable. sysctl is the platform truth.
  130. total = int(_stdout("/usr/sbin/sysctl", "-n", "hw.memsize").strip() or 0)
  131. if total <= 0:
  132. return 0, 0
  133. avail = total // 2 # conservative fallback
  134. with suppress(OSError, ValueError):
  135. out = _stdout("/usr/bin/vm_stat")
  136. page_m = re.search(r"page size of (\d+)", out)
  137. page = int(page_m.group(1)) if page_m else 16384
  138. # free + inactive + purgeable ≈ reclaimable-on-demand; the speculative pool is
  139. # dropped by the OS under pressure too.
  140. pages = sum(int(m.group(1)) for key in (
  141. "Pages free", "Pages inactive", "Pages purgeable", "Pages speculative")
  142. if (m := re.search(rf"{key}:\s+(\d+)\.", out)))
  143. if pages > 0:
  144. avail = pages * page
  145. return total, avail
  146. if sys.platform.startswith("linux"):
  147. meminfo = _linux_meminfo_text()
  148. if meminfo is not None:
  149. linux_ram = _linux_ram_from_meminfo(meminfo)
  150. if linux_ram is not None:
  151. return linux_ram
  152. # POSIX
  153. page = int(_stdout("getconf", "PAGE_SIZE") or 4096)
  154. total = int(_stdout("getconf", "_PHYS_PAGES") or 0) * page
  155. avail = total // 2 # conservative when _AVPHYS is unavailable
  156. with suppress(OSError, ValueError):
  157. avail = int(_stdout("getconf", "_AVPHYS_PAGES") or 0) * page or avail
  158. return total, avail
  159. except (OSError, ValueError):
  160. return 0, 0
  161. # nvidia-smi lives at a fixed path under the driver install; PATH presence varies by session type
  162. # (services and gateways often run minimal environments) and by driver generation (the legacy
  163. # NVSMI dir was never on PATH). Cached: the driver doesn't move mid-process.
  164. _smi_path_cache: tuple[str | None] | None = None
  165. def _nvidia_smi_path() -> str | None:
  166. """Absolute path to nvidia-smi, or None. PATH first (respects user overrides), then the
  167. driver's known install locations on Windows and WSL."""
  168. global _smi_path_cache
  169. if _smi_path_cache is not None:
  170. return _smi_path_cache[0]
  171. found = shutil.which("nvidia-smi")
  172. if found is None and os.name == "nt":
  173. windir = os.environ.get("SystemRoot", r"C:\Windows")
  174. candidates = (
  175. # DCH drivers (every modern install) place it in System32.
  176. Path(windir) / "System32" / "nvidia-smi.exe",
  177. # Legacy standalone drivers used NVSMI, never on PATH.
  178. Path(os.environ.get("ProgramFiles", r"C:\Program Files"))
  179. / "NVIDIA Corporation" / "NVSMI" / "nvidia-smi.exe",
  180. )
  181. found = next((str(c) for c in candidates if c.exists()), None)
  182. elif found is None and sys.platform.startswith("linux"):
  183. # The Windows display driver exposes nvidia-smi here inside WSL, but does not guarantee
  184. # that the directory is present on PATH.
  185. candidate = Path("/usr/lib/wsl/lib/nvidia-smi")
  186. if candidate.exists():
  187. found = str(candidate)
  188. _smi_path_cache = (found,)
  189. return found
  190. def _nvidia_vram() -> tuple[int, int, str, int | None] | None:
  191. """(total bytes, free bytes, name, optional packed PCI ID) from the shared nvidia-smi query, or None."""
  192. query = _cached_nvidia_gpu_query()
  193. if query is None:
  194. return None
  195. return query["total_bytes"], query["free_bytes"], query["gpu_name"], query.get("gpu_pci_id")
  196. _gpu_query_cache: tuple[float, dict | None] | None = None
  197. # The statusbar polls /api/local-models/hardware every 5s and the endpoint needs
  198. # name/util/vram; the budget probe needs total/free. One shared query (with a TTL
  199. # shorter than the poll) serves both, so one poll = one nvidia-smi spawn (#120262)
  200. # instead of two, and the spawn carries CREATE_NO_WINDOW so a console-less backend
  201. # never flashes a window (#101895).
  202. _GPU_QUERY_TTL_S = 4.0
  203. def _cached_nvidia_gpu_query(ttl_s: float = _GPU_QUERY_TTL_S) -> dict | None:
  204. """One nvidia-smi read shared by the budget probe and the hardware endpoint.
  205. Returns ``dict(gpu_name=, total_bytes=, free_bytes=, used_bytes=, gpu_util_percent=,
  206. gpu_pci_id=)`` or None when nvidia-smi is absent, fails, or is not an NVIDIA card.
  207. Cached for ``ttl_s`` (failures too — a missing smi must not spawn per poll).
  208. """
  209. global _gpu_query_cache
  210. now = time.monotonic()
  211. if _gpu_query_cache is not None and now - _gpu_query_cache[0] < ttl_s:
  212. return _gpu_query_cache[1]
  213. exe = _nvidia_smi_path()
  214. if exe is None:
  215. _gpu_query_cache = (now, None)
  216. return None
  217. from hermes_cli._subprocess_compat import windows_hide_flags
  218. with suppress(OSError, ValueError, subprocess.TimeoutExpired):
  219. out = subprocess.run(
  220. [exe, "--query-gpu=memory.total,memory.free,name,pci.device_id,memory.used,utilization.gpu",
  221. "--format=csv,noheader,nounits"],
  222. capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10,
  223. creationflags=windows_hide_flags(), check=False)
  224. if out.returncode != 0 or not out.stdout.strip():
  225. _gpu_query_cache = (now, None)
  226. return None
  227. total_mib, free_mib, name, raw_id, used_mib, util = next(
  228. csv.reader(out.stdout.strip().splitlines(), skipinitialspace=True))
  229. pci_id = None
  230. with suppress(ValueError): # N/A or unsupported identity must not lose memory data.
  231. pci_id = int(raw_id, 16)
  232. data = {
  233. "gpu_name": name.strip(),
  234. "total_bytes": int(total_mib) << 20,
  235. "free_bytes": int(free_mib) << 20,
  236. "used_bytes": int(used_mib) << 20,
  237. "gpu_util_percent": int(util),
  238. "gpu_pci_id": pci_id,
  239. }
  240. _gpu_query_cache = (now, data)
  241. return data
  242. _gpu_query_cache = (now, None)
  243. return None
  244. def _cuda_driver_pool() -> tuple[int, bool | None] | None:
  245. """(allocator_total_bytes, integrated_or_None) from the CUDA driver API via ctypes against the
  246. driver's own DLL/SO — no toolkit, no subprocess, ~ms. INTEGRATED is the vendor's own
  247. unified-memory declaration; total is the pool the allocator will actually hand out (on
  248. carve-out devices, several times what nvidia-smi reports)."""
  249. import ctypes
  250. for name in ("nvcuda.dll", "libcuda.so.1", "libcuda.so"):
  251. try:
  252. cuda = ctypes.CDLL(name)
  253. break
  254. except OSError:
  255. continue
  256. else:
  257. return None
  258. with suppress(OSError, AttributeError):
  259. if cuda.cuInit(0) != 0:
  260. return None
  261. dev = ctypes.c_int()
  262. if cuda.cuDeviceGet(ctypes.byref(dev), 0) != 0:
  263. return None
  264. total = ctypes.c_size_t()
  265. getter = getattr(cuda, "cuDeviceTotalMem_v2", None) or cuda.cuDeviceTotalMem
  266. if getter(ctypes.byref(total), dev) != 0 or total.value <= 0:
  267. return None
  268. integrated: bool | None = None
  269. attr = ctypes.c_int()
  270. if cuda.cuDeviceGetAttribute(
  271. ctypes.byref(attr), _CU_DEVICE_ATTRIBUTE_INTEGRATED, dev) == 0:
  272. integrated = bool(attr.value)
  273. return total.value, integrated
  274. return None
  275. def _configured_engine():
  276. """The engine the probes below run: the managed server's own when one is up or booting, else
  277. the installed engine for ``local_runtime.backend``."""
  278. from hermes_cli.config import load_config_readonly
  279. from hermes_cli.local_runtime.binaries import installed_engine
  280. from hermes_cli.local_runtime.bootstrap import serving_engine
  281. engine = serving_engine()
  282. if engine is not None:
  283. return engine
  284. section = load_config_readonly().get("local_runtime") or {}
  285. return installed_engine(section.get("backend") or "auto")
  286. def _engine_device_pool() -> tuple[int, bool | None] | None:
  287. """(engine_total_bytes, None) from the installed runtime's own --list-devices, or None. The
  288. fallback when the driver API is unreachable: asks the exact binary that will do the
  289. allocating. Carries no integrated verdict — callers must gate it."""
  290. with suppress(Exception): # a probe miss must never block budgeting
  291. engine = _configured_engine()
  292. if engine is None:
  293. return None
  294. exe = engine.binary
  295. out = subprocess.run([str(exe), "--list-devices"], capture_output=True,
  296. text=True, encoding="utf-8", errors="replace", timeout=30, cwd=str(exe.parent), check=False)
  297. if out.returncode != 0:
  298. return None
  299. for line in (out.stdout + out.stderr).splitlines():
  300. m = _DEVICE_LINE_RE.search(line)
  301. if m:
  302. return int(m.group(1)) << 20, None
  303. return None
  304. def _device_pool_view() -> tuple[int, bool | None] | None:
  305. """Best available allocator-side view, cached: a hit is permanent for the process, a miss
  306. retries after a short TTL (the engine binary can appear mid-session via a pane install)."""
  307. global _pool_probe_cache
  308. now = time.monotonic()
  309. if _pool_probe_cache is not None:
  310. stamp, view = _pool_probe_cache
  311. if view is not None or now - stamp < _POOL_NEGATIVE_TTL_S:
  312. return view
  313. view = _cuda_driver_pool() or _engine_device_pool()
  314. _pool_probe_cache = (now, view)
  315. return view
  316. _accelerator_cache: dict[str, tuple[float, dict | None]] = {}
  317. def _accelerator_device(*, fresh: bool = False) -> dict | None:
  318. """The device a Vulkan/HIP engine will place layers on, or None (no such engine, probe miss).
  319. Mirrors llama.cpp's own placement: discrete devices whenever one exists, an integrated one only
  320. otherwise; among discrete devices the largest. Cached per engine binary like
  321. ``_device_pool_view`` (a hit lasts the process, a miss retries after the TTL): the hardware
  322. endpoint polls this every few seconds and each probe initializes the GPU driver. ``fresh`` skips
  323. the cache read for a launch-time free-memory answer.
  324. """
  325. with suppress(Exception): # a probe miss must never block budgeting
  326. from hermes_cli.local_runtime.devices import probe_devices
  327. engine = _configured_engine()
  328. if engine is None or engine.backend not in _GPU_ENGINE_BACKENDS:
  329. return None
  330. key, now = str(engine.binary), time.monotonic()
  331. cached = _accelerator_cache.get(key)
  332. if not fresh and cached is not None and (cached[1] is not None or now - cached[0] < _POOL_NEGATIVE_TTL_S):
  333. return cached[1]
  334. devices = probe_devices(engine.binary.parent, engine.backend)
  335. device = max(devices, key=lambda d: (d["type"] == GGML_DEVICE_GPU, d["total"]), default=None)
  336. _accelerator_cache[key] = (now, device)
  337. return device
  338. return None
  339. def _unified_pool_bytes(smi_total: int, ram_total: int) -> int | None:
  340. """The real pool size when this NVIDIA device is unified memory behind a carve-out, else None.
  341. The driver's INTEGRATED attribute decides in BOTH directions when readable; only the
  342. attribute-less engine fallback needs the two numeric gates, both of which must hold.
  343. """
  344. view = _device_pool_view()
  345. if view is None:
  346. return None
  347. pool, integrated = view
  348. if integrated is not None:
  349. return pool if integrated else None
  350. if (smi_total > 0 and pool >= int(smi_total * _POOL_DISAGREEMENT_FACTOR)
  351. and ram_total > 0 and pool >= int(ram_total * _POOL_RAM_FRACTION)):
  352. return pool
  353. return None
  354. def _uma_budget(base: int, total: int, *, gpu_name: str = "",
  355. gpu_pci_id: int | None = None, lazy_reads: bool = True) -> HardwareBudget:
  356. usable = max(0, int(base * (1 - _UMA_HEADROOM_FRACTION)))
  357. return HardwareBudget(usable_vram_bytes=usable, total_device_bytes=total,
  358. ram_available_bytes=0, uma=True, gpu_name=gpu_name, platform=sys.platform,
  359. gpu_pci_id=gpu_pci_id, lazy_reads=lazy_reads)
  360. def _windows_commit_room(own_bytes: int = 0) -> int | None:
  361. """Commit the managed server may take on Windows right now, or None off Windows.
  362. Windows charges every GPU allocation to commit: the RAM Windows can see plus the page file.
  363. Memory carved out for the GPU is invisible to Windows, so a bigger carve-out leaves less
  364. commit for the model it was meant to hold. ``own_bytes`` is what the managed server holds now
  365. and frees before the new instance loads. A desktop margin stays free.
  366. """
  367. stat = _windows_memory_status()
  368. if stat is None or not stat.ullTotalPageFile:
  369. return None
  370. margin = max(_MARGIN_FLOOR, int(stat.ullTotalPhys * _MARGIN_FRACTION))
  371. return max(0, int(stat.ullAvailPageFile) + max(0, own_bytes) - margin)
  372. def model_commit_bytes(model_bytes: int) -> int:
  373. """What Windows charges to commit for a model estimated at ``model_bytes`` of device memory:
  374. the estimate runs ~20% under the charge, which is the UMA headroom."""
  375. return int(model_bytes / (1 - _UMA_HEADROOM_FRACTION))
  376. def prompt_cache_bytes(live: HardwareBudget | None, model_bytes: int, residents: int = 1) -> int | None:
  377. """Size for llama-server's prompt cache beside a model of ``model_bytes`` (the estimate the
  378. window was planned with), or None when the engine default applies.
  379. Windows unified memory only: each loaded model's cache comes out of the same commit as the
  380. models, so each of the ``residents`` models the router may hold at once (its ``--models-max``)
  381. gets an even share of what they leave.
  382. """
  383. room = live.commit_room_bytes if live is not None else None
  384. if room is None or model_bytes <= 0:
  385. return None
  386. residents = max(1, residents)
  387. model_commit = model_commit_bytes(model_bytes)
  388. cache = min(_PROMPT_CACHE_MAX, max(0, room - residents * model_commit) // residents)
  389. return cache if cache >= _PROMPT_CACHE_MIN else 0
  390. def usable_beside_prompt_cache(live: HardwareBudget, residents: int = 1) -> int:
  391. """Device memory a model may plan with on Windows unified memory and still leave each of the
  392. ``residents`` models the smallest useful prompt cache: the inverse of ``prompt_cache_bytes``."""
  393. share = live.commit_room_bytes // max(1, residents) - _PROMPT_CACHE_MIN
  394. return min(live.usable_vram_bytes, max(0, int(share * (1 - _UMA_HEADROOM_FRACTION))))
  395. def _gpu_engine_runs_the_model() -> bool:
  396. """Whether a Vulkan/HIP engine runs the model: the serving or installed engine, else the one
  397. setup would install for ``local_runtime.backend``. False when the engine can't be resolved."""
  398. with suppress(Exception):
  399. engine = _configured_engine()
  400. if engine is not None:
  401. return engine.backend in _GPU_ENGINE_BACKENDS
  402. from hermes_cli.config import load_config_readonly
  403. from hermes_cli.local_runtime.binaries import resolve_backend
  404. section = load_config_readonly().get("local_runtime") or {}
  405. return resolve_backend(section.get("backend") or "auto") in _GPU_ENGINE_BACKENDS
  406. return False
  407. def _windows_igpu_bytes(device: dict | None) -> int | None:
  408. """What Windows lets the integrated GPU allocate (Task Manager's GPU memory), or None.
  409. The adapter is the one the engine named, else the only non-NVIDIA hardware adapter when a
  410. Vulkan/HIP engine will run the model; the CPU engine (chosen, or the only build for a Snapdragon)
  411. keeps the model in system RAM. Not the engine's own figure: ggml-vulkan sums every memory heap
  412. of an integrated GPU, host-visible ones included, so it can exceed what Windows grants.
  413. """
  414. from hermes_platform.host.gpu_adapters import windows_gpu_adapters
  415. hardware = [a for a in windows_gpu_adapters() if not a.software]
  416. named = [a for a in hardware if device is not None and a.description == device["description"]]
  417. candidates = named or [a for a in hardware if a.vendor_id != _NVIDIA_VENDOR_ID]
  418. if len(candidates) != 1 or (device is None and not _gpu_engine_runs_the_model()):
  419. return None
  420. return candidates[0].memory_bytes
  421. def probe_budget(*, planning: bool = False) -> HardwareBudget:
  422. """Construct the budget per the source rules above.
  423. ``planning=False``: LIVE budget (free VRAM now) for launch-time fit and growth re-grants.
  424. ``planning=True``: CAPACITY budget (total minus margin) for catalog pricing and quant
  425. selection — pricing against live-free while a model was loaded made every row read as too
  426. large. The managed server unloads/relaunches itself, so capacity is real.
  427. """
  428. ram_total, ram_avail = _ram_bytes()
  429. vram = _nvidia_vram()
  430. # Unified-memory NVIDIA: the CUDA allocator pool is the real capacity. Classification comes
  431. # from the driver API/engine and must not require nvidia-smi (stripped-PATH sessions lose smi
  432. # but nvcuda loads via the system loader). Deliberately NOT clamped to OS RAM: carved-out
  433. # memory is invisible to GlobalMemoryStatusEx, so a RAM clamp would throw away exactly that
  434. # capacity. On Windows, commit narrows the launch window instead (``launch_budget``).
  435. unified = _unified_pool_bytes(vram[0] if vram else 0, ram_total)
  436. if unified is not None:
  437. logger.info(
  438. "unified-memory NVIDIA device: allocator pool %.1f GiB "
  439. "(nvidia-smi carve-out: %s); budgeting from the pool",
  440. unified / _GIB,
  441. f"{vram[0] / _GIB:.1f} GiB" if vram else "unavailable")
  442. if planning:
  443. base = unified
  444. else:
  445. # Live: dedicated-free plus what the OS can still give. smi's free saturates at the
  446. # carve-out so this under-counts a bit — the safe direction (the pool edge is a
  447. # measured soft cliff: decode collapses ~3.5x when concurrent demand hits it).
  448. live = (vram[1] + ram_avail) if vram else ram_avail
  449. base = min(unified, live)
  450. return _uma_budget(base, unified, gpu_name=vram[2] if vram else "",
  451. gpu_pci_id=vram[3] if vram else None)
  452. if vram is None:
  453. device = _accelerator_device()
  454. if device is not None and device["type"] == GGML_DEVICE_GPU:
  455. # Discrete AMD/Intel card behind a Vulkan/HIP engine: its own memory, with system RAM
  456. # as spill. Capacity serves the polled live view; launch_budget reads free memory.
  457. total = device["total"]
  458. margin = max(_MARGIN_FLOOR, int(total * _MARGIN_FRACTION))
  459. return HardwareBudget(usable_vram_bytes=max(0, total - margin), total_device_bytes=total,
  460. ram_available_bytes=ram_total if planning else ram_avail,
  461. uma=False, gpu_name=device["description"], platform=sys.platform)
  462. # Integrated GPU, Metal, CPU, or no answer: budget from RAM as unified memory, capped at what
  463. # Windows lets an integrated GPU allocate (18 GB of 31.5 on an Arc B390). llama.cpp loads
  464. # lazy tensors up front on an AMD/Intel integrated GPU, where reading them on demand halved
  465. # prefill (#28160).
  466. pool = min(ram_total, _windows_igpu_bytes(device) or ram_total)
  467. return _uma_budget(pool if planning else min(pool, ram_avail), pool,
  468. lazy_reads=gpu_class() not in ("amd", "intel"))
  469. total, free, gpu_name, gpu_pci_id = vram
  470. margin = max(_MARGIN_FLOOR, int(total * _MARGIN_FRACTION))
  471. return HardwareBudget(usable_vram_bytes=max(0, (total if planning else free) - margin),
  472. total_device_bytes=total,
  473. ram_available_bytes=ram_total if planning else ram_avail,
  474. uma=False, gpu_name=gpu_name, platform=sys.platform, gpu_pci_id=gpu_pci_id)
  475. def launch_budget(capacity: HardwareBudget, *, own_bytes: int = 0) -> HardwareBudget | None:
  476. """Capacity less what other programs hold on the card right now, or None when that can't apply.
  477. Capacity assumes other programs hold no more than the fixed margin. Beside a heavier desktop the
  478. capacity-sized window doesn't fit, and Windows pages part of the model to host memory without
  479. an error: a 32 GiB card with 4.5-6.3 GiB held by other apps decoded at ~24 tok/s at the
  480. capacity window, and at ~90 tok/s at a window sized from free memory.
  481. ``own_bytes`` is the device-memory estimate of what the managed server holds now and frees
  482. before the new instance loads. A stopped server's memory reads as free by the time its process
  483. has exited (measured on Windows: free memory was fully back at the first reading after exit).
  484. Unified memory: free commit on Windows (the model and its prompt cache must fit there), with
  485. ``own_bytes`` converted to the commit Windows charged for it, else None (its live budget is
  486. already free memory). None too when the device query fails, so callers keep the capacity plan.
  487. """
  488. if capacity.uma:
  489. room = _windows_commit_room(model_commit_bytes(own_bytes) if own_bytes > 0 else 0)
  490. if room is None:
  491. return None
  492. usable = min(capacity.usable_vram_bytes, int(room * (1 - _UMA_HEADROOM_FRACTION)))
  493. return replace(capacity, usable_vram_bytes=usable, commit_room_bytes=room)
  494. vram = _nvidia_vram()
  495. if vram is not None:
  496. total, free = vram[0], vram[1]
  497. else:
  498. # A discrete card behind Vulkan/HIP: the driver's free count includes other programs.
  499. device = _accelerator_device(fresh=True)
  500. if device is None or device["type"] != GGML_DEVICE_GPU:
  501. return None
  502. total, free = device["total"], device["free"]
  503. others = max(0, total - free - max(0, own_bytes))
  504. usable = min(capacity.usable_vram_bytes, max(0, total - others - _LAUNCH_HEADROOM))
  505. return replace(capacity, usable_vram_bytes=usable)