estimator.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237
  1. """Per-layer context-memory estimator + physics check.
  2. The estimator is ADVISORY: fit's allocation is authoritative at launch and the touch generation is
  3. ground truth after it. Unknown shapes round UP (never underestimate memory).
  4. """
  5. from __future__ import annotations
  6. from dataclasses import dataclass, field, replace
  7. from enum import Enum
  8. from hermes_cli.local_runtime.gguf import GGUFHeader
  9. # q8_0: 34-byte blocks of 32 f16-equivalent elements (exact).
  10. _Q8_BYTES_PER_ELEM = 34 / 32
  11. _F16_BYTES_PER_ELEM = 2.0
  12. # Architectures with a known SWA layer pattern: arch -> fraction of layers that are
  13. # sliding-window. Unknown SWA archs treat every layer as full attention (overestimate; safe).
  14. _SWA_LAYER_FRACTION = {"gemma3": 5 / 6, "gemma2": 1 / 2}
  15. # Architectures whose llama.cpp loader expands a scalar `sliding_window_pattern` period with
  16. # dense_first=True (the full-attention layer opens each period-length cycle instead of closing
  17. # it). Confirmed against llama.cpp's per-arch hparams.set_swa_pattern() calls; every other
  18. # architecture — including the whole Gemma family — uses the dense_first=False default.
  19. _SWA_DENSE_FIRST_ARCHS = {"cohere2moe", "modern-bert", "smallthinker", "laguna"}
  20. def _expand_swa_period(period: int, n_layer: int, dense_first: bool) -> list[int]:
  21. """Per-layer SWA/full split from a scalar period, matching llama.cpp's
  22. `llama_hparams::set_swa_pattern()`: one full-attention layer per `period`-length cycle, the
  23. rest sliding-window."""
  24. if period <= 0:
  25. return []
  26. if dense_first:
  27. return [0 if i % period == 0 else 1 for i in range(n_layer)]
  28. return [1 if i % period < period - 1 else 0 for i in range(n_layer)]
  29. # Per-recurrent-layer state allowance (bytes/seq). Deliberately generous: an entire measured
  30. # hybrid slot state is ~99 MB including 8K tokens of full-attn KV, so tens of MiB total is the
  31. # right order; unknown SSM shapes must never underestimate.
  32. _RECURRENT_STATE_PER_LAYER = 4 << 20
  33. # Compute-buffer bytes per microbatch token per window token, per llama.cpp context, for
  34. # architectures whose attention scores the whole window each microbatch. qwen4exp's QSA indexer:
  35. # measured on b11370 with the router's unified KV (every slot sees the full window), device plus
  36. # host buffers, target and MTP head contexts alike (39.5 + 6.1 B; 5.9 GiB per context at 256K and
  37. # -ub 512, ~23 GiB at -ub 2048).
  38. _WINDOW_COMPUTE_BYTES = {"qwen4exp": 46}
  39. class LayerKind(Enum):
  40. FULL = "full"
  41. SWA = "swa"
  42. RECURRENT = "recurrent"
  43. @dataclass
  44. class ModelProfile:
  45. """Everything the policy needs, decoupled from GGUF parsing so decision-table tests can
  46. construct profiles directly."""
  47. name: str
  48. # Weights the engine loads when it reads lazy_bytes from disk on demand; as_loaded() adds them
  49. # back on a machine where it reads them up front.
  50. weights_bytes: int
  51. embd_table_bytes: int
  52. n_ctx_train: int
  53. layers: list[tuple[LayerKind, int]] # (kind, kv_bytes_per_token_f16); SWA capped, recurrent ignored
  54. swa_window: int = 0
  55. moe: bool = False
  56. architecture: str = ""
  57. n_vocab: int = 0 # prices logits buffers (ubatch x vocab)
  58. # Context-cost multiplier. MTP spec decode keeps a small draft context beside the main one;
  59. # calibrated against four measured server-RSS points on Qwen3.8 Q4 (128K/221K/256K, both
  60. # postures): the draft adds ~17% to per-token KV; 1.2 rounds up so the error stays on the safe
  61. # side (+250 MiB at 256K, never negative).
  62. kv_scale: float = 1.0
  63. # block index -> FFN weight bytes (from the tensor table); empty when unknown.
  64. ffn_block_bytes: dict[int, int] = field(default_factory=dict)
  65. # Bytes of architecture-marked tensors (gguf._LAZY_READ_TENSORS) the engine can read from disk
  66. # on demand instead of loading.
  67. lazy_bytes: int = 0
  68. # Compute-buffer bytes per window token at the launch posture (plan_launch sets it from
  69. # window_compute_bytes, the microbatch and the context count); zero prices none.
  70. window_compute_per_token: int = 0
  71. # One copy of one sequence's recurrent state, exact from the SSM shape (0 = unknown). The
  72. # window price keeps its per-layer allowance; this prices the extra copies speculative
  73. # decoding keeps.
  74. recurrent_state_bytes: int = 0
  75. @property
  76. def window_compute_bytes(self) -> int:
  77. return _WINDOW_COMPUTE_BYTES.get(self.architecture, 0)
  78. @property
  79. def per_token_kv_f16(self) -> int:
  80. """Uncapped per-token KV cost (full + SWA share)."""
  81. return sum(b for kind, b in self.layers if kind != LayerKind.RECURRENT)
  82. @property
  83. def recurrent_layer_count(self) -> int:
  84. return sum(1 for kind, _ in self.layers if kind == LayerKind.RECURRENT)
  85. @dataclass
  86. class HardwareBudget:
  87. """Memory the physics check may budget against. Discrete cards may trust the device query;
  88. unified-memory devices must budget from OS free memory minus headroom (device queries observed
  89. off by 3x). Callers construct this accordingly; the estimator just consumes it."""
  90. usable_vram_bytes: int # live free (discrete) / derived (UMA)
  91. total_device_bytes: int
  92. ram_available_bytes: int
  93. uma: bool = False
  94. gpu_name: str = "" # display name; legacy fallback for performance estimates
  95. platform: str = "" # sys.platform of the machine being priced
  96. gpu_pci_id: int | None = None # nvidia-smi's packed PCI device/vendor ID
  97. # The engine reads lazy tensors from disk here. llama.cpp's own default does so everywhere
  98. # except integrated GPUs (b11370 #28160); Hermes passes --lazy-mode on to NVIDIA's, so only
  99. # AMD/Intel integrated GPUs load them up front.
  100. lazy_reads: bool = True
  101. # Windows unified memory, live budgets only: commit the server may take for the models and
  102. # their prompt caches. Windows charges every GPU allocation to commit (RAM it can see plus
  103. # the page file), so this is the real ceiling there. None everywhere else.
  104. commit_room_bytes: int | None = None
  105. def as_loaded(profile: ModelProfile, budget: HardwareBudget) -> ModelProfile:
  106. """The profile as this machine loads it: on-demand tensors count as weights where the engine
  107. reads them up front."""
  108. if profile.lazy_bytes and not budget.lazy_reads:
  109. return replace(profile, weights_bytes=profile.weights_bytes + profile.lazy_bytes, lazy_bytes=0)
  110. return profile
  111. def profile_from_gguf(header: GGUFHeader) -> ModelProfile:
  112. kv_heads = header.head_counts_kv()
  113. dk, dv = header.head_dim_k, header.head_dim_v
  114. dk_swa = header.key_length_swa or dk
  115. dv_swa = header.value_length_swa or dv
  116. # Priority ladder, highest first: (1) the file's own per-layer pattern, array or scalar-period
  117. # form — architecture-agnostic and exact; (2) a known-architecture fraction, for older files
  118. # that declare `sliding_window` but no per-layer pattern; (3) no signal at all -> every layer
  119. # priced as full attention (overestimate; safe).
  120. pattern = header.sliding_window_pattern
  121. if pattern is None and header.sliding_window_pattern_period > 0:
  122. pattern = _expand_swa_period(header.sliding_window_pattern_period, header.n_layer,
  123. header.architecture in _SWA_DENSE_FIRST_ARCHS)
  124. has_pattern = (pattern is not None and header.sliding_window > 0
  125. and len(pattern) == len(kv_heads))
  126. swa_fraction = _SWA_LAYER_FRACTION.get(header.architecture, 0.0)
  127. has_fraction = not has_pattern and header.sliding_window > 0 and swa_fraction > 0
  128. n_attn_total = sum(1 for h in kv_heads if h > 0)
  129. n_swa = round(n_attn_total * swa_fraction) if has_fraction else 0
  130. layers: list[tuple[LayerKind, int]] = []
  131. n_attn_seen = 0
  132. for i, heads in enumerate(kv_heads):
  133. if heads == 0:
  134. layers.append((LayerKind.RECURRENT, 0))
  135. continue
  136. if has_pattern:
  137. is_swa = bool(pattern[i])
  138. else:
  139. # Distribute the SWA share across the first n_swa attention layers; only the
  140. # full/SWA split matters to the totals, not which indexes.
  141. is_swa = n_attn_seen < n_swa
  142. layer_dk, layer_dv = (dk_swa, dv_swa) if is_swa else (dk, dv)
  143. per_token = round(heads * (layer_dk + layer_dv) * _F16_BYTES_PER_ELEM)
  144. layers.append((LayerKind.SWA if is_swa else LayerKind.FULL, per_token))
  145. n_attn_seen += 1
  146. return ModelProfile(
  147. name=header.path, weights_bytes=header.tensor_bytes - header.lazy_bytes,
  148. embd_table_bytes=header.embd_table_bytes,
  149. n_ctx_train=header.n_ctx_train, layers=layers, swa_window=header.sliding_window,
  150. moe=header.expert_count > 0, architecture=header.architecture, n_vocab=header.n_vocab,
  151. ffn_block_bytes=dict(header.ffn_block_bytes), lazy_bytes=header.lazy_bytes,
  152. recurrent_state_bytes=header.recurrent_state_bytes)
  153. def kv_dtype_factor(flash_attention: bool) -> float:
  154. """q8_0 with FA (every backend we ship); f16 on exotic non-FA fallbacks — the 64K guarantee
  155. stands either way, the physics check just prices the doubled KV."""
  156. return (_Q8_BYTES_PER_ELEM / _F16_BYTES_PER_ELEM) if flash_attention else 1.0
  157. def ctx_bytes(profile: ModelProfile, window: int, *, flash_attention: bool = True) -> int:
  158. """Memory that grows with the window: full layers linear in T, SWA layers capped at the sliding
  159. window, recurrent layers constant, KV scaled by profile.kv_scale (MTP draft context), plus the
  160. posture's window-scaled compute buffers."""
  161. factor = kv_dtype_factor(flash_attention)
  162. total = 0.0
  163. for kind, per_token_f16 in profile.layers:
  164. if kind == LayerKind.RECURRENT:
  165. total += _RECURRENT_STATE_PER_LAYER
  166. elif kind == LayerKind.SWA:
  167. total += per_token_f16 * factor * min(window, profile.swa_window)
  168. else:
  169. total += per_token_f16 * factor * window
  170. return int(total * profile.kv_scale) + profile.window_compute_per_token * window
  171. @dataclass
  172. class PhysicsRefusal:
  173. """The only true refusal: weights + floor-KV + state exceed VRAM + RAM. The remedy is a
  174. smaller quant, never a smaller window."""
  175. needed_bytes: int
  176. available_bytes: int
  177. message: str
  178. def footprint_bytes(profile: ModelProfile, window: int, *, flash_attention: bool = True,
  179. overhead_bytes: int = 0) -> int:
  180. """Complete estimated footprint; the hardware budget already excludes its reserve."""
  181. return (profile.weights_bytes + ctx_bytes(profile, window, flash_attention=flash_attention)
  182. + max(0, overhead_bytes))
  183. def physics_check(profile: ModelProfile, budget: HardwareBudget,
  184. floor: int, *, flash_attention: bool = True,
  185. overhead_bytes: int = 0) -> PhysicsRefusal | None:
  186. needed = footprint_bytes(profile, min(floor, profile.n_ctx_train or floor),
  187. flash_attention=flash_attention, overhead_bytes=overhead_bytes)
  188. available = budget.usable_vram_bytes + budget.ram_available_bytes
  189. if needed <= available:
  190. return None
  191. gib = 1 << 30
  192. return PhysicsRefusal(
  193. needed_bytes=needed, available_bytes=available,
  194. message=(f"{profile.name}: needs ~{needed / gib:.1f} GiB at the "
  195. f"{floor // 1024}K floor but only ~{available / gib:.1f} GiB "
  196. "of VRAM+RAM are available — try a smaller model or a supported smaller quant"))