gguf.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361
  1. """GGUF metadata + tensor-table reader (stdlib only).
  2. Reads the header only (metadata + tensor infos); never touches tensor data, so it is fast enough to
  3. run at picker time on multi-GB files.
  4. """
  5. from __future__ import annotations
  6. import logging
  7. import re
  8. import struct
  9. from dataclasses import dataclass, field
  10. from pathlib import Path
  11. logger = logging.getLogger(__name__)
  12. _GGUF_MAGIC = b"GGUF"
  13. # Split GGUF naming: "<stem>-00001-of-00003.gguf"; the part suffix is not part of the model id.
  14. SPLIT_PART_RE = re.compile(r"-(\d{5})-of-(\d{5})\.gguf$")
  15. _PART_SUFFIX_RE = re.compile(r"-\d{5}-of-\d{5}$")
  16. # Same tensor selection as context_policy's per-block FFN -ot override.
  17. _FFN_WEIGHT = re.compile(r"blk\.(\d+)\.ffn_.*\.weight")
  18. # A block with an SSM conv filter keeps recurrent state (Mamba2 and gated delta-net alike).
  19. _SSM_CONV = re.compile(r"blk\.(\d+)\.ssm_conv1d\.weight")
  20. # Tensors llama.cpp leaves in the file and reads row by row on demand instead of loading them
  21. # (create_tensor's TENSOR_READ_LAZY, marked per architecture in the engine's model code). Under the
  22. # engine's default --lazy-mode auto, only a marked tensor larger than 4 GiB is read this way;
  23. # a smaller one loads like any other weight.
  24. _LAZY_READ_TENSORS = {
  25. "qwen4exp": frozenset({"per_layer_token_embd.weight"}),
  26. "gemma4": frozenset({"per_layer_token_embd.weight"}),
  27. }
  28. _LAZY_READ_CANDIDATES = frozenset().union(*_LAZY_READ_TENSORS.values())
  29. _LAZY_READ_MIN_BYTES = 4 << 30
  30. def model_id_from_stem(stem: str) -> str:
  31. """Model id from a GGUF file stem (split-part suffix stripped)."""
  32. return _PART_SUFFIX_RE.sub("", stem)
  33. # ggml tensor type sizes: type_id -> (block_bytes, block_elems). IQ-family verified against
  34. # ggml-common.h.
  35. _GGML_TYPE_SIZES = {
  36. 0: (4, 1), 1: (2, 1), 2: (18, 32), 3: (20, 32), 6: (22, 32), 7: (24, 32),
  37. 8: (34, 32), 9: (36, 32), 10: (84, 256), 11: (110, 256), 12: (144, 256),
  38. 13: (176, 256), 14: (210, 256), 15: (292, 256), 16: (66, 256),
  39. 17: (74, 256), 18: (98, 256), 19: (50, 256), 20: (18, 32),
  40. 21: (110, 256), 22: (82, 256), 23: (136, 256), 24: (1, 1), 25: (2, 1),
  41. 26: (4, 1), 27: (8, 1), 28: (8, 1), 29: (56, 256), 30: (2, 1),
  42. 39: (17, 32), # MXFP4 — 32 elements per 17-byte block (gpt-oss family)
  43. 40: (36, 64), # NVFP4 — 4 E4M3 sub-block scales + 32 bytes of E2M1 (Nemotron 3.5 Bolt)
  44. 41: (18, 128), # Q1_0
  45. 42: (18, 64), # Q2_0
  46. }
  47. # GGUF metadata value types -> struct format; STRING (8) and ARRAY (9) are variable-length.
  48. _V_STRING, _V_ARRAY = 8, 9
  49. _SCALAR_FMT = {
  50. 0: "<B", 1: "<b", 2: "<H", 3: "<h", # uint8 int8 uint16 int16
  51. 4: "<I", 5: "<i", 6: "<f", 7: "<?", # uint32 int32 float32 bool
  52. 10: "<Q", 11: "<q", 12: "<d", # uint64 int64 float64
  53. }
  54. # general.sampling.* metadata key -> preset INI key.
  55. _SAMPLING_INI_KEY = {"temp": "temp", "temperature": "temp", "top_p": "top-p",
  56. "top_k": "top-k", "min_p": "min-p",
  57. "repeat_penalty": "repeat-penalty",
  58. "presence_penalty": "presence-penalty"}
  59. @dataclass
  60. class GGUFHeader:
  61. path: str
  62. version: int
  63. metadata: dict = field(default_factory=dict)
  64. n_tensors: int = 0
  65. tensor_bytes: int = 0 # exact sum over the tensor table
  66. embd_table_bytes: int = 0 # token_embd.weight (duplicated host-side when fully offloaded)
  67. # block index -> bytes of that block's FFN weights (the tensors a `blk\.N\.ffn_.*\.weight`
  68. # -ot override moves), so spill placement can move only as many blocks as it needs.
  69. ffn_block_bytes: dict[int, int] = field(default_factory=dict)
  70. # Sizes of the tensors some architecture reads lazily, by name. ``lazy_bytes`` applies this
  71. # model's architecture and the size threshold.
  72. lazy_candidate_bytes: dict[str, int] = field(default_factory=dict)
  73. # Blocks that keep recurrent state. Counted from the tensor table: hybrids like Nemotron-H
  74. # also have MoE-only blocks without KV heads, which hold none.
  75. recurrent_blocks: frozenset[int] = frozenset()
  76. # ── typed accessors ──────────────────────────────────────
  77. @property
  78. def architecture(self) -> str:
  79. return str(self.metadata.get("general.architecture", ""))
  80. @property
  81. def lazy_bytes(self) -> int:
  82. """Bytes of weights llama.cpp keeps on disk and reads row by row instead of loading."""
  83. marked = _LAZY_READ_TENSORS.get(self.architecture, frozenset())
  84. return sum(nbytes for name, nbytes in self.lazy_candidate_bytes.items()
  85. if name in marked and nbytes > _LAZY_READ_MIN_BYTES)
  86. def _arch_key(self, suffix: str):
  87. return self.metadata.get(f"{self.architecture}.{suffix}")
  88. @property
  89. def recurrent_state_bytes(self) -> int:
  90. """One copy of one sequence's recurrent state: an f32 conv window plus the SSM state, per
  91. block that keeps state. llama.cpp holds one copy per slot per speculative draft position.
  92. 0 when the file declares no SSM shape."""
  93. def ssm(key: str) -> int:
  94. return int(self._arch_key(f"ssm.{key}") or 0)
  95. inner, state = ssm("inner_size"), ssm("state_size")
  96. if not (self.recurrent_blocks and inner and state):
  97. return 0
  98. conv_dim = inner + 2 * ssm("group_count") * state
  99. per_block = 4 * (inner * state + max(0, ssm("conv_kernel") - 1) * conv_dim)
  100. return len(self.recurrent_blocks) * per_block
  101. def _arch_int(suffix: str, doc: str = ""):
  102. return property(lambda self: int(self._arch_key(suffix) or 0), doc=doc)
  103. n_layer = _arch_int("block_count")
  104. n_ctx_train = _arch_int("context_length")
  105. n_embd = _arch_int("embedding_length")
  106. sliding_window = _arch_int("attention.sliding_window")
  107. expert_count = _arch_int("expert_count")
  108. full_attention_interval = _arch_int(
  109. "full_attention_interval",
  110. "GDN-hybrid discriminator (qwen35 family): every Nth layer is full attention, the rest "
  111. "are linear/recurrent. 0 = not present.")
  112. key_length_swa = _arch_int(
  113. "attention.key_length_swa",
  114. "Per-token key size for sliding-window layers when it differs from the global "
  115. "attention.key_length (e.g. gemma3/gemma4). 0 = not present.")
  116. value_length_swa = _arch_int(
  117. "attention.value_length_swa",
  118. "Per-token value size for sliding-window layers when it differs from the global "
  119. "attention.value_length. 0 = not present.")
  120. del _arch_int
  121. @property
  122. def sliding_window_pattern(self) -> list[int] | None:
  123. """Per-layer SWA pattern declared by the file itself: a truthy entry marks a
  124. sliding-window layer, falsy marks global. None when the file doesn't declare the per-layer
  125. array form (older GGUFs, architectures without per-layer SWA metadata, or files that use
  126. the scalar period form instead — see `sliding_window_pattern_period`)."""
  127. v = self._arch_key("attention.sliding_window_pattern")
  128. return [int(x) for x in v] if isinstance(v, list) else None
  129. @property
  130. def sliding_window_pattern_period(self) -> int:
  131. """Scalar SWA period declared by the file: llama.cpp also permits
  132. `attention.sliding_window_pattern` as a single integer N (every Nth layer is full
  133. attention, e.g. Gemma-family writers) rather than a per-layer array. 0 when absent or when
  134. the file uses the array form instead — callers should fall back to a coarser signal."""
  135. v = self._arch_key("attention.sliding_window_pattern")
  136. return int(v) if isinstance(v, int) and not isinstance(v, bool) else 0
  137. @property
  138. def n_vocab(self) -> int:
  139. """Vocabulary size (prices the GPU logits buffers): vocab_size metadata when present, else
  140. the tokenizer list length."""
  141. v = self._arch_key("vocab_size")
  142. if v:
  143. return int(v)
  144. toks = self.metadata.get("tokenizer.ggml.tokens")
  145. return len(toks) if isinstance(toks, list) else 0
  146. @property
  147. def sampling_defaults(self) -> dict:
  148. """Upstream's recommended sampling as preset INI keys, when the file carries it.
  149. Publishers bake ``general.sampling.*`` keys into the GGUF (llama-server reads them as that
  150. model's defaults), so the file is the source of truth — it ships with the download and
  151. updates with every re-upload, no catalog needed. Empty if absent.
  152. """
  153. out = {}
  154. for key, value in self.metadata.items():
  155. if not key.startswith("general.sampling."):
  156. continue
  157. name = _SAMPLING_INI_KEY.get(key.rsplit(".", 1)[-1])
  158. if name is not None and isinstance(value, (int, float)):
  159. num = round(float(value), 4)
  160. out[name] = str(int(num)) if num == int(num) else str(num)
  161. return out
  162. @property
  163. def n_head(self) -> int:
  164. v = self._arch_key("attention.head_count")
  165. if isinstance(v, list):
  166. return int(max(v))
  167. return int(v or 0)
  168. def head_counts_kv(self) -> list[int]:
  169. """Per-layer KV head counts; 0 marks a recurrent/linear layer (n_head_kv == 0).
  170. Three GGUF shapes: a per-layer array (nemotron_h_moe) is used as-is; a scalar plus
  171. ``full_attention_interval`` (qwen35) applies to every N-th layer (1-indexed) and is zero
  172. elsewhere — pricing all layers as attention was a 4x overestimate; a plain scalar (dense)
  173. broadcasts to every layer.
  174. """
  175. v = self._arch_key("attention.head_count_kv")
  176. if isinstance(v, list):
  177. return [int(x) for x in v]
  178. scalar = int(v or 0)
  179. interval = self.full_attention_interval
  180. if interval > 1:
  181. return [scalar if (i + 1) % interval == 0 else 0
  182. for i in range(self.n_layer)]
  183. return [scalar] * self.n_layer
  184. @property
  185. def head_dim_k(self) -> int:
  186. v = self._arch_key("attention.key_length")
  187. if v:
  188. return int(v)
  189. return self.n_embd // self.n_head if self.n_head else 0
  190. @property
  191. def head_dim_v(self) -> int:
  192. v = self._arch_key("attention.value_length")
  193. if v:
  194. return int(v)
  195. return self.head_dim_k
  196. def split_parts(path: Path) -> list[Path] | None:
  197. """Every on-disk part of the split ``path`` belongs to, first part first; None when ``path`` is
  198. not a split member or no other part is present.
  199. A split is priced as the set, never as one file: publishers lay shards out so the first part
  200. can hold little more than metadata while the bulk sits in the later ones, so reading one part
  201. prices the model at whatever fraction of its weights that part happens to hold."""
  202. m = SPLIT_PART_RE.search(path.name)
  203. if m is None:
  204. return None
  205. stem, total = path.name[: m.start()], int(m.group(2))
  206. parts = [p for p in (path.with_name(f"{stem}-{i:05d}-of-{total:05d}.gguf")
  207. for i in range(1, total + 1)) if p.is_file()]
  208. return parts if len(parts) > 1 else None
  209. def _read_part(path: Path) -> GGUFHeader:
  210. """One file's own header: metadata and that file's tensor table."""
  211. def read(f, fmt: str):
  212. # A header cut short raises ValueError, which every caller already treats as "skip this
  213. # model"; struct.error would escape them and take the whole preset pass down.
  214. size = struct.calcsize(fmt)
  215. data = f.read(size)
  216. if len(data) != size:
  217. raise ValueError(f"truncated GGUF header: {path}")
  218. return struct.unpack(fmt, data)
  219. def read_str(f) -> str:
  220. (n,) = read(f, "<Q")
  221. return f.read(n).decode("utf-8", errors="replace")
  222. def read_value(f, vtype: int):
  223. if vtype == _V_STRING:
  224. return read_str(f)
  225. if vtype == _V_ARRAY:
  226. etype, n = read(f, "<IQ")
  227. return [read_value(f, etype) for _ in range(n)]
  228. return read(f, _SCALAR_FMT[vtype])[0]
  229. with open(path, "rb") as f:
  230. if f.read(4) != _GGUF_MAGIC:
  231. raise ValueError(f"not a GGUF file: {path}")
  232. version, n_tensors, n_kv = read(f, "<IQQ")
  233. metadata: dict = {}
  234. for _ in range(n_kv):
  235. key = read_str(f)
  236. (vtype,) = read(f, "<I")
  237. metadata[key] = read_value(f, vtype)
  238. tensor_bytes = 0
  239. embd_bytes = 0
  240. ffn_block_bytes: dict[int, int] = {}
  241. lazy_candidate_bytes: dict[str, int] = {}
  242. recurrent_blocks: set[int] = set()
  243. for _ in range(n_tensors):
  244. name = read_str(f)
  245. (n_dims,) = read(f, "<I")
  246. dims = read(f, f"<{n_dims}Q")
  247. (ttype,) = read(f, "<I")
  248. f.read(8) # offset
  249. size = _GGML_TYPE_SIZES.get(ttype)
  250. if size is None:
  251. raise ValueError(f"unknown ggml tensor type {ttype} in {path}")
  252. block_bytes, block_elems = size
  253. elems = 1
  254. for d in dims:
  255. elems *= d
  256. nbytes = (elems // block_elems) * block_bytes
  257. tensor_bytes += nbytes
  258. if name == "token_embd.weight":
  259. embd_bytes = nbytes
  260. elif m := _FFN_WEIGHT.match(name):
  261. block = int(m.group(1))
  262. ffn_block_bytes[block] = ffn_block_bytes.get(block, 0) + nbytes
  263. elif name in _LAZY_READ_CANDIDATES:
  264. lazy_candidate_bytes[name] = nbytes
  265. elif m := _SSM_CONV.match(name):
  266. recurrent_blocks.add(int(m.group(1)))
  267. return GGUFHeader(path=str(path), version=version, metadata=metadata,
  268. n_tensors=n_tensors, tensor_bytes=tensor_bytes,
  269. embd_table_bytes=embd_bytes, ffn_block_bytes=ffn_block_bytes,
  270. lazy_candidate_bytes=lazy_candidate_bytes,
  271. recurrent_blocks=frozenset(recurrent_blocks))
  272. def read_gguf_header(path: str | Path) -> GGUFHeader:
  273. """Header for a MODEL, not for a file: a split GGUF is priced as the sum of its parts.
  274. Weights, the host-side embedding-table duplicate and the per-block FFN map are all summed
  275. across the shards on disk, because a split is a layout choice, not a smaller model — pricing
  276. part 1 alone underprices every model whose first shard is a metadata stub (the Hugging Face
  277. layout), which then makes the physics check and the residency cap admit giants the card cannot
  278. hold. Architecture metadata (block count, train context, the per-layer SWA pattern, vocab) is
  279. taken from the first part: gguf-split writes the model's metadata there only, and every later
  280. part carries just its ``split.*`` keys.
  281. A part that has gone missing or become unreadable is skipped rather than fatal: a half-arrived
  282. split prices at what is actually on disk. Refusing it is ``staged_in(require_complete=True)``'s
  283. job — an incomplete split is never servable, it is only underpriced here."""
  284. path = Path(path)
  285. parts = split_parts(path)
  286. if parts is None:
  287. return _read_part(path)
  288. first = _read_part(parts[0])
  289. readable = [first]
  290. for part in parts[1:]:
  291. try:
  292. readable.append(_read_part(part))
  293. except (ValueError, OSError) as exc:
  294. logger.debug("split part unreadable %s: %s", part.name, exc)
  295. if len(readable) == 1:
  296. return first
  297. ffn_block_bytes: dict[int, int] = {}
  298. for header in readable:
  299. for block, nbytes in header.ffn_block_bytes.items():
  300. ffn_block_bytes[block] = ffn_block_bytes.get(block, 0) + nbytes
  301. return GGUFHeader(
  302. # The path stays the part the caller named: it is the model id source and the preset's
  303. # ``model`` key, and llama.cpp resolves the split from any one of its members.
  304. path=str(path), version=first.version, metadata=first.metadata,
  305. n_tensors=sum(h.n_tensors for h in readable),
  306. tensor_bytes=sum(h.tensor_bytes for h in readable),
  307. embd_table_bytes=sum(h.embd_table_bytes for h in readable),
  308. ffn_block_bytes=ffn_block_bytes,
  309. lazy_candidate_bytes={name: nbytes for h in readable
  310. for name, nbytes in h.lazy_candidate_bytes.items()},
  311. recurrent_blocks=frozenset().union(*(h.recurrent_blocks for h in readable)))