context_policy.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323
  1. """Context policy — the window ladder for managed local models.
  2. One contract: any model runs at any window up to its native max; hardware and session depth only
  3. change tokens/s. Constants, not knobs — nothing in this module reads config. The policy encodes
  4. behavior measured on real hardware (llama.cpp, discrete NVIDIA on Windows/WDDM, unified-memory).
  5. """
  6. from __future__ import annotations
  7. from dataclasses import dataclass, field, replace
  8. from hermes_cli.local_runtime.estimator import (
  9. HardwareBudget, LayerKind, ModelProfile, PhysicsRefusal, ctx_bytes, footprint_bytes,
  10. physics_check)
  11. FLOOR = 64 * 1024 # = target; one internal constant
  12. _LADDER_GROWTH = 1.5
  13. _GROW_AT_OCCUPANCY = 0.85 # of the current window, at turn boundary
  14. SPEED_FLOOR_TOK_S = 6.0 # deepest measured spill bottomed near this
  15. _EARLY_COST_CTX_FRACTION = 0.15 # bounded early cost when weights spill
  16. # The smallest ladder rung at which compression becomes the exception rather than the routine.
  17. # Measured over 161 real agentic sessions: 66% complete uncompressed in 64K, 82% in 96K, 91% in
  18. # 144K — and the marginal gain past 144K (+6 points for 216K) falls below the quality cost of
  19. # stepping down another quant. The FLOOR remains the guarantee.
  20. TARGET_WINDOW = 144 * 1024
  21. # What a load really costs beyond weights + KV: CUDA contexts and compute buffers at the DEFAULT
  22. # microbatch (-ub 512, no MTP). Measured on a 32 GiB card: a model estimated at 29.3 GiB loaded at
  23. # ~31.2 GiB resident and fit still shaved a layer to CPU. Microbatch/MTP logits buffers are priced
  24. # separately per model (ub_logits_bytes — they scale with vocab and once packed a card 3.9 GiB
  25. # past this constant). Callers add mmproj bytes on top.
  26. RUNTIME_OVERHEAD_BYTES = int(1.5 * (1 << 30))
  27. # llama.cpp's default microbatch, and the larger one launch_args passes for faster prefill.
  28. DEFAULT_UBATCH = 512
  29. PREFILL_UBATCH = 2048
  30. def ladder(native: int) -> list[int]:
  31. """64K -> 96K -> 128K -> ... -> native (native always the last rung)."""
  32. rungs: list[int] = []
  33. step = float(FLOOR)
  34. while step < native:
  35. rungs.append(int(step))
  36. step *= _LADDER_GROWTH
  37. rungs.append(native)
  38. return rungs
  39. @dataclass
  40. class WindowDecision:
  41. window: int
  42. spill_bytes: int # weights displaced to host at this window
  43. kv_on_gpu: bool
  44. reasons: list[str] = field(default_factory=list)
  45. @property
  46. def spilled(self) -> bool:
  47. return self.spill_bytes > 0
  48. def initial_window(profile: ModelProfile, budget: HardwareBudget, *, flash_attention: bool = True,
  49. overhead_bytes: int = 0) -> WindowDecision | PhysicsRefusal:
  50. """The launch decision: largest cheap rung, never below the floor.
  51. Zero-spill rung: weights + ctx + overhead fit usable VRAM entirely. Bounded-early-cost rung
  52. (weights already exceed VRAM): largest rung whose ctx stays <= ~15% of usable VRAM. Floor
  53. everywhere, capped at native. ``overhead_bytes`` is runtime cost beyond weights+KV; zero keeps
  54. this pure physics for decision-table tests, production callers pass it.
  55. """
  56. refusal = physics_check(profile, budget, FLOOR, flash_attention=flash_attention,
  57. overhead_bytes=overhead_bytes)
  58. if refusal:
  59. return refusal
  60. native = profile.n_ctx_train or FLOOR
  61. rungs = ladder(native)
  62. def kv(rung: int) -> int:
  63. return ctx_bytes(profile, rung, flash_attention=flash_attention)
  64. def need(rung: int) -> int:
  65. return footprint_bytes(profile, rung, flash_attention=flash_attention,
  66. overhead_bytes=overhead_bytes)
  67. best_zero_spill: int | None = None
  68. for rung in rungs:
  69. if need(rung) > budget.usable_vram_bytes:
  70. break
  71. best_zero_spill = rung
  72. if best_zero_spill is not None and best_zero_spill >= min(FLOOR, native):
  73. window = best_zero_spill
  74. reason = f"largest zero-spill rung ({window // 1024}K)"
  75. else:
  76. # Weights spill from turn one (steep-curve model on a small card) — hold the floor, bound
  77. # the early ctx cost.
  78. cap = int(budget.usable_vram_bytes * _EARLY_COST_CTX_FRACTION)
  79. window = min(FLOOR, native)
  80. for rung in rungs:
  81. if rung < window:
  82. continue
  83. if (kv(rung) > cap
  84. or need(rung) > budget.usable_vram_bytes + budget.ram_available_bytes):
  85. break
  86. window = rung
  87. reason = f"floor held at {window // 1024}K; weights spill (deliberate price of the guarantee)"
  88. kv_bytes = kv(window)
  89. return WindowDecision(window=window, reasons=[reason],
  90. spill_bytes=max(0, need(window) - budget.usable_vram_bytes),
  91. kv_on_gpu=kv_bytes + overhead_bytes <= budget.usable_vram_bytes)
  92. @dataclass
  93. class LaunchPlan:
  94. decision: WindowDecision | PhysicsRefusal
  95. mtp_prefill: bool
  96. overhead_bytes: int
  97. profile: ModelProfile | None = None # as priced at the chosen posture (posture_profile)
  98. def posture_profile(profile: ModelProfile, *, mtp_capable: bool, mtp_prefill: bool) -> ModelProfile:
  99. """``profile`` at one launch posture: window-scaled compute buffers grow with the microbatch
  100. launch_args passes, and MTP runs a second context beside the target."""
  101. ubatch = PREFILL_UBATCH if mtp_prefill or not mtp_capable else DEFAULT_UBATCH
  102. contexts = 2 if mtp_capable else 1
  103. return replace(profile, window_compute_per_token=profile.window_compute_bytes * ubatch * contexts)
  104. def plan_launch(profile: ModelProfile, budget: HardwareBudget, *, mtp_capable: bool = False,
  105. fixed_overhead: int = RUNTIME_OVERHEAD_BYTES,
  106. requested_window: int | None = None) -> LaunchPlan:
  107. """Window first, then prefill; price both postures at the effective window.
  108. A restored window may fit only under lean MTP. Evaluate it before discarding it because
  109. stacked exceeds memory, and keep deliberate spill when neither posture is resident.
  110. """
  111. if mtp_capable and profile.kv_scale == 1.0:
  112. profile = replace(profile, kv_scale=1.2)
  113. initial: dict[bool, WindowDecision | PhysicsRefusal] = {}
  114. def candidate(stacked: bool) -> LaunchPlan:
  115. posture = posture_profile(profile, mtp_capable=mtp_capable, mtp_prefill=stacked)
  116. overhead = fixed_overhead + ub_logits_bytes(
  117. profile.n_vocab, mtp_capable=mtp_capable, mtp_prefill=stacked)
  118. decision = initial_window(posture, budget, overhead_bytes=overhead)
  119. initial[stacked] = decision
  120. if isinstance(decision, WindowDecision) and requested_window:
  121. target = min(requested_window, profile.n_ctx_train or requested_window)
  122. if target > decision.window and physics_check(
  123. posture, budget, target, overhead_bytes=overhead) is None:
  124. need = footprint_bytes(posture, target, overhead_bytes=overhead)
  125. decision = WindowDecision(
  126. window=target, spill_bytes=max(0, need - budget.usable_vram_bytes),
  127. kv_on_gpu=ctx_bytes(posture, target) + overhead <= budget.usable_vram_bytes,
  128. reasons=[f"grown window restored ({target // 1024}K)"])
  129. return LaunchPlan(decision, stacked, overhead, posture)
  130. lean = candidate(False)
  131. if not mtp_capable:
  132. return lean
  133. stacked = candidate(True)
  134. if isinstance(stacked.decision, PhysicsRefusal):
  135. return lean
  136. if isinstance(lean.decision, PhysicsRefusal):
  137. return stacked
  138. if stacked.decision.window < lean.decision.window:
  139. return lean
  140. if not stacked.decision.spilled:
  141. return stacked
  142. # A previously granted window keeps its spill policy unless lean can make it resident.
  143. stacked_initial, lean_initial = initial[True], initial[False]
  144. if (requested_window and isinstance(stacked_initial, WindowDecision)
  145. and isinstance(lean_initial, WindowDecision) and not stacked_initial.spilled
  146. and stacked_initial.window >= lean_initial.window
  147. and stacked.decision.window > stacked_initial.window and lean.decision.spilled):
  148. return stacked
  149. return lean
  150. def fit_to_free_memory(plan: LaunchPlan, profile: ModelProfile, live: HardwareBudget, *,
  151. mtp_capable: bool = False,
  152. fixed_overhead: int = RUNTIME_OVERHEAD_BYTES) -> LaunchPlan:
  153. """Narrow a resident capacity plan to the window the card's free memory holds now.
  154. Only the window moves, and only down to the floor. A plan that already spills keeps its
  155. placement, and so does a card too busy to hold even the floor: moving weights to the CPU on a
  156. live reading is how a launch once pinned a fitting model to the CPU, because the reading still
  157. counted memory the outgoing server was about to free.
  158. """
  159. decision = plan.decision
  160. if not isinstance(decision, WindowDecision) or decision.spilled:
  161. return plan
  162. now = plan_launch(profile, live, mtp_capable=mtp_capable, fixed_overhead=fixed_overhead)
  163. if isinstance(now.decision, WindowDecision) and not now.decision.spilled:
  164. if now.decision.window >= decision.window:
  165. return plan
  166. return LaunchPlan(replace(now.decision, reasons=[
  167. f"{now.decision.window // 1024}K fits beside other programs' GPU memory "
  168. f"({decision.window // 1024}K would not)"]), now.mtp_prefill, now.overhead_bytes, now.profile)
  169. floor = min(FLOOR, profile.n_ctx_train or FLOOR)
  170. if decision.window <= floor:
  171. return plan
  172. return LaunchPlan(WindowDecision(window=floor, spill_bytes=0, kv_on_gpu=True, reasons=[
  173. f"floor held at {floor // 1024}K; other programs hold most of the GPU memory"]),
  174. plan.mtp_prefill, plan.overhead_bytes, plan.profile)
  175. @dataclass
  176. class GrowthDecision:
  177. action: str # "grow" | "hold" | "compress-default"
  178. next_window: int | None = None
  179. reason: str = ""
  180. def growth_decision(profile: ModelProfile, budget: HardwareBudget, *,
  181. current_window: int, session_tokens: int, measured_decode_tok_s: float | None,
  182. server_idle: bool, flash_attention: bool = True,
  183. occupancy_confirmed: bool = False) -> GrowthDecision:
  184. """One growth evaluation, END-OF-TURN ONLY (recurrent state cannot rewind mid-sequence).
  185. Gate order: occupancy (~85%) → native cap → idleness (growth only on an otherwise-idle
  186. server) → speed floor (below it compression is the default) → re-fit against LIVE free memory
  187. (the rung must fit NOW, not at launch). ``occupancy_confirmed`` skips gate 1 when the caller's
  188. own compression gate already fired, so two edge definitions can't deadlock into
  189. compress-before-grow.
  190. """
  191. if not occupancy_confirmed and session_tokens < current_window * _GROW_AT_OCCUPANCY:
  192. return GrowthDecision("hold", reason="session below growth occupancy")
  193. native = profile.n_ctx_train or current_window
  194. if current_window >= native:
  195. return GrowthDecision("compress-default", reason="at native window; compression is the only move")
  196. if not server_idle:
  197. return GrowthDecision("hold", reason="server busy; re-grant deferred to idle")
  198. if measured_decode_tok_s is not None and measured_decode_tok_s < SPEED_FLOOR_TOK_S:
  199. return GrowthDecision(
  200. "compress-default",
  201. reason=(f"decode {measured_decode_tok_s:.1f} tok/s below the "
  202. f"~{SPEED_FLOOR_TOK_S:.0f} tok/s floor; growth is now an "
  203. "explicit per-session choice"))
  204. next_rung = next((r for r in ladder(native) if r > current_window), native)
  205. # Re-fit against live free memory: allocation beyond residency is the slow path, so a rung
  206. # that no longer fits doesn't get granted.
  207. kv = ctx_bytes(profile, next_rung, flash_attention=flash_attention)
  208. if profile.weights_bytes + kv > budget.usable_vram_bytes + budget.ram_available_bytes:
  209. return GrowthDecision("compress-default", reason="next rung exceeds physics; compression instead")
  210. return GrowthDecision("grow", next_window=next_rung,
  211. reason=f"rung {current_window // 1024}K -> {next_rung // 1024}K")
  212. def recurrent_spill_blocks(profile: ModelProfile, spill_bytes: int | None = None) -> list[int]:
  213. """Recurrent (n_head_kv==0) block indices whose FFN weights cover ``spill_bytes``.
  214. Takes the fewest blocks, lowest index first (llama.cpp's -ngl also keeps the front blocks on
  215. the host). Every recurrent block when ``spill_bytes`` is None, when any recurrent block's FFN
  216. size is unknown, or when all of them together still fall short (fit spills the rest).
  217. """
  218. recurrent = [i for i, (kind, _) in enumerate(profile.layers) if kind == LayerKind.RECURRENT]
  219. if spill_bytes is None or any(i not in profile.ffn_block_bytes for i in recurrent):
  220. return recurrent
  221. chosen: list[int] = []
  222. covered = 0
  223. for i in recurrent:
  224. if covered >= spill_bytes:
  225. break
  226. chosen.append(i)
  227. covered += profile.ffn_block_bytes[i]
  228. return chosen
  229. def spill_overrides(profile: ModelProfile, spill_bytes: int | None = None) -> list[str]:
  230. """-ot placement for spilled configs: expert/FFN weights to host so attention + KV stay
  231. GPU-resident. MoE gets the expert pattern; hybrids push the FFNs of just enough recurrent
  232. blocks to cover ``spill_bytes`` (their n_head_kv==0 layers carry no KV worth protecting, and
  233. full-attention FFNs stay on the GPU)."""
  234. if profile.moe:
  235. return ["-ot", r"blk\.\d+\.ffn_.*_exps\.weight=CPU"]
  236. blocks = recurrent_spill_blocks(profile, spill_bytes)
  237. if blocks:
  238. return ["-ot", r"blk\.({})\.ffn_.*\.weight=CPU".format("|".join(map(str, blocks)))]
  239. return [] # dense: fit's back-to-front layer cut is the only axis
  240. def launch_args(profile: ModelProfile, decision: WindowDecision, *, flash_attention: bool = True,
  241. mtp_capable: bool = False, mtp_draft_depth: int = 3, uma: bool = False,
  242. mtp_prefill: bool = False) -> list[str]:
  243. """Per-model launch flags from a window decision. Explicit -c puts fit into
  244. spill-weights-and-hold-ctx; q8 KV cache wherever flash attention exists; -ot placement on
  245. spilled configs — DISCRETE cards only."""
  246. args = ["-c", str(decision.window)]
  247. if mtp_capable:
  248. args += ["--spec-type", "draft-mtp", "--spec-draft-n-max", str(mtp_draft_depth),
  249. "--backend-sampling", "--spec-draft-backend-sampling"]
  250. if mtp_prefill:
  251. args += ["-b", "4096", "-ub", str(PREFILL_UBATCH)]
  252. else:
  253. args += ["-b", str(PREFILL_UBATCH), "-ub", str(PREFILL_UBATCH)]
  254. if flash_attention:
  255. args += ["-ctk", "q8_0", "-ctv", "q8_0", "-fa", "on"]
  256. if decision.spilled and not uma:
  257. args += spill_overrides(profile, decision.spill_bytes)
  258. return args
  259. def ub_logits_bytes(n_vocab: int, *, mtp_capable: bool, mtp_prefill: bool = False) -> int:
  260. """GPU logits/compute-buffer cost of the microbatch posture chosen by launch_args, priced from
  261. the model's own vocab and calibrated against measured server RSS (Qwen3.8 Q4, both postures,
  262. three windows)."""
  263. v = max(0, int(n_vocab))
  264. if mtp_capable and mtp_prefill:
  265. return int(2048 * v * 4 * 1.5)
  266. if mtp_capable:
  267. return 512 * v * 4 * 2
  268. return 2048 * v * 4