growth.py 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144
  1. """In-session context growth for the managed llama.cpp runtime.
  2. Scope guard: only a server THIS process supervises grows. Detected external servers and
  3. other-process supervisors keep their own policies.
  4. """
  5. from __future__ import annotations
  6. from contextlib import suppress
  7. import json
  8. import logging
  9. logger = logging.getLogger(__name__)
  10. def window_overrides_path():
  11. from hermes_cli.local_runtime.binaries import runtimes_root
  12. return runtimes_root() / "window_overrides.json"
  13. def load_window_overrides() -> dict:
  14. """model_id -> granted window (int). Empty on any read problem."""
  15. try:
  16. with open(window_overrides_path(), encoding="utf-8-sig") as fh:
  17. data = json.load(fh)
  18. return {str(k): int(v) for k, v in data.items()}
  19. except Exception:
  20. return {}
  21. def _write_overrides(overrides: dict) -> None:
  22. path = window_overrides_path()
  23. path.parent.mkdir(parents=True, exist_ok=True)
  24. path.write_text(json.dumps(overrides, indent=1), encoding="utf-8")
  25. def save_window_override(model_id: str, window: int) -> None:
  26. overrides = load_window_overrides()
  27. overrides[model_id] = int(window)
  28. _write_overrides(overrides)
  29. def clear_window_override(model_id: str) -> None:
  30. """Drop a model's growth state (delete/re-download paths)."""
  31. overrides = load_window_overrides()
  32. if model_id in overrides:
  33. del overrides[model_id]
  34. _write_overrides(overrides)
  35. def is_managed_endpoint(base_url: str) -> bool:
  36. """True when base_url is the server this process's state file points at."""
  37. with suppress(Exception):
  38. from hermes_cli.local_runtime.endpoint import _state_endpoint
  39. state = _state_endpoint()
  40. return state is not None and (
  41. (base_url or "").rstrip("/") == str(state.get("base_url", "")).rstrip("/"))
  42. return False
  43. def maybe_grow_window(model_id: str, *, base_url: str, session_tokens: int,
  44. current_window: int,
  45. measured_decode_tok_s: float | None = None) -> int | None:
  46. """One growth evaluation + execution. Returns the NEW window when the ladder granted a bigger
  47. one, else None (hold / compress / not ours).
  48. The caller sits at a request boundary by construction (the pre-API compression gate), so
  49. re-prefill growth is safe at any call: the next request rebuilds server state in the larger
  50. window — nothing rewinds.
  51. """
  52. from hermes_cli.local_runtime.bootstrap import (
  53. _launch_budget, get_supervisor, refresh_local_runtime, staged_models)
  54. from hermes_cli.local_runtime.context_policy import growth_decision
  55. from hermes_cli.local_runtime.estimator import as_loaded, profile_from_gguf
  56. from hermes_cli.local_runtime.gguf import model_id_from_stem, read_gguf_header
  57. from hermes_cli.local_runtime.hardware import probe_budget
  58. from hermes_cli.local_runtime.presets import (
  59. preset_for_model, read_preset_decisions, resident_footprint)
  60. sup = get_supervisor()
  61. if sup is None or not is_managed_endpoint(base_url):
  62. return None
  63. gguf = next((p for p in staged_models() if model_id_from_stem(p.stem) == model_id), None)
  64. if gguf is None:
  65. return None
  66. try:
  67. profile = profile_from_gguf(read_gguf_header(gguf))
  68. except (ValueError, OSError) as exc:
  69. logger.debug("growth skip %s: unreadable gguf (%s)", model_id, exc)
  70. return None
  71. try:
  72. server_idle = sup.is_idle(model_id)
  73. except Exception:
  74. server_idle = False
  75. budget = probe_budget(planning=True)
  76. decision = growth_decision(
  77. # Capacity budget, not live-free: growth executes via a server bounce, so the grown
  78. # instance loads onto a freed card. Live-free is distorted by the very model being grown
  79. # — it reads its own residency as unavailable and vetoes rungs that fit.
  80. as_loaded(profile, budget), budget,
  81. current_window=current_window,
  82. session_tokens=session_tokens,
  83. measured_decode_tok_s=measured_decode_tok_s,
  84. server_idle=server_idle,
  85. # The caller IS the occupancy signal: this runs from the agent's compression gate, which
  86. # fired on its own threshold. Two separately-derived edges must not deadlock into
  87. # compress-before-grow.
  88. occupancy_confirmed=True,
  89. )
  90. if decision.action != "grow" or not decision.next_window:
  91. logger.debug("growth %s: %s (%s)", model_id, decision.action, decision.reason)
  92. return None
  93. # The grown instance loads after this one exits, so the model's own memory counts as free,
  94. # draft included when its running preset has one. Other loaded models still count as held,
  95. # which errs toward a smaller window.
  96. running = read_preset_decisions().get(model_id)
  97. drafting = running is not None and (running.keys or {}).get("spec-type") == "draft-dflash"
  98. own_bytes = resident_footprint(gguf, budget, current_window, drafting=drafting) or 0
  99. live = _launch_budget(budget, own_bytes=own_bytes)
  100. plan = preset_for_model(gguf, budget, set(), requested_window=decision.next_window, live=live)
  101. if plan is None or plan.refusal or plan.window < decision.next_window:
  102. logger.debug("growth %s: the next rung does not fit beside other programs' GPU memory",
  103. model_id)
  104. return None
  105. logger.info("context growth %s: %s", model_id, decision.reason)
  106. save_window_override(model_id, decision.next_window)
  107. if not refresh_local_runtime():
  108. # The override still lands at the next boot; report no growth NOW so the caller
  109. # compresses instead of overflowing a stale window.
  110. logger.warning("growth %s: server refresh failed; compression proceeds", model_id)
  111. return None
  112. materialized = read_preset_decisions().get(model_id)
  113. if materialized is None or materialized.window < decision.next_window:
  114. logger.warning("growth %s: refreshed preset did not grant the requested window", model_id)
  115. return None
  116. return materialized.window