image_token_cost.py 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149
  1. """Per-image token cost learned from the provider's own usage, never from a vendor formula.
  2. A flat per-image constant is wrong in both directions: a 1920x1080 screenshot costs ~1,100 tokens
  3. on one provider and 4,000+ on a local mmproj model. The provider prices every image exactly on the
  4. request that carries it, so the cost is observable: with a fresh usage anchor (real prompt count of
  5. the previous response), the residual between the next real ``prompt_tokens`` and
  6. ``anchor + text-only delta`` is the price of the N images that delta introduced (#70328).
  7. The learned value is kept per ``model@host`` in ``~/.hermes/cache/image_token_costs.json`` so a new
  8. session starts calibrated, and bound per turn through a ContextVar so every estimator
  9. (preflight trigger, tail-budget walk, gateway hygiene) prices images the same way.
  10. """
  11. from __future__ import annotations
  12. import contextlib
  13. import logging
  14. from contextvars import ContextVar
  15. from typing import Any, Dict, List, Optional
  16. logger = logging.getLogger(__name__)
  17. DEFAULT_IMAGE_TOKEN_COST = 1500
  18. # Observations outside this band are text-estimate noise, not an image price.
  19. _MIN_PLAUSIBLE, _MAX_PLAUSIBLE = 64, 32_768
  20. _EMA_ALPHA = 0.5
  21. _image_cost_var: ContextVar[Optional[int]] = ContextVar("hermes_image_token_cost", default=None)
  22. _LEARNED: dict[str, int] = {}
  23. _LOADED = False
  24. # Routed profiles (multiplexed gateway) keep their own table, loaded from THEIR cache file: the
  25. # module slot above is the launch profile's and would otherwise be persisted into every home.
  26. _LEARNED_BY_HOME: dict[str, dict[str, int]] = {}
  27. def _cache_path():
  28. from agent.model_metadata import _cache_file
  29. return _cache_file("image_token_costs.json")
  30. def _key(model: Any, base_url: Any) -> str:
  31. from utils import base_url_hostname
  32. return f"{model or ''}@{base_url_hostname(base_url or '') or ''}"
  33. def _read_cache() -> dict[str, int]:
  34. from agent.model_metadata import _load_json_dict
  35. return {k: v for k, v in _load_json_dict(_cache_path()).items()
  36. if isinstance(v, int) and _MIN_PLAUSIBLE <= v <= _MAX_PLAUSIBLE}
  37. def _table() -> dict[str, int]:
  38. """The active profile's learned table, loaded lazily from its cache file."""
  39. global _LOADED
  40. from hermes_constants import get_hermes_home_override, hermes_home_key
  41. if get_hermes_home_override() is None:
  42. if not _LOADED:
  43. _LOADED = True
  44. _LEARNED.update(_read_cache())
  45. return _LEARNED
  46. home_key = hermes_home_key()
  47. table = _LEARNED_BY_HOME.get(home_key)
  48. if table is None:
  49. table = _LEARNED_BY_HOME[home_key] = _read_cache()
  50. return table
  51. def learned_image_token_cost(model: Any, base_url: Any) -> int:
  52. """Learned per-image cost for ``model@host``, else the flat default."""
  53. return _table().get(_key(model, base_url), DEFAULT_IMAGE_TOKEN_COST)
  54. def current_image_token_cost() -> int:
  55. """Per-image cost bound for the running turn (see :func:`image_cost_context`), else the default."""
  56. bound = _image_cost_var.get()
  57. return bound if bound is not None else DEFAULT_IMAGE_TOKEN_COST
  58. @contextlib.contextmanager
  59. def image_cost_context(cost: Optional[int]):
  60. token = _image_cost_var.set(cost)
  61. try:
  62. yield
  63. finally:
  64. _image_cost_var.reset(token)
  65. def bind_image_token_cost(agent: Any) -> None:
  66. """Bind the agent's learned per-image cost to the current context for the rest of the turn."""
  67. _image_cost_var.set(learned_image_token_cost(getattr(agent, "model", None), getattr(agent, "base_url", None)))
  68. def count_images(messages: list[dict[str, Any]]) -> int:
  69. from agent.model_metadata import _count_image_tokens
  70. return sum(_count_image_tokens(m, 1) for m in messages if isinstance(m, dict))
  71. def calibrate_from_usage(agent: Any, messages: list[dict[str, Any]], prompt_tokens: Any) -> Optional[int]:
  72. """Learn the per-image cost from the response that just priced ``messages``.
  73. Requires the PREVIOUS anchor (real count of the prior request) to still match: the residual
  74. ``prompt_tokens - (anchor + text-only delta)`` is then the provider's price for the images the
  75. delta introduced. Returns the new learned cost, or None when this response teaches nothing
  76. (no anchor, no new images, implausible residual)."""
  77. from agent.usage_anchor import anchored_context_tokens
  78. anchor = getattr(agent, "_usage_anchor", None)
  79. try:
  80. real = int(prompt_tokens or 0)
  81. except (TypeError, ValueError):
  82. return None
  83. if real <= 0 or not isinstance(anchor, dict) or not isinstance(messages, list):
  84. return None
  85. base_count = int(anchor.get("base_count") or 0)
  86. delta = messages[base_count:]
  87. if delta and isinstance(delta[0], dict) and delta[0].get("role") == "assistant":
  88. delta = delta[1:]
  89. n_images = count_images(delta)
  90. if n_images <= 0:
  91. return None
  92. with image_cost_context(0):
  93. text_only = anchored_context_tokens(messages, anchor)
  94. if text_only is None:
  95. return None
  96. per_image = (real - text_only) // n_images
  97. if not _MIN_PLAUSIBLE <= per_image <= _MAX_PLAUSIBLE:
  98. return None
  99. key = _key(getattr(agent, "model", None), getattr(agent, "base_url", None))
  100. table = _table()
  101. prior = table.get(key)
  102. learned = per_image if prior is None else int(prior + _EMA_ALPHA * (per_image - prior))
  103. table[key] = learned
  104. _image_cost_var.set(learned)
  105. try:
  106. from utils import atomic_json_write
  107. atomic_json_write(_cache_path(), dict(table), indent=0, separators=(",", ":"))
  108. except Exception:
  109. logger.debug("image token cost persist failed", exc_info=True)
  110. logger.info(
  111. "Image token cost calibrated from provider usage: %s images priced %s tokens each (learned %s for %s)",
  112. n_images, f"{per_image:,}", f"{learned:,}", key,
  113. )
  114. return learned