model_cost_guard.py 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. """Expensive-model confirmation helpers for model selection surfaces."""
  2. from __future__ import annotations
  3. from dataclasses import dataclass
  4. from decimal import Decimal, InvalidOperation
  5. from typing import Optional
  6. from agent.models_dev import ModelInfo, models_dev_id_for
  7. INPUT_COST_WARNING_THRESHOLD = Decimal(20)
  8. OUTPUT_COST_WARNING_THRESHOLD = Decimal(100)
  9. GPT55_PRO_OPENROUTER_ID = "openai/gpt-5.5-pro"
  10. GPT55_SUGGESTION = "did you mean to select openai/gpt-5.5?"
  11. @dataclass(frozen=True)
  12. class ExpensiveModelWarning:
  13. """Confirmation payload for models above Hermes' cost guardrail."""
  14. model: str
  15. provider: str
  16. input_cost_per_million: Optional[Decimal]
  17. output_cost_per_million: Optional[Decimal]
  18. source: str
  19. message: str
  20. def _to_decimal(value: object) -> Optional[Decimal]:
  21. try:
  22. return None if value is None else Decimal(str(value))
  23. except (InvalidOperation, ValueError):
  24. return None
  25. def _format_money(value: Optional[Decimal]) -> str:
  26. return "unknown" if value is None else f"${value:.2f}/M"
  27. def _pricing_from_model_info(
  28. model_info: Optional[ModelInfo]) -> tuple[Optional[Decimal], Optional[Decimal], str]:
  29. if model_info is None or not model_info.has_cost_data():
  30. return None, None, ""
  31. return _to_decimal(model_info.cost_input), _to_decimal(model_info.cost_output), "models.dev"
  32. def _known_models_dev_provider(provider: Optional[str]) -> Optional[str]:
  33. normalized = (provider or "").strip().lower()
  34. return models_dev_id_for(normalized) if normalized else None
  35. def _can_trust_model_info_pricing(
  36. provider: Optional[str], model_info: Optional[ModelInfo]) -> bool:
  37. expected_provider = _known_models_dev_provider(provider)
  38. if not expected_provider or model_info is None:
  39. return False
  40. actual_provider = str(getattr(model_info, "provider_id", "") or "").strip().lower()
  41. return not actual_provider or actual_provider == expected_provider
  42. def _can_trust_pricing_lookup(
  43. model_name: str, *, provider: Optional[str], base_url: Optional[str]) -> bool:
  44. try:
  45. from agent.usage_pricing import resolve_billing_route
  46. route = resolve_billing_route(model_name, provider=provider, base_url=base_url)
  47. except Exception:
  48. return False
  49. return route.billing_mode != "unknown"
  50. def expensive_model_warning(
  51. model_name: str, *, provider: Optional[str] = None, base_url: Optional[str] = None,
  52. api_key: Optional[str] = None, model_info: Optional[ModelInfo] = None,
  53. ) -> Optional[ExpensiveModelWarning]:
  54. """Warning payload when KNOWN pricing exceeds the safety thresholds (never fires on unknown
  55. pricing). Call after model resolution so aliases / provider-specific ids have settled."""
  56. model = (model_name or "").strip()
  57. if not model:
  58. return None
  59. input_cost: Optional[Decimal] = None
  60. output_cost: Optional[Decimal] = None
  61. source = ""
  62. if _can_trust_model_info_pricing(provider, model_info):
  63. input_cost, output_cost, source = _pricing_from_model_info(model_info)
  64. def _unpriced() -> bool:
  65. return input_cost is None and output_cost is None
  66. if _unpriced() and _known_models_dev_provider(provider):
  67. try:
  68. from agent.models_dev import get_model_info
  69. input_cost, output_cost, source = _pricing_from_model_info(get_model_info(provider, model))
  70. except Exception:
  71. pass
  72. if _unpriced() and _can_trust_pricing_lookup(model, provider=provider, base_url=base_url):
  73. try:
  74. from agent.usage_pricing import get_pricing_entry
  75. entry = get_pricing_entry(model, provider=provider, base_url=base_url, api_key=api_key)
  76. except Exception:
  77. entry = None
  78. if entry is not None:
  79. input_cost = entry.input_cost_per_million
  80. output_cost = entry.output_cost_per_million
  81. source = entry.source
  82. is_known_gpt55_pro_confusion = model.lower() == GPT55_PRO_OPENROUTER_ID
  83. over_input = input_cost is not None and input_cost > INPUT_COST_WARNING_THRESHOLD
  84. over_output = output_cost is not None and output_cost > OUTPUT_COST_WARNING_THRESHOLD
  85. if not over_input and not over_output and not is_known_gpt55_pro_confusion:
  86. return None
  87. lines = [
  88. "!!! EXPENSIVE MODEL WARNING !!!",
  89. "",
  90. f"{model} has known pricing above Hermes' safety threshold.",
  91. f"Input tokens: {_format_money(input_cost)}",
  92. f"Output tokens: {_format_money(output_cost)}",
  93. "Threshold: more than $20/M input tokens or more than $100/M output tokens."]
  94. if source:
  95. lines.append(f"Pricing source: {source}.")
  96. if is_known_gpt55_pro_confusion:
  97. lines.append(GPT55_SUGGESTION)
  98. lines.append("Confirm only if you intend to use this model.")
  99. return ExpensiveModelWarning(
  100. model=model, provider=(provider or "").strip(), input_cost_per_million=input_cost,
  101. output_cost_per_million=output_cost, source=source or "unknown", message="\n".join(lines))