empty_response_guard.py 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243
  1. """Deterministic-empty detection and cost-aware retry budgets.
  2. On an empty completion the loop retries up to 3 times, then walks the fallback chain —
  3. each attempt re-bills the full input. Signaled refusals (``content_filter``, Anthropic
  4. ``refusal``, Bedrock guardrails) are already terminal; this handles *unsignaled* empties
  5. (success, zero output tokens, generic finish reason — typical of portal-proxied refusals).
  6. Two independent guards, both failing OPEN to legacy behaviour:
  7. 1. Deterministic-empty: two consecutive empties, both with usage present and
  8. ``output_tokens == 0``, from the same (model, provider, finish_reason) → skip the
  9. remaining retries and go straight to the fallback chain. Missing usage or
  10. ``output_tokens > 0`` (think-block stripping, whitespace, flaky decoding) never classifies.
  11. 2. Cost-aware budget: when one empty attempt's estimated input cost exceeds the threshold
  12. (default $0.25), the retry budget drops from 3 to 1. Unknown pricing / missing usage /
  13. included routes leave it untouched.
  14. 1. **Deterministic-empty detection** — two consecutive empty attempts from
  15. the same (model, provider, finish_reason) are treated as deterministic
  16. when usage proves zero output, or when usage is absent and the assembled
  17. responses contain neither content nor reasoning. Remaining retries are
  18. skipped and the loop proceeds straight to the fallback chain (a different
  19. model may behave differently). Mixed evidence or any generated tokens keep
  20. the full retry budget.
  21. 2. **Cost-aware retry budget** — when the estimated input cost of a
  22. single empty attempt exceeds the configured threshold (default
  23. $0.25), the empty-retry budget for this streak drops from 3 to 1.
  24. Unknown pricing, missing usage, or included/subscription routes
  25. leave the budget untouched.
  26. Configured via the additive ``agent.empty_response_guard`` section in
  27. ``config.yaml`` (resolved once at agent init by ``agent_init``)::
  28. agent:
  29. empty_response_guard:
  30. enabled: true # false = legacy fixed 3-retry behaviour
  31. cost_threshold_usd: 0.25 # per-attempt cost that halves the budget
  32. Per project policy, no ``HERMES_*`` environment variables are involved —
  33. ``.env`` is reserved for credentials; behavioural settings live in
  34. ``config.yaml``.
  35. """
  36. from __future__ import annotations
  37. import logging
  38. from dataclasses import dataclass
  39. from decimal import Decimal
  40. from typing import Any, List, Optional, Tuple
  41. logger = logging.getLogger(__name__)
  42. DEFAULT_EMPTY_RETRY_BUDGET = 3
  43. REDUCED_EMPTY_RETRY_BUDGET = 1
  44. DEFAULT_COST_THRESHOLD_USD = Decimal("0.25")
  45. DEFAULT_GUARD_ENABLED = True
  46. # Agent-object attribute names. State is scoped to one consecutive empty streak: cleared
  47. # whenever ``_empty_content_retries == 0`` at record time, so every existing counter-reset
  48. # site (turn start, compaction, tool success, fallback activation) is honoured.
  49. _ATTEMPTS_ATTR = "_empty_attempt_history"
  50. _STREAK_COST_ATTR = "_empty_streak_cost_usd"
  51. _ENABLED_ATTR = "_empty_guard_enabled"
  52. _THRESHOLD_ATTR = "_empty_guard_cost_threshold_usd"
  53. @dataclass(frozen=True)
  54. class EmptyAttempt:
  55. """One observed empty completion within the current streak."""
  56. model: str
  57. provider: str
  58. finish_reason: str
  59. usage_present: bool
  60. zero_output: bool
  61. observed_generation: bool
  62. @property
  63. def signature(self) -> tuple:
  64. return (self.model, self.provider, self.finish_reason)
  65. def resolve_guard_settings(section: Any) -> tuple[bool, Decimal]:
  66. """Resolve ``agent.empty_response_guard`` into (enabled, threshold); malformed input → schema defaults."""
  67. if not isinstance(section, dict):
  68. return (DEFAULT_GUARD_ENABLED, DEFAULT_COST_THRESHOLD_USD)
  69. enabled = section.get("enabled", DEFAULT_GUARD_ENABLED)
  70. if isinstance(enabled, str): # YAML quoting can turn true/false into strings.
  71. enabled = enabled.strip().lower() not in ("0", "false", "no", "off")
  72. elif not isinstance(enabled, bool):
  73. enabled = DEFAULT_GUARD_ENABLED
  74. threshold = DEFAULT_COST_THRESHOLD_USD
  75. threshold_raw = section.get("cost_threshold_usd")
  76. if threshold_raw is not None and not isinstance(threshold_raw, bool):
  77. try:
  78. candidate = Decimal(str(threshold_raw))
  79. if candidate > 0:
  80. threshold = candidate
  81. except Exception:
  82. logger.debug("empty-guard: invalid cost_threshold_usd %r, using default", threshold_raw)
  83. return (enabled, threshold)
  84. def guard_enabled(agent: Any) -> bool:
  85. """Config-resolved enabled flag; agents built without config default to enabled."""
  86. value = getattr(agent, _ENABLED_ATTR, DEFAULT_GUARD_ENABLED)
  87. return value if isinstance(value, bool) else DEFAULT_GUARD_ENABLED
  88. def _cost_threshold_usd(agent: Any) -> Decimal:
  89. value = getattr(agent, _THRESHOLD_ATTR, None)
  90. return value if isinstance(value, Decimal) and value > 0 else DEFAULT_COST_THRESHOLD_USD
  91. def _attempts(agent: Any) -> list[EmptyAttempt]:
  92. attempts = getattr(agent, _ATTEMPTS_ATTR, None)
  93. if attempts is None:
  94. attempts = []
  95. setattr(agent, _ATTEMPTS_ATTR, attempts)
  96. return attempts
  97. def _normalized_usage(agent: Any, response: Any, what: str) -> Any:
  98. """Canonical usage for ``response`` or None (no usage / normalization failed)."""
  99. raw_usage = getattr(response, "usage", None)
  100. if not raw_usage:
  101. return None
  102. try:
  103. from agent.usage_pricing import normalize_usage
  104. return normalize_usage(raw_usage, provider=getattr(agent, "provider", None),
  105. api_mode=getattr(agent, "api_mode", None))
  106. except Exception:
  107. logger.debug("empty-guard: %s failed", what, exc_info=True)
  108. return None
  109. def _estimate_attempt_cost(agent: Any, response: Any) -> Optional[Decimal]:
  110. """Best-effort USD estimate for one attempt. None when unknown."""
  111. canonical = _normalized_usage(agent, response, "cost estimation")
  112. if canonical is None:
  113. return None
  114. try:
  115. from agent.usage_pricing import estimate_usage_cost
  116. result = estimate_usage_cost(
  117. getattr(agent, "model", "") or "", canonical, provider=getattr(agent, "provider", None),
  118. base_url=getattr(agent, "base_url", None), api_key=getattr(agent, "api_key", None),
  119. )
  120. except Exception:
  121. logger.debug("empty-guard: cost estimation failed", exc_info=True)
  122. return None
  123. return getattr(result, "amount_usd", None)
  124. def _zero_output(agent: Any, response: Any) -> tuple:
  125. """Return (usage_present, zero_output) for a response, failing open."""
  126. canonical = _normalized_usage(agent, response, "usage normalization")
  127. if canonical is None:
  128. return (False, False)
  129. output = getattr(canonical, "output_tokens", None)
  130. # A present-but-empty usage object (some proxies) normalizes to all zeros;
  131. # a genuine completion always has input tokens — no evidence, fail open.
  132. if output is None or getattr(canonical, "prompt_tokens", 0) <= 0:
  133. return (False, False)
  134. # Reasoning tokens are real generation: a reasoning-only response is NOT
  135. # a deterministic empty (the prefill-continuation path owns that case).
  136. reasoning = getattr(canonical, "reasoning_tokens", 0) or 0
  137. return (True, (output + reasoning) == 0)
  138. def record_empty_attempt(
  139. agent: Any,
  140. *,
  141. finish_reason: str,
  142. response: Any,
  143. observed_generation: bool = True,
  144. ) -> None:
  145. """Record one empty completion in the current streak.
  146. Call BEFORE ``_empty_content_retries`` is incremented: a counter of 0 marks a new
  147. streak and clears prior history."""
  148. attempts = _attempts(agent)
  149. if getattr(agent, "_empty_content_retries", 0) == 0:
  150. attempts.clear()
  151. setattr(agent, _STREAK_COST_ATTR, Decimal(0))
  152. usage_present, zero_output = _zero_output(agent, response)
  153. attempts.append(
  154. EmptyAttempt(
  155. model=str(getattr(agent, "model", "") or ""),
  156. provider=str(getattr(agent, "provider", "") or ""),
  157. finish_reason=str(finish_reason or ""),
  158. usage_present=usage_present,
  159. zero_output=zero_output,
  160. observed_generation=bool(observed_generation),
  161. )
  162. )
  163. cost = _estimate_attempt_cost(agent, response)
  164. if cost is not None and cost > 0:
  165. prior = getattr(agent, _STREAK_COST_ATTR, Decimal(0)) or Decimal(0)
  166. setattr(agent, _STREAK_COST_ATTR, prior + cost)
  167. def deterministic_empty(agent: Any) -> bool:
  168. """True when the current streak looks deterministic.
  169. Requires >= 2 consecutive attempts with an identical (model, provider,
  170. finish_reason) signature. Usage-backed attempts must all prove zero output.
  171. Usage-absent attempts must all have no observed content or reasoning. Mixed
  172. evidence fails open so ambiguous transients keep their retries.
  173. """
  174. if not guard_enabled(agent):
  175. return False
  176. attempts = getattr(agent, _ATTEMPTS_ATTR, None) or []
  177. if len(attempts) < 2:
  178. return False
  179. first = attempts[0]
  180. same_signature = all(a.signature == first.signature for a in attempts)
  181. usage_proves_empty = all(a.usage_present and a.zero_output for a in attempts)
  182. response_proves_empty = all(
  183. not a.usage_present and not a.observed_generation for a in attempts
  184. )
  185. return same_signature and (usage_proves_empty or response_proves_empty)
  186. def empty_retry_budget(agent: Any, response: Any) -> int:
  187. """Empty-retry budget for the current streak (3, or 1 when a single attempt is
  188. estimated to cost more than the configured threshold)."""
  189. if not guard_enabled(agent):
  190. return DEFAULT_EMPTY_RETRY_BUDGET
  191. cost = _estimate_attempt_cost(agent, response)
  192. if cost is not None and cost >= _cost_threshold_usd(agent):
  193. return REDUCED_EMPTY_RETRY_BUDGET
  194. return DEFAULT_EMPTY_RETRY_BUDGET
  195. def streak_cost_usd(agent: Any) -> Optional[Decimal]:
  196. """Accumulated estimated cost of the current empty streak, if known."""
  197. cost = getattr(agent, _STREAK_COST_ATTR, None)
  198. return cost if cost is not None and cost > 0 else None