auxiliary_health.py 3.6 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273
  1. """Endpoint identity for auxiliary custom-provider health checks."""
  2. import contextlib
  3. from typing import Any, Optional
  4. from hermes_cli.route_identity import normalize_route_base_url
  5. def _unhealthy_cache_key(provider: str, base_url: Optional[str] = None) -> Any:
  6. """Provider-wide key, or endpoint-specific key for an explicit custom endpoint — prefixed with the
  7. active profile home: a 402 on profile A's account must not hide the provider from profile B's
  8. (differently funded) account in the same multiplexed process."""
  9. from agent.auxiliary_client import _normalize_chain_label
  10. from hermes_constants import hermes_home_key
  11. label = _normalize_chain_label(provider)
  12. endpoint = normalize_route_base_url(_custom_health_base_url(provider, base_url))
  13. home_key = hermes_home_key()
  14. if endpoint:
  15. return home_key, "custom-endpoint", endpoint
  16. return home_key, label
  17. def _custom_health_base_url(provider: str, explicit_base_url: Optional[str] = None) -> str:
  18. """Return the concrete custom endpoint used to scope health and failed-route checks."""
  19. from agent.auxiliary_client import _current_custom_base_url
  20. explicit = str(explicit_base_url or "").strip()
  21. from agent.auxiliary_client import _normalize_chain_label
  22. label = _normalize_chain_label(provider)
  23. if label == "local/custom":
  24. return explicit or _current_custom_base_url()
  25. if label.startswith("custom:") and explicit:
  26. return explicit
  27. with contextlib.suppress(ImportError):
  28. from hermes_cli.runtime_provider import _get_named_custom_provider, _resolves_to_custom
  29. if _resolves_to_custom(label):
  30. return explicit or _current_custom_base_url()
  31. entry = _get_named_custom_provider(provider)
  32. if entry:
  33. return explicit or str(entry.get("base_url") or "").strip()
  34. return ""
  35. def fallback_candidate_unavailable_reason(exc: Exception) -> Optional[str]:
  36. """Why a fallback candidate cannot serve this walk (``_FALLBACK_REASONS`` label), or None.
  37. The same capacity classes that admitted the primary failure into the chain (payment/quota,
  38. rate limit, connection, route-incompatible model, malformed response) mean "this lane is out
  39. for now, try the next configured one"; anything else (a 400 request-shape error, a ValueError)
  40. is the caller's bug and must still propagate. Auth errors are excluded on purpose: they have
  41. their own refresh-then-quarantine path in the candidate helpers (#106367)."""
  42. from agent.auxiliary_client import _FALLBACK_REASONS
  43. return next(
  44. (label for predicate, label in _FALLBACK_REASONS if label != "auth error" and predicate(exc)),
  45. None,
  46. )
  47. # Quarantine hold per unavailable-reason label. Payment/quota depletion and a dead credential
  48. # last hours, so those keep the long default TTL (None); a per-minute 429, a dropped connection or
  49. # a garbled body clears in seconds — holding the lane for 10 minutes process-wide would hide a
  50. # healthy fallback from every aux task over one transient blip.
  51. _TRANSIENT_CANDIDATE_QUARANTINE_SECONDS = 60.0
  52. _CANDIDATE_QUARANTINE_TTL: dict[str, Optional[float]] = {
  53. "rate limit": _TRANSIENT_CANDIDATE_QUARANTINE_SECONDS,
  54. "connection error": _TRANSIENT_CANDIDATE_QUARANTINE_SECONDS,
  55. "invalid provider response": _TRANSIENT_CANDIDATE_QUARANTINE_SECONDS,
  56. }
  57. def fallback_candidate_quarantine_ttl(reason: Optional[str]) -> Optional[float]:
  58. """Seconds to hide a fallback candidate for ``reason`` (a ``_FALLBACK_REASONS`` label, or None
  59. for a stale credential); None means the long default TTL."""
  60. return _CANDIDATE_QUARANTINE_TTL.get(reason or "")