backend_identity.py 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. """Single owner for backend identity and failure-scoped skip decisions.
  2. Every fallback / dedup / skip / quarantine decision asks: "is this candidate the same backend
  3. as the one that failed, along the axis that failure invalidated?" Answering inline at each
  4. call site kept reintroducing the same bugs (same-shim aliases treated as distinct, sibling
  5. models skipped for one model's timeout, dedup ignoring ``base_url``). "provider" conflates
  6. three axes — credential surface (401/402), endpoint (DNS/refused), model deployment
  7. (timeout/overload/429). Build :class:`BackendIdentity` values, ask :func:`should_skip_candidate`.
  8. """
  9. from __future__ import annotations
  10. import logging
  11. from dataclasses import dataclass
  12. from enum import Enum
  13. from typing import Optional
  14. from hermes_cli.route_identity import normalize_route_base_url
  15. logger = logging.getLogger(__name__)
  16. class FailureScope(Enum):
  17. """Which identity axis a failure invalidates."""
  18. #: Timeout, overload/429, connection blip, model-incompatible, invalid response:
  19. #: evidence against ONE model deployment only.
  20. MODEL = "model"
  21. #: Auth 401 / payment 402: evidence against the shared credential.
  22. CREDENTIAL = "credential"
  23. #: DNS / connection-refused / unreachable host: evidence against the endpoint.
  24. ENDPOINT = "endpoint"
  25. def _norm(value: Optional[str]) -> str:
  26. return (value or "").strip().lower()
  27. @dataclass(frozen=True)
  28. class BackendIdentity:
  29. """Normalized identity of one (provider, model, endpoint) deployment.
  30. Empty fields mean "unknown" — an unknown axis can neither prove sameness nor difference
  31. on its own; the remaining axes decide."""
  32. provider: str = ""
  33. model: str = ""
  34. base_url: str = ""
  35. @classmethod
  36. def build(
  37. cls, provider: Optional[str] = None, model: Optional[str] = None,
  38. base_url: Optional[str] = None,
  39. ) -> BackendIdentity:
  40. return cls(
  41. provider=_norm(provider), model=_norm(model),
  42. base_url=normalize_route_base_url(base_url),
  43. )
  44. def _both_first_class(a: BackendIdentity, b: BackendIdentity) -> bool:
  45. """True when both providers are distinct registered first-class providers.
  46. Two different registry providers have distinct credential surfaces even when they share an
  47. inference host (xai-oauth vs xai). Custom/shim aliases are NOT in the registry, so two
  48. aliases pointing at one URL still count as the same backend."""
  49. if not a.provider or not b.provider or a.provider == b.provider:
  50. return False
  51. try:
  52. from hermes_cli.auth import PROVIDER_REGISTRY
  53. return a.provider in PROVIDER_REGISTRY and b.provider in PROVIDER_REGISTRY
  54. except Exception:
  55. return False
  56. def same_credential_surface(a: BackendIdentity, b: BackendIdentity) -> bool:
  57. """Do two identities share the credential a 401/402 just invalidated?
  58. Conservative: an unprovable axis answers "different" (one wasted RTT) rather than "same"
  59. (stranded failover). Same label = same configured credential; custom entries can each carry
  60. their own api_key, so a shared URL alone is only a weak signal when a label is missing."""
  61. if a.provider and b.provider:
  62. # Different labels = different credential config (first-class registry providers explicitly so —
  63. # #70893; custom entries can each carry their own api_key, so sameness is unprovable and we must not
  64. # skip).
  65. return a.provider == b.provider
  66. return bool(a.base_url and a.base_url == b.base_url)
  67. def same_endpoint(a: BackendIdentity, b: BackendIdentity) -> bool:
  68. """Do two identities sit behind the endpoint that just went unreachable?
  69. An unknown base_url inherits the provider default, so a shared label implies the same endpoint."""
  70. if a.base_url and b.base_url:
  71. return a.base_url == b.base_url
  72. return bool(a.provider and a.provider == b.provider)
  73. def same_deployment(a: BackendIdentity, b: BackendIdentity) -> bool:
  74. """Are these the exact same model deployment (the thing a timeout kills)?
  75. Provider+model must match; base_url distinguishes only when BOTH sides carry an explicit URL
  76. (same provider+model on two explicit URLs is a pool, not a dup). Different labels with the
  77. same URL + model are still one deployment (same-host shim aliases) — unless both labels are
  78. first-class registry providers."""
  79. if not (a.provider and b.provider and a.provider == b.provider):
  80. return bool(
  81. a.base_url
  82. # Same-host different-label shims: same URL + same model IS the same deployment even when the
  83. # alias labels differ (#22548) — unless both labels are first-class registry providers (#70893).
  84. and a.base_url == b.base_url
  85. and a.model
  86. and a.model == b.model
  87. and not _both_first_class(a, b)
  88. )
  89. if not (a.model and b.model and a.model == b.model):
  90. return False
  91. return not (a.base_url and b.base_url and a.base_url != b.base_url)
  92. _SCOPE_PREDICATES = {
  93. FailureScope.CREDENTIAL: same_credential_surface, FailureScope.ENDPOINT: same_endpoint,
  94. FailureScope.MODEL: same_deployment,
  95. }
  96. def should_skip_candidate(
  97. candidate: BackendIdentity, failed: BackendIdentity, scope: FailureScope = FailureScope.MODEL
  98. ) -> bool:
  99. """THE skip predicate: would trying ``candidate`` just repeat the failure?
  100. True when it is the same backend as ``failed`` along the axis ``scope`` invalidated.
  101. Every fallback/dedup/skip site must call this."""
  102. return _SCOPE_PREDICATES.get(scope, same_deployment)(candidate, failed)