bot_failure_reasons.py 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150
  1. """Typed failure-reason codes for bot turns and relay replies.
  2. A closed vocabulary of machine-readable reason codes carried ALONGSIDE the
  3. free-text ``error`` fields (additive — old consumers keep working). Platform-side
  4. codes are assigned by the transport/relay layer; agent-side codes are derived
  5. from raw agent/provider error text via ``classify_agent_error``. Classifier
  6. precedence is the order of ``_RULES``: auth outranks quota by design — real
  7. provider 401 bodies (e.g. Anthropic) say "invalid, blocked or out of funds".
  8. """
  9. from __future__ import annotations
  10. import re
  11. from typing import Any
  12. # platform-side
  13. RUNTIME_OFFLINE = "runtime_offline"
  14. QUEUED_EXPIRED = "queued_expired"
  15. DELIVERY_TIMEOUT = "delivery_timeout"
  16. AGENT_BLOCKED = "agent_blocked"
  17. CANCELLED = "cancelled"
  18. # agent-side
  19. PROVIDER_AUTH_OR_ACCESS = "provider_auth_or_access"
  20. PROVIDER_QUOTA_LIMIT = "provider_quota_limit"
  21. PROVIDER_RATE_LIMIT = "provider_rate_limit"
  22. PROVIDER_SERVER_ERROR = "provider_server_error"
  23. CONTEXT_OVERFLOW = "context_overflow"
  24. MISSING_CONFIG = "missing_config"
  25. MODEL_UNAVAILABLE = "model_unavailable"
  26. # A spawn site could not resolve the target profile's secret scope: no turn ran, no provider was called.
  27. TARGET_SCOPE_UNRESOLVED = "target_scope_unresolved"
  28. UNKNOWN = "unknown"
  29. ALL_REASONS = frozenset({
  30. RUNTIME_OFFLINE, QUEUED_EXPIRED, DELIVERY_TIMEOUT, AGENT_BLOCKED, CANCELLED,
  31. PROVIDER_AUTH_OR_ACCESS, PROVIDER_QUOTA_LIMIT, PROVIDER_RATE_LIMIT,
  32. PROVIDER_SERVER_ERROR, CONTEXT_OVERFLOW, MISSING_CONFIG, MODEL_UNAVAILABLE,
  33. TARGET_SCOPE_UNRESOLVED, UNKNOWN,
  34. })
  35. # Not a bot-turn reason (kept out of ALL_REASONS): the runtime refusal code a Bot Chat delivery id's
  36. # settled not-sent receipt carries (``gateway/session_bot.py:_not_sent_record``). The Bot DM tool
  37. # recognises that definite no-send by it, so owner and tool share this one literal.
  38. HELD_NOT_SENT = "runtime_coordination_required"
  39. #: Reasons a supervisor may retry automatically without human intervention.
  40. AUTO_RETRYABLE = frozenset({RUNTIME_OFFLINE, DELIVERY_TIMEOUT, PROVIDER_RATE_LIMIT, PROVIDER_SERVER_ERROR})
  41. def is_auto_retryable(reason: str) -> bool:
  42. return reason in AUTO_RETRYABLE
  43. # Retry session policy: a retried bot turn NEVER mints a fresh session. Transient
  44. # classes resume as-is; context_overflow runs context compression (the one
  45. # sanctioned context mutation) on the same session first; everything else
  46. # (auth/quota/config/model/unknown) is never auto-retried — it can't be fixed by
  47. # a retry and only burns quota.
  48. # See #93091.
  49. RETRY_RESUME = "resume"
  50. RETRY_COMPRESS_THEN_RESUME = "compress_then_resume"
  51. RETRY_NONE = "none"
  52. def retry_action(reason: str) -> str:
  53. """Map a failure reason to the bot-turn retry action (see policy above)."""
  54. if reason in AUTO_RETRYABLE:
  55. return RETRY_RESUME
  56. if reason == CONTEXT_OVERFLOW:
  57. return RETRY_COMPRESS_THEN_RESUME
  58. return RETRY_NONE
  59. _STATUS = r"(?:error code:?\s*|status(?:\s*code)?:?\s*|http\s*)"
  60. # Ordered (pattern, code) — first match wins.
  61. _RULES: tuple[tuple[re.Pattern[str], str], ...] = tuple(
  62. (re.compile(pat, re.IGNORECASE), code)
  63. for pat, code in (
  64. # FIRST: the copy says "API key" and "restart", so it must be claimed before the auth rules.
  65. # ``UnscopedSecretError`` names the secret when it has one ("this profile's OPENROUTER_API_KEY");
  66. # the ``served_profile_child_env`` detail rides in tracebacks.
  67. (r"could not read this profile's [^(]+\(an internal profile-scoping bug"
  68. r"|served_profile_child_env\(inherit_credentials=true\)", TARGET_SCOPE_UNRESOLVED),
  69. (rf"authentication_error|invalid api key|{_STATUS}(?:401|403)\b", PROVIDER_AUTH_OR_ACCESS),
  70. (rf"{_STATUS}402\b|out of funds|quota|balance", PROVIDER_QUOTA_LIMIT),
  71. (rf"{_STATUS}429\b|rate.?limit", PROVIDER_RATE_LIMIT),
  72. # ``server[ _]?error`` / ``overloaded_error`` are the providers' own JSON spellings of the
  73. # agent's typed ``server_error`` / ``overloaded`` verdicts (FailoverReason), which the
  74. # in-process lanes classify from ``failure_reason`` rather than a status number.
  75. (rf"{_STATUS}5\d{{2}}\b|server[ _]?error|overloaded", PROVIDER_SERVER_ERROR),
  76. (r"context length|context_overflow|maximum context", CONTEXT_OVERFLOW),
  77. (r"no llm provider configured|missing config|no access token", MISSING_CONFIG),
  78. (r"model .*(not found|does not exist)|model_not_found", MODEL_UNAVAILABLE),
  79. )
  80. )
  81. def turn_failure_text(stdout: str | None, stderr: str | None) -> str:
  82. """The error text of a failed ``hermes … -Q`` delivery turn: both streams, in the order the CLI
  83. writes them. The provider prose is the turn's final_response and lands on STDOUT; stderr carries
  84. session bookkeeping (``session_id: …``) on every run, so ``stderr or stdout`` only ever saw the
  85. banner and every transient failure classified as ``unknown``.
  86. The in-process lanes hand over the same two pieces of prose as an agent result dict
  87. (``error`` + ``failure_reason``); ``result_retry_action`` joins them here too, so one classifier
  88. sees every lane's failure text."""
  89. return "\n".join(text.strip() for text in (stdout, stderr) if text and text.strip())
  90. def result_retry_action(result: Any) -> str:
  91. """The retry action for a finished IN-PROCESS turn (an ``_run_agent`` result dict).
  92. The transport-agnostic twin of the child-process lanes' ``retry_action(classify_agent_error(
  93. turn_failure_text(stdout, stderr)))``: same policy, the failure text just arrives as result fields
  94. instead of two streams — ``error`` carries the raw provider summary (status codes included) and
  95. ``failure_reason`` the turn loop's own typed verdict, which is what names an overflow the copy only
  96. describes in prose. ``RETRY_NONE`` for anything that did not fail (and for a non-dict result), so a
  97. successful turn whose text happens to mention 429 is never re-run."""
  98. if not isinstance(result, dict) or not result.get("failed"):
  99. return RETRY_NONE
  100. return retry_action(classify_agent_error(
  101. turn_failure_text(result.get("error"), result.get("failure_reason"))))
  102. def classify_agent_error(text: str) -> str:
  103. """Map raw agent/provider error text to a closed reason code (``unknown`` when unmatched/empty)."""
  104. raw = str(text or "")
  105. if raw.strip():
  106. for pattern, code in _RULES:
  107. if pattern.search(raw):
  108. return code
  109. return UNKNOWN
  110. def delivery_failure_reason(error: BaseException) -> str:
  111. """The typed reason for a delivery refusal, kept inside the documented vocabulary.
  112. An exception's own ``reason`` is trusted only when it names a real code: ``TurnBusyError``
  113. carries ``target_busy``, but ``reason`` is also a stdlib attribute on ``ssl.SSLError`` and
  114. ``urllib.error.URLError``, and forwarding one of those would put free text where consumers
  115. expect a closed set. Anything else is classified like every other failure. Shared by the
  116. relay lane (``bot_relay.deliver``) and the local runner (``bot_mode_dm --run-delivery``).
  117. """
  118. # 'target_busy' extends the structured refusal enum and predates ALL_REASONS.
  119. supplied = str(getattr(error, "reason", "") or "").strip()
  120. if supplied == "target_busy" or supplied in ALL_REASONS:
  121. return supplied
  122. return classify_agent_error(str(error))