scheduler_failure_copy.py 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148
  1. """Plain-language copy for the one-line cron failure notice delivered to a job's chat.
  2. The scheduler classifies the failure text through ``agent.error_classifier.classify_api_error``
  3. (one classifier for the whole app, no cron-local regex ladder) and looks the verdict up here.
  4. Every notice says WHAT happened and WHAT TO DO, and names the exact ``hermes cron`` command plus
  5. the real output directory — "cron output" alone sent operators hunting.
  6. """
  7. from __future__ import annotations
  8. import re
  9. from typing import Any, Optional
  10. from hermes_constants import display_hermes_home
  11. def cron_output_dir_display(job_id: str) -> str:
  12. """User-facing path of a job's saved run output (profile-aware)."""
  13. return f"{display_hermes_home()}/cron/output/{job_id}/"
  14. _HTTP_STATUS_IN_TEXT = re.compile(r"(?:\bHTTP\b|\bError code\b|\bstatus(?: code)?\b)\W{0,3}(\b[45]\d\d\b)", re.IGNORECASE)
  15. _LEADING_EXC_TYPE = re.compile(r"^(?:[\w.]+\.)?([A-Z]\w*(?:Error|Timeout|Exception))\s*:")
  16. def classify_cron_failure_reason(text: str) -> str:
  17. """``FailoverReason`` value for a cron failure string (``"unknown"`` when unclassifiable).
  18. The scheduler only has ``str(exc)``, so the status code and exception type the classifier
  19. keys on are rebuilt from the text: a whole-token HTTP code after ``HTTP`` / ``Error code`` /
  20. ``status`` (a bare ``429`` inside a job id or hash never counts, #83188) and a leading
  21. ``ReadTimeout:``-style type prefix."""
  22. from agent.error_classifier import classify_api_error
  23. status = _HTTP_STATUS_IN_TEXT.search(text)
  24. type_name = _LEADING_EXC_TYPE.match(text)
  25. exc_cls = type(type_name.group(1), (Exception,), {}) if type_name else Exception
  26. exc = exc_cls(text)
  27. if status:
  28. exc.status_code = int(status.group(1))
  29. return classify_api_error(exc).reason.value
  30. # What happened, per reason: the one gloss table shared with subagent notices lives in
  31. # agent/turn_failure_copy.py so the two never drift; the job is the subject here.
  32. def _provider_failure_cause(reason: str) -> Optional[str]:
  33. from agent.turn_failure_copy import failure_cause_gloss
  34. return failure_cause_gloss(reason, subject="this job", possessive="the job's")
  35. _TRANSIENT_REASONS = frozenset({"timeout", "rate_limit", "upstream_rate_limit", "overloaded", "server_error"})
  36. # Reason -> what to do. Transient reasons get the backup-provider clause from the scheduler
  37. # (it knows whether a fallback chain is configured) instead of a fixed sentence.
  38. _PROVIDER_FAILURE_ACTION: dict[str, str] = {
  39. "billing": (
  40. "Top up or wait for the limit to reset, or pin another provider with "
  41. "`hermes cron edit {job_id} --provider <name>`."
  42. ),
  43. "auth": (
  44. "Sign in again with /login (or `{relogin}` in a terminal), or pin a "
  45. "working provider with `hermes cron edit {job_id} --provider <name>`, then "
  46. "`hermes cron run {job_id}` to retry."
  47. ),
  48. "model_not_found": "Pick another model with `hermes cron edit {job_id} --model <name>`.",
  49. "upstream_blocked": (
  50. "A firewall in front of the provider blocked the request (not your key): set a User-Agent "
  51. "via `extra_headers` on the provider's custom_providers entry, or pin another provider with "
  52. "`hermes cron edit {job_id} --provider <name>`."
  53. ),
  54. "context_overflow": "Shorten the job's prompt with `hermes cron edit {job_id} --prompt <text>`.",
  55. }
  56. _PROVIDER_FAILURE_ACTION["auth_permanent"] = _PROVIDER_FAILURE_ACTION["auth"]
  57. _PROVIDER_FAILURE_ACTION["billing_unverified"] = _PROVIDER_FAILURE_ACTION["billing"]
  58. _PROVIDER_FAILURE_ACTION["payload_too_large"] = _PROVIDER_FAILURE_ACTION["context_overflow"]
  59. _PROVIDER_FAILURE_ACTION["content_policy_blocked"] = (
  60. "Reword the job's prompt with `hermes cron edit {job_id} --prompt <text>`, or pick another "
  61. "model with `hermes cron edit {job_id} --model <name>`."
  62. )
  63. _PROVIDER_FAILURE_ACTION["provider_policy_blocked"] = (
  64. "Retrying won't help: check the account's status and data/privacy settings with the provider, "
  65. "or pin another model with `hermes cron edit {job_id} --model <name>`."
  66. )
  67. _DEFAULT_FAILURE_ACTION = "Run it again with `hermes cron run {job_id}`, or edit it with `hermes cron edit {job_id}`."
  68. def provider_failure_notice(
  69. job_name: str, job_id: str, reason: str, *, backup_provider_phrase: str, provider: Any = None,
  70. ) -> Optional[str]:
  71. """The notice for a provider-shaped ``reason``, or None when the reason is not one.
  72. ``provider`` is the job's pinned slug (if any) so the auth action names its exact sign-in."""
  73. cause = _provider_failure_cause(reason)
  74. if cause is None:
  75. return None
  76. if reason in _TRANSIENT_REASONS:
  77. action = (
  78. f"{backup_provider_phrase} It will run again at its next scheduled time; "
  79. f"`hermes cron run {job_id}` tries now."
  80. )
  81. else:
  82. from agent.turn_failure_copy import relogin_command_hint
  83. action = _PROVIDER_FAILURE_ACTION.get(reason, _DEFAULT_FAILURE_ACTION).format(
  84. job_id=job_id, relogin=relogin_command_hint(provider))
  85. return (
  86. f"⚠️ Cron '{job_name}' failed: {cause}. {action} "
  87. f"Run log: `hermes cron runs {job_id}`."
  88. )
  89. def generic_failure_notice(job_name: str, job_id: str, cleaned_error: str) -> str:
  90. """Unclassified failure: the cleaned error text plus where to look and what to do."""
  91. return (
  92. f"⚠️ Cron '{job_name}' failed: {cleaned_error}. "
  93. f"See the full run with `hermes cron runs {job_id}` (output saved under "
  94. f"{cron_output_dir_display(job_id)}); run it again with `hermes cron run {job_id}`, "
  95. f"edit it with `hermes cron edit {job_id}`, or pause it with `hermes cron pause {job_id}`."
  96. )
  97. def script_timeout_notice(job_name: str, job_id: str) -> str:
  98. return (
  99. f"⚠️ Cron '{job_name}' failed: its script timed out. No model was invoked. "
  100. f"Check the script's output under {cron_output_dir_display(job_id)} or `hermes cron runs {job_id}`, "
  101. f"then run it again with `hermes cron run {job_id}`."
  102. )
  103. def inactivity_notice(job_name: str, job_id: str) -> str:
  104. return (
  105. f"⚠️ Cron '{job_name}' failed: the job stalled — it stopped doing anything for too long "
  106. f"and was cut off. Check what it was doing in the saved output under "
  107. f"{cron_output_dir_display(job_id)} (`hermes cron runs {job_id}`), then run it again with "
  108. f"`hermes cron run {job_id}`."
  109. )
  110. def blocked_config_notice(job_name: str, reason: str) -> str:
  111. """One-time notice when the pre-run configuration check refused to start the job."""
  112. reason = reason.rstrip()
  113. if reason and reason[-1] not in ".!?":
  114. reason += "."
  115. return (
  116. f"⛔ Cron '{job_name}' did not run: {reason} Nothing was charged. Hermes will try again at "
  117. "the next scheduled time and will not repeat this alert; check with "
  118. "`hermes cron doctor`."
  119. )