delivery_ledger.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564
  1. """Durable delivery-obligation ledger for gateway final responses (rows in the shared ``state.db``;
  2. WAL, owner pid + process-start liveness, bounded retention) so a crash between finalize and
  3. platform ACK cannot lose a response silently. Checkpoints: record_obligation() 'pending' before
  4. any send | mark_attempting() 'attempting' right before the await | mark_delivered() 'delivered'
  5. only on SendResult.success | mark_failed() 'failed' on a definitive rejection. Crash semantics
  6. (never silently resend an ambiguous send): pending = never started, redeliver plainly; attempting
  7. = crashed mid-await, platform MAY have it, redeliver WITH a visible recovered marker; failed =
  8. rejected once, restart is a retry boundary, also marked; delivered = prune. Attempts are capped
  9. and stale rows expire, both -> 'abandoned' (kept briefly, then pruned). Everything is
  10. best-effort: ledger failures must never block a send; callers wrap every call in try/except.
  11. """
  12. from __future__ import annotations
  13. import hashlib
  14. import logging
  15. import os
  16. import re
  17. import sqlite3
  18. import threading
  19. import time
  20. from typing import Any, Dict, List, Optional
  21. from gateway.dead_targets import classify_dead_error
  22. from hermes_cli.sqlite_util import add_column_if_missing
  23. from hermes_constants import get_process_hermes_home
  24. logger = logging.getLogger(__name__)
  25. _DB_LOCK = threading.Lock()
  26. # Redelivery policy knobs (deliberately not config — the ledger is gated by
  27. # ``gateway.delivery_ledger`` and these only matter in the rare recovery path).
  28. MAX_ATTEMPTS = 3
  29. STALE_AFTER_SECONDS = 24 * 60 * 60
  30. _RETENTION_SECONDS = 7 * 24 * 60 * 60
  31. _MAX_ROWS = 500
  32. # Visible prefixes for redeliveries that might duplicate an already-received message (crash mid-send /
  33. # post-rejection retry) — honest at-least-once. Runtime recovery uses a distinct marker: no restart
  34. # occurred, but a network rejection's acknowledgement can still have been lost independently.
  35. RECOVERED_MARKER = "♻️ Recovered reply — the gateway restarted during delivery, so this may be a duplicate:\n\n"
  36. RECONNECTED_MARKER = ("♻️ Recovered reply — the messaging platform reconnected after the original "
  37. "delivery failed, so this may be a duplicate:\n\n")
  38. # A reply refused by flood control may have gone out as several requests (the adapter chunks long replies,
  39. # and MarkdownV2 escaping alone can push a reply that fits one message into two), and the platform may have
  40. # accepted the first chunk(s) before refusing the rest; the send result does not say which landed. The raw
  41. # length of the stored text says nothing about that, so every flood redelivery carries a marker, and neither
  42. # of the markers above tells the truth here (no restart, no reconnect): the rate limit gets its own.
  43. FLOOD_MARKER = ("♻️ Recovered reply — the messaging platform's rate limit refused the original, so part of "
  44. "it may already have arrived above:\n\n")
  45. # Errors whose send contract proves the platform never saw the request: retried as soon as the adapter
  46. # is back, no backoff. Every other rejection is retried too (#91653: a 5xx or a transient parse error
  47. # used to strand the reply in ``failed`` until the next restart), but only after a backoff that grows
  48. # with the attempts already spent, so a platform-side outage is not hammered by the redelivery timer.
  49. # A whole-chat death (blocked bot, deleted group, deactivated user) is never retried: the target is gone.
  50. _RUNTIME_RETRYABLE_ERRORS = frozenset({"send_path_degraded"})
  51. # One tier per in-process retry; the last budgeted attempt is left to the boot sweep (retry_not_before).
  52. _RETRY_BACKOFF_SECONDS = (30.0, 120.0)
  53. assert len(_RETRY_BACKOFF_SECONDS) == MAX_ATTEMPTS - 1
  54. # A final send the platform refused with flood control is the other transient case: a 429 means the
  55. # refused request was never accepted, and the platform said how long to wait. Adapters fail such sends
  56. # closed as ``flood_control:<seconds>`` on purpose (#91969) so that this ledger owns the wait instead of
  57. # the send coroutine sleeping through it. Before this the row simply sat in ``failed`` until the next
  58. # restart's sweep, which then redelivered it hours late under the "gateway restarted during delivery"
  59. # marker.
  60. FLOOD_ERROR_PREFIX = "flood_control:"
  61. FLOOD_RETRY_DEFAULT_SECONDS = 60.0
  62. FLOOD_RETRY_CAP_SECONDS = 15 * 60.0
  63. FLOOD_RETRY_SLACK_SECONDS = 2.0
  64. # The canonical prefix above is what the adapters produce for a flood they decide not to sleep. It is
  65. # not the only shape that reaches this ledger. PTB raises ``RetryAfter``, whose own text reads
  66. # "Flood control exceeded. Retry in 185 seconds", and that text is what lands in ``last_error``
  67. # whenever a send fails on a path that has not been normalized, or was written by an older build
  68. # before it was. Such a row has to be recognised too: unrecognised, it is treated as an ordinary
  69. # failure, so no redelivery timer is armed for it and a boot sweep claims it immediately instead of
  70. # adopting it until its deadline, spending the one attempt inside the penalty that caused it.
  71. # Matching requires the "flood control" wording as well as the delay, so an unrelated error that
  72. # merely suggests retrying is never mistaken for a flood.
  73. _RAW_FLOOD_RE = re.compile(r"flood control exceeded.*?retry in\s+(\d+(?:\.\d+)?)", re.IGNORECASE)
  74. def _raw_flood_wait(text: str) -> Optional[float]:
  75. """Seconds asked for by a flood error still carrying the platform's own wording, else ``None``."""
  76. match = _RAW_FLOOD_RE.search(text or "")
  77. if not match:
  78. return None
  79. try:
  80. return float(match.group(1))
  81. except (TypeError, ValueError):
  82. return None
  83. def is_reconnect_only(error: Any) -> bool:
  84. """True for a row that only an adapter reconnect may retry (no timer, no backoff)."""
  85. return str(error or "").strip().lower() in _RUNTIME_RETRYABLE_ERRORS
  86. def is_flood_error(error: Any) -> bool:
  87. """True for a flood refusal: the adapters' fail-closed ``flood_control:<seconds>`` result, or a
  88. row still carrying the platform's own flood wording (see ``_RAW_FLOOD_RE``)."""
  89. text = str(error or "").strip().lower()
  90. return text.startswith(FLOOD_ERROR_PREFIX) or _raw_flood_wait(text) is not None
  91. def flood_wait_seconds(error: Any, default: float = FLOOD_RETRY_DEFAULT_SECONDS) -> float:
  92. """The wait the platform asked for, read from ``flood_control:<seconds>``; ``default`` when unreadable."""
  93. text = str(error or "").strip().lower()
  94. wait = default
  95. if text.startswith(FLOOD_ERROR_PREFIX):
  96. try:
  97. wait = float(text[len(FLOOD_ERROR_PREFIX):].strip())
  98. except ValueError:
  99. wait = default
  100. else:
  101. # A row still carrying the platform's own flood wording states its delay just as precisely,
  102. # and the deadline must use it rather than the generic default.
  103. raw = _raw_flood_wait(text)
  104. if raw is not None:
  105. wait = raw
  106. return wait if wait > 0 else default
  107. def flood_retry_delay(seconds: Any) -> float:
  108. """How long a redelivery timer sleeps for a wait of ``seconds``: capped so a huge or bogus value cannot
  109. park the timer for hours (the row's own deadline, not the timer, decides eligibility when it fires),
  110. plus a little slack so the timer lands after the deadline rather than on it."""
  111. try:
  112. wait = float(seconds)
  113. except (TypeError, ValueError):
  114. wait = FLOOD_RETRY_DEFAULT_SECONDS
  115. return min(max(wait, 0.0), FLOOD_RETRY_CAP_SECONDS) + FLOOD_RETRY_SLACK_SECONDS
  116. def _failed_stamp(updated_at: Any) -> float:
  117. try:
  118. return float(updated_at or 0.0)
  119. except (TypeError, ValueError):
  120. return 0.0
  121. def flood_not_before(updated_at: Any, last_error: Any) -> float:
  122. """Earliest moment a flood-refused row may be resent: the refusal's timestamp (``mark_failed`` sets
  123. ``updated_at``) plus the platform's wait. Enforced by the sweeps so neither an early timer nor a
  124. reconnect sweep spends a redelivery attempt inside the penalty window."""
  125. return _failed_stamp(updated_at) + flood_wait_seconds(last_error)
  126. def retry_not_before(updated_at: Any, last_error: Any, attempts: Any) -> Optional[float]:
  127. """Earliest moment a failed row may be resent, or ``None`` for a row the runtime must leave alone:
  128. a flood refusal keeps the platform's own wait, an allowlisted reconnect error is due at once, a
  129. whole-chat death is final, and any other rejection backs off by the attempts already spent — but
  130. never spends the LAST budgeted attempt. An unclassified outage can outlast any timer, and a row
  131. the timer abandoned would be lost for good; leaving one attempt keeps it recoverable by the boot
  132. sweep after a restart, which is a real recovery signal."""
  133. if is_flood_error(last_error):
  134. return flood_not_before(updated_at, last_error)
  135. text = str(last_error or "").strip().lower()
  136. if is_reconnect_only(text):
  137. return _failed_stamp(updated_at)
  138. if classify_dead_error(text):
  139. return None
  140. spent = int(attempts or 0)
  141. if spent >= MAX_ATTEMPTS - 1:
  142. return None
  143. return _failed_stamp(updated_at) + _RETRY_BACKOFF_SECONDS[spent]
  144. def _db_path():
  145. # Launch home, not get_hermes_home(): a multiplexed gateway records a served profile's replies
  146. # under that profile's home override, but the boot sweep reads from the launch context, so both
  147. # must open the one shared store (adapter_profile tells the bots apart). No get_hermes_home()
  148. # fallback for an unset HERMES_HOME: a default gateway run in the foreground has none.
  149. return get_process_hermes_home() / "state.db"
  150. def _connect() -> sqlite3.Connection:
  151. from hermes_cli.sqlite_util import open_db
  152. # Shared state.db: SessionDB owns the durable PRAGMA set; this opener keeps the plain-tuple rows
  153. # and the 10 s busy timeout it always had.
  154. return open_db(_db_path(), db_label="state.db (delivery_ledger)", busy_timeout_ms=10_000,
  155. row_factory=None, initialize=_initialize_schema)
  156. def _initialize_schema(conn: sqlite3.Connection) -> None:
  157. conn.execute(
  158. """CREATE TABLE IF NOT EXISTS delivery_obligations (
  159. obligation_id TEXT PRIMARY KEY,
  160. session_key TEXT NOT NULL,
  161. platform TEXT NOT NULL,
  162. chat_id TEXT NOT NULL,
  163. thread_id TEXT,
  164. content TEXT NOT NULL,
  165. state TEXT NOT NULL,
  166. attempts INTEGER NOT NULL DEFAULT 0,
  167. created_at REAL NOT NULL,
  168. updated_at REAL NOT NULL,
  169. owner_pid INTEGER,
  170. owner_started_at INTEGER,
  171. last_error TEXT,
  172. adapter_profile TEXT
  173. )"""
  174. )
  175. if "adapter_profile" not in {row[1] for row in conn.execute("PRAGMA table_info(delivery_obligations)")}:
  176. add_column_if_missing(conn, "delivery_obligations", "adapter_profile", "adapter_profile TEXT")
  177. def _transaction():
  178. from hermes_cli.sqlite_util import transaction
  179. return transaction(_connect())
  180. def _start_time(pid: int) -> Optional[int]:
  181. try:
  182. from gateway.status import get_process_start_time # lazy: tests monkeypatch gateway.status
  183. return get_process_start_time(pid)
  184. except Exception:
  185. return None
  186. def _owner_stamp() -> tuple[int, Optional[int]]:
  187. pid = os.getpid()
  188. return pid, _start_time(pid)
  189. def _owner_alive(pid: Any, started_at: Any) -> bool:
  190. """True when the recorded owning process still exists (pid + start time)."""
  191. if not pid:
  192. return False
  193. try:
  194. pid = int(pid)
  195. except (TypeError, ValueError):
  196. return False
  197. current_start = _start_time(pid)
  198. if current_start is None:
  199. # Start time unreadable: alive iff the pid exists. Route through the cross-platform probe — on Windows
  200. # ``os.kill(pid, 0)`` is NOT a no-op (bpo-14484: it maps to ``GenerateConsoleCtrlEvent(0, pid)`` and could
  201. # Ctrl+C the gateway's own console group). ``_pid_exists`` keeps EPERM-means-alive (pid owned by another user).
  202. try:
  203. from gateway.status import _pid_exists
  204. except Exception:
  205. if os.name == "nt":
  206. return False # never fall back to a raw sig-0 probe on Windows
  207. try:
  208. os.kill(pid, 0) # windows-footgun: ok — POSIX-only fallback branch
  209. return True
  210. except OSError as exc: # incl. ProcessLookupError; EPERM means the pid exists
  211. return isinstance(exc, PermissionError)
  212. try:
  213. return bool(_pid_exists(pid))
  214. except Exception:
  215. return False
  216. try:
  217. from gateway.status import start_time_fingerprints_match
  218. return started_at is None or start_time_fingerprints_match(started_at, current_start)
  219. except (TypeError, ValueError):
  220. return True
  221. def compute_obligation_id(session_key: str, message_ref: str, content: str) -> str:
  222. """Stable id: same turn + same content re-records idempotently, while distinct threads/topics on one
  223. chat never collide (session_key carries platform/chat/thread; ``message_ref`` = inbound message id)."""
  224. return hashlib.sha256(f"{session_key}|{message_ref}|{content}".encode("utf-8", "replace")).hexdigest()[:24]
  225. def record_obligation(*, obligation_id: str, session_key: str, platform: str, chat_id: str,
  226. thread_id: Optional[str], content: str, adapter_profile: Optional[str] = None) -> None:
  227. """Record a final response as owed to the platform (state='pending')."""
  228. now, (pid, started) = time.time(), _owner_stamp()
  229. with _DB_LOCK, _transaction() as conn:
  230. conn.execute(
  231. """INSERT OR REPLACE INTO delivery_obligations
  232. (obligation_id, session_key, platform, chat_id, thread_id,
  233. content, state, attempts, created_at, updated_at,
  234. owner_pid, owner_started_at, adapter_profile)
  235. VALUES (?, ?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?, ?, ?)""",
  236. (obligation_id, session_key, platform, str(chat_id), str(thread_id) if thread_id else None,
  237. content, now, now, pid, started, str(adapter_profile).strip() if adapter_profile else "default"))
  238. # Same transaction, same connection: the cron ledgers prune this way too
  239. # (cron/delivery_queue._prune_terminal_unlocked, cron/executions._prune_unlocked).
  240. _prune_unlocked(conn, now)
  241. def record_crash_left_reply(*, obligation_id: str, session_key: str, platform: str, chat_id: str,
  242. thread_id: Optional[str], content: str, since: float,
  243. adapter_profile: Optional[str] = None) -> None:
  244. """Adopt a reply a killed process persisted but never ledgered. Unowned, so this boot's sweep
  245. claims it, and 'attempting', because a streamed reply may already be on screen: it is
  246. redelivered once, with the recovered marker. A no-op when the same reply was already ledgered
  247. since *since* (the turn start), and idempotent across boots that die before their sweep."""
  248. now = time.time()
  249. with _DB_LOCK, _transaction() as conn:
  250. conn.execute(
  251. """INSERT OR IGNORE INTO delivery_obligations
  252. (obligation_id, session_key, platform, chat_id, thread_id,
  253. content, state, attempts, created_at, updated_at,
  254. owner_pid, owner_started_at, adapter_profile)
  255. SELECT ?, ?, ?, ?, ?, ?, 'attempting', 0, ?, ?, NULL, NULL, ?
  256. WHERE NOT EXISTS (SELECT 1 FROM delivery_obligations
  257. WHERE session_key = ? AND content = ? AND created_at >= ?)""",
  258. (obligation_id, session_key, platform, str(chat_id), str(thread_id) if thread_id else None,
  259. content, now, now, str(adapter_profile).strip() if adapter_profile else "default",
  260. session_key, content, since))
  261. def mark_attempting(obligation_id: str) -> None:
  262. _update_state(obligation_id, "attempting")
  263. def mark_delivered(obligation_id: str) -> None:
  264. _update_state(obligation_id, "delivered")
  265. def mark_failed(obligation_id: str, error: str = "") -> None:
  266. _update_state(obligation_id, "failed", error=error)
  267. def release_runtime_claim(obligation_id: str, error: str = "") -> bool:
  268. """Return an unsent runtime claim to ``failed`` without spending an attempt.
  269. Runtime recovery claims before clearing ``resume_pending`` so two reconnect paths cannot send the
  270. same row; if the flag cannot be cleared no send was attempted and the claim must not consume the
  271. redelivery budget. Fail-closed to the exact current process instance and ``attempting`` state."""
  272. pid, started = _owner_stamp()
  273. if started is None:
  274. return False
  275. with _DB_LOCK, _transaction() as conn:
  276. cursor = conn.execute(
  277. """UPDATE delivery_obligations
  278. SET state='failed', attempts=CASE
  279. WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
  280. updated_at=?, last_error=?
  281. WHERE obligation_id=? AND state='attempting'
  282. AND owner_pid IS ? AND owner_started_at IS ?""",
  283. (time.time(), error[:500] if error else None, obligation_id, pid, started))
  284. return bool(cursor.rowcount)
  285. def _update_state(obligation_id: str, state: str, error: str = "") -> None:
  286. with _DB_LOCK, _transaction() as conn:
  287. conn.execute(
  288. """UPDATE delivery_obligations
  289. SET state=?, updated_at=?, last_error=?
  290. WHERE obligation_id=?""",
  291. (state, time.time(), error[:500] if error else None, obligation_id))
  292. def _claimed_row(oid, session_key, platform, chat_id, thread_id, content, attempts, profile, *,
  293. needs_marker: bool, runtime: bool = False, flood: bool = False,
  294. last_error: Optional[str] = None) -> dict[str, Any]:
  295. """Claimed-row dict handed back for redelivery. A marked row names its own cause: ``flood`` (a reply
  296. the rate limit refused, possibly after accepting part of it) gets FLOOD_MARKER at boot or at runtime, a
  297. ``runtime`` reconnect replay gets RECONNECTED_MARKER, and a boot-recovered crash keeps the runner's
  298. restart marker default. ``last_error`` is the row's pre-claim error, carried so a runtime claim that is
  299. released unsent goes back to ``failed`` with the same error and keeps its retry eligibility."""
  300. marker = FLOOD_MARKER if flood else (RECONNECTED_MARKER if runtime else None)
  301. return {"obligation_id": oid, "session_key": session_key, "platform": platform, "chat_id": chat_id,
  302. "thread_id": thread_id, "content": content, "needs_marker": needs_marker,
  303. **({"marker": marker} if needs_marker and marker else {}), "profile": profile,
  304. **({"runtime_recovery": True} if runtime else {}),
  305. **({"last_error": last_error} if last_error else {}), "attempts": attempts + 1}
  306. def sweep_recoverable(now: Optional[float] = None, *, deliverable_platforms: Optional[set] = None,
  307. deliverable_targets: Optional[set] = None) -> list[dict[str, Any]]:
  308. """Claim undelivered rows owned by dead processes; return them for redelivery.
  309. Claiming atomically re-stamps the owner to THIS process, moves the row to 'attempting' and increments
  310. ``attempts`` (the UPDATE is guarded on the previous owner stamp, so a second gateway racing the same
  311. sweep cannot double-claim).
  312. Rows over the attempts cap or stale cutoff become 'abandoned'. ``deliverable_platforms`` restricts
  313. claiming to platforms the caller can send on this boot: ``attempts`` is the redelivery budget and
  314. must only be spent on a real send, else a platform that failed to connect burns one attempt per boot
  315. and hits the cap having never been sent once (the stale cutoff still bounds untouched rows).
  316. ``deliverable_targets`` further scopes multiplexed gateways by exact ``(platform, adapter_profile)``
  317. so one connected bot cannot spend another disconnected bot's retry budget.
  318. A flood-refused row still inside its wait is adopted (owner re-stamped, no attempt spent) and
  319. returned flagged ``adopted`` with its ``not_before``: the caller clears its session's resume flag
  320. like any other claimed row, since the answer is in the ledger, but must not send it; the flood
  321. timer does once the wait has passed. A legacy row without ``adapter_profile`` is normalised to
  322. ``'default'`` on claim or adoption (the caller only accepts such rows when it is not multiplexed),
  323. because the runtime sweep matches profiles exactly and could otherwise never claim it."""
  324. now, (pid, started) = now if now is not None else time.time(), _owner_stamp()
  325. claimed: list[dict[str, Any]] = []
  326. with _DB_LOCK, _transaction() as conn:
  327. rows = conn.execute(
  328. """SELECT obligation_id, session_key, platform, chat_id, thread_id,
  329. content, state, attempts, created_at,
  330. owner_pid, owner_started_at, adapter_profile, last_error, updated_at
  331. FROM delivery_obligations
  332. WHERE state IN ('pending', 'attempting', 'failed')"""
  333. ).fetchall()
  334. for (oid, session_key, platform, chat_id, thread_id, content, state, attempts, created_at,
  335. owner_pid, owner_started_at, adapter_profile, last_error, updated_at) in rows:
  336. if _owner_alive(owner_pid, owner_started_at):
  337. continue # a live gateway still owns this row
  338. if attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS: # exhausted -> abandoned
  339. conn.execute(
  340. """UPDATE delivery_obligations
  341. SET state='abandoned', updated_at=? WHERE obligation_id=?""", (now, oid))
  342. continue
  343. if ((deliverable_platforms is not None and platform not in deliverable_platforms)
  344. or (deliverable_targets is not None and (platform, adapter_profile) not in deliverable_targets)):
  345. continue # no adapter this boot — claiming would spend an attempt on a no-op
  346. flood_row = state == "failed" and is_flood_error(last_error)
  347. if flood_row and now < flood_not_before(updated_at, last_error):
  348. # Still inside the platform's wait: adopt the dead owner's row without spending an attempt
  349. # (state and error kept) so this process's flood timer can claim it once the wait passes.
  350. cursor = conn.execute(
  351. """UPDATE delivery_obligations
  352. SET owner_pid=?, owner_started_at=?,
  353. adapter_profile=COALESCE(adapter_profile, 'default')
  354. WHERE obligation_id=? AND (owner_pid IS ? OR owner_pid=?)""",
  355. (pid, started, oid, owner_pid, owner_pid))
  356. if cursor.rowcount:
  357. claimed.append({
  358. "obligation_id": oid, "session_key": session_key, "platform": platform,
  359. "chat_id": chat_id, "thread_id": thread_id, "content": content,
  360. "profile": adapter_profile or "default", "attempts": attempts,
  361. "adopted": True, "not_before": flood_not_before(updated_at, last_error)})
  362. continue
  363. # Every claim starts a send, so the row leaves 'pending'/'failed' for 'attempting' in the same
  364. # CAS: a boot killed inside the redelivery then leaves proof the platform may have it (next boot
  365. # marks it), and the runtime sweep, which only takes 'failed', cannot re-claim it mid-send. A
  366. # claimed flood row also drops its stale refusal, so an interrupted resend has no error.
  367. cursor = conn.execute(
  368. """UPDATE delivery_obligations
  369. SET owner_pid=?, owner_started_at=?, attempts=attempts+1, updated_at=?,
  370. adapter_profile=COALESCE(adapter_profile, 'default'), state='attempting',
  371. last_error=CASE WHEN ? THEN NULL ELSE last_error END
  372. WHERE obligation_id=? AND (owner_pid IS ? OR owner_pid=?)""",
  373. (pid, started, now, 1 if flood_row else 0, oid, owner_pid, owner_pid))
  374. if cursor.rowcount:
  375. # A never-claimed pending row was never sent: redeliver plainly. Anything else (crashed
  376. # mid-await, other rejection, a flood refusal whose earlier chunks the platform may have
  377. # accepted, or a pending row an older build already claimed and may have sent) carries
  378. # the marker.
  379. claimed.append(_claimed_row(oid, session_key, platform, chat_id, thread_id, content, attempts,
  380. adapter_profile or "default",
  381. needs_marker=state != "pending" or attempts > 0, flood=flood_row))
  382. return claimed
  383. def sweep_failed_for_runtime(platform: str, now: Optional[float] = None, *,
  384. profile: Optional[str] = None) -> list[dict[str, Any]]:
  385. """Claim this process's failed rows that are due for another send, for one adapter.
  386. ``profile`` scopes multiplexed gateways to the bot identity that owned the failed send (``None`` =
  387. primary/default adapter); unowned rows and rows owned by another process are left for the
  388. startup/dead-owner sweep. Startup recovery ignores rows owned by a live gateway, so a response
  389. rejected with ``send_path_degraded`` would stay stranded when only the adapter reconnects; this closes
  390. that gap without weakening ownership: only rows stamped to this exact process instance, only rows
  391. past their ``retry_not_before`` deadline, same attempts/staleness bounds, every update guarded by the
  392. prior owner stamp and ``failed`` state. Claimed rows always carry a marker (the failed send's ack is not safe to
  393. infer): the reconnect one, or the rate-limit one for a flood-refused row."""
  394. now, (pid, started) = now if now is not None else time.time(), _owner_stamp()
  395. if started is None: # PID alone cannot distinguish this process from a stale row left after PID
  396. return [] # reuse; runtime replay is optional, so fail closed (startup recovery remains).
  397. expected_profile = "default" if not profile or profile == "default" else str(profile)
  398. claimed: list[dict[str, Any]] = []
  399. with _DB_LOCK, _transaction() as conn:
  400. rows = conn.execute(
  401. """SELECT obligation_id, session_key, platform, chat_id, thread_id,
  402. content, attempts, created_at, owner_pid,
  403. owner_started_at, last_error, adapter_profile, updated_at
  404. FROM delivery_obligations
  405. WHERE state='failed' AND platform=?""", (platform,)).fetchall()
  406. for (oid, session_key, row_platform, chat_id, thread_id, content, attempts, created_at,
  407. owner_pid, owner_started_at, last_error, adapter_profile, updated_at) in rows:
  408. # Exact process-start matching prevents PID reuse from stealing work.
  409. if adapter_profile != expected_profile or owner_pid != pid or owner_started_at != started:
  410. continue
  411. due = retry_not_before(updated_at, last_error, attempts)
  412. if due is None:
  413. continue
  414. owner_guard = (now, oid, owner_pid, owner_started_at)
  415. if attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS: # exhausted -> abandoned
  416. conn.execute(
  417. """UPDATE delivery_obligations
  418. SET state='abandoned', updated_at=?
  419. WHERE obligation_id=? AND state='failed'
  420. AND owner_pid IS ? AND owner_started_at IS ?""", owner_guard)
  421. continue
  422. if now < due:
  423. continue # the platform's wait or the backoff has not passed; the timer comes back for it
  424. # The claim clears the stale error: this is a fresh attempt, and if it is interrupted the next
  425. # boot must see 'attempting' with no proof of non-delivery, hence the marker.
  426. cursor = conn.execute(
  427. """UPDATE delivery_obligations
  428. SET state='attempting', attempts=attempts+1, updated_at=?, last_error=NULL
  429. WHERE obligation_id=? AND state='failed'
  430. AND owner_pid IS ? AND owner_started_at IS ?""", owner_guard)
  431. if cursor.rowcount:
  432. # The failed send's ack may have been lost (reconnect) or its earlier chunks accepted
  433. # (flood): every runtime redelivery carries a marker. The pre-claim error rides along so a
  434. # claim released unsent keeps its flood retry eligibility.
  435. claimed.append(_claimed_row(oid, session_key, row_platform, chat_id, thread_id, content,
  436. attempts, adapter_profile, needs_marker=True, runtime=True,
  437. flood=is_flood_error(last_error), last_error=last_error))
  438. return claimed
  439. def pending_retries(now: Optional[float] = None) -> list[dict[str, Any]]:
  440. """This process's failed rows that still await redelivery, one entry per adapter identity with the
  441. earliest deadline (``not_before``). The runner arms one redelivery timer per entry, so a row adopted
  442. at boot, skipped because its wait had not passed, or rejected again is never stranded. Rows past the
  443. attempts cap or stale cutoff are left for the sweeps to abandon."""
  444. now, (pid, started) = now if now is not None else time.time(), _owner_stamp()
  445. if started is None:
  446. return []
  447. with _DB_LOCK, _transaction() as conn:
  448. rows = conn.execute(
  449. """SELECT platform, adapter_profile, updated_at, last_error, attempts, created_at
  450. FROM delivery_obligations
  451. WHERE state='failed' AND owner_pid IS ? AND owner_started_at IS ?""", (pid, started)).fetchall()
  452. earliest: dict[tuple, float] = {}
  453. for platform, adapter_profile, updated_at, last_error, attempts, created_at in rows:
  454. # Reconnect-only rows (a claim released because the adapter was gone) are re-claimed by the
  455. # reconnect sweep; a timer would claim and release them every tick until the adapter is back.
  456. if is_reconnect_only(last_error):
  457. continue
  458. due = retry_not_before(updated_at, last_error, attempts)
  459. if due is None or attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS:
  460. continue
  461. key = (platform, adapter_profile or "default")
  462. if key not in earliest or due < earliest[key]:
  463. earliest[key] = due
  464. return [{"platform": platform, "profile": profile, "not_before": due}
  465. for (platform, profile), due in sorted(earliest.items())]
  466. def _prune_unlocked(conn, now: float) -> None:
  467. """Retention DELETEs on the caller's open connection — must run inside the caller's transaction."""
  468. conn.execute(
  469. """DELETE FROM delivery_obligations
  470. WHERE state IN ('delivered', 'abandoned') AND updated_at < ?""", (now - _RETENTION_SECONDS,))
  471. total = conn.execute("SELECT COUNT(*) FROM delivery_obligations").fetchone()[0]
  472. if total > _MAX_ROWS:
  473. conn.execute(
  474. """DELETE FROM delivery_obligations WHERE obligation_id IN (
  475. SELECT obligation_id FROM delivery_obligations
  476. ORDER BY CASE state
  477. WHEN 'delivered' THEN 0
  478. WHEN 'abandoned' THEN 1
  479. ELSE 2
  480. END, updated_at ASC
  481. LIMIT ?)""", (total - _MAX_ROWS,))
  482. def ledger_enabled(config: Optional[dict[str, Any]] = None) -> bool:
  483. """Read the ``gateway.delivery_ledger`` config gate (default on)."""
  484. try:
  485. if config is None:
  486. from hermes_cli.config import load_config
  487. config = load_config()
  488. value = (config.get("gateway") or {}).get("delivery_ledger", True)
  489. return value.strip().lower() not in {"false", "0", "no", "off"} if isinstance(value, str) else bool(value)
  490. except Exception:
  491. return True