stream-staleness.mjs 3.9 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586
  1. // Pure decision helpers for the zombie-stream (half-open gRPC) watchdog.
  2. //
  3. // spectrum-ts only reconnects when its inbound async iterator throws or ends.
  4. // A half-open ("zombie") socket makes the iterator hang forever — no error,
  5. // no end — so inbound silently dies while /healthz still looks fine. The
  6. // watchdog in index.mjs tracks the last time the inbound iterator yielded and,
  7. // once the stream has been silent past a conservative threshold, drives a
  8. // cheap authenticated unary read over the same channel. STRICT semantics:
  9. //
  10. // - probe resolves, or rejects with a not-found-shaped error for our
  11. // synthetic id -> ALIVE (the wire round-tripped)
  12. // - probe rejects any other way (UNAVAILABLE, DEADLINE_EXCEEDED, network
  13. // down, ...) -> INCONCLUSIVE — never treated as alive, and
  14. // never treated as zombie-proof either
  15. //
  16. // A zombie is only declared when the stream is silent past the threshold AND
  17. // a probe proves connectivity (the wire works but the stream is deaf). Silence
  18. // alone NEVER degrades the stream: shared lines can be legitimately quiet for
  19. // hours. Inconclusive probes NEVER degrade it either: the network may simply
  20. // be down, and in that case the iterator will eventually throw and the
  21. // existing re-subscribe loop recovers on its own.
  22. //
  23. // These helpers are pure (no SDK, no timers) so tests can execute them under
  24. // node — see tests/plugins/platforms/photon/test_zombie_stream_watchdog.py.
  25. import { randomUUID } from "node:crypto";
  26. // gRPC NOT_FOUND is code 5; SDKs also surface it as "not found" / "NotFound"
  27. // message text. Anything not clearly not-found is inconclusive.
  28. const NOT_FOUND_RE = /not[\s_-]?found/i;
  29. /** Return a unique message id accepted by Spectrum's message-id parser. */
  30. export function createProbeMessageId() {
  31. return randomUUID();
  32. }
  33. /**
  34. * Classify the rejection of the synthetic-id probe read.
  35. *
  36. * @param {unknown} err error thrown by `space.getMessage(<synthetic id>)`
  37. * @returns {{alive: boolean, inconclusive: boolean, reason: string}}
  38. */
  39. export function classifyProbeRejection(err) {
  40. const code = err && typeof err === "object" ? err.code : undefined;
  41. const message =
  42. err && typeof err === "object" && err.message
  43. ? String(err.message)
  44. : String(err);
  45. if (code === 5 || code === "notFound" || NOT_FOUND_RE.test(message)) {
  46. // Expected: the synthetic id doesn't exist. The unary call completed a
  47. // round-trip, so the channel is provably alive.
  48. return { alive: true, inconclusive: false, reason: "not-found round-trip" };
  49. }
  50. // Anything else (UNAVAILABLE, DEADLINE_EXCEEDED, TLS, auth, ...) does NOT
  51. // prove liveness — and doesn't prove a zombie either.
  52. return { alive: false, inconclusive: true, reason: message };
  53. }
  54. /**
  55. * Should the watchdog probe at all this tick?
  56. *
  57. * @param {number} silentForMs ms since the inbound iterator last yielded
  58. * @param {number} thresholdMs silence threshold (<= 0 disables the watchdog)
  59. * @param {number} sinceLastProbeMs ms since the previous probe attempt
  60. * @param {number} probeCooldownMs min spacing between probe attempts
  61. * @returns {boolean}
  62. */
  63. export function shouldProbe(silentForMs, thresholdMs, sinceLastProbeMs, probeCooldownMs) {
  64. if (!(thresholdMs > 0)) return false;
  65. if (silentForMs < thresholdMs) return false;
  66. return sinceLastProbeMs >= probeCooldownMs;
  67. }
  68. /**
  69. * Final classification: zombie only on silence past threshold + probe-proven
  70. * connectivity. Never on silence alone, never on an inconclusive probe.
  71. *
  72. * @param {number} silentForMs ms since the inbound iterator last yielded
  73. * @param {number} thresholdMs silence threshold (<= 0 disables the watchdog)
  74. * @param {{alive: boolean}} probeOutcome
  75. * @returns {boolean}
  76. */
  77. export function isZombieSuspect(silentForMs, thresholdMs, probeOutcome) {
  78. if (!(thresholdMs > 0)) return false;
  79. if (silentForMs < thresholdMs) return false;
  80. return probeOutcome != null && probeOutcome.alive === true;
  81. }