drain_control.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. """External drain-control marker contract (dashboard → gateway).
  2. No control channel exists into a running gateway, so begin/cancel-drain writes
  3. (or removes) ``{HERMES_HOME}/.drain_request.json`` and a gateway watcher reacts;
  4. an ACTIVE marker means ``gateway_state -> "draining"``. Two lenient staleness
  5. signals (either suffices): epoch mismatch (HERMES_HOME is a durable volume on
  6. Hermes Cloud, so a marker survives the restart a drain-gated action ends in and
  7. would park the fresh gateway in ``draining`` forever) and expiry (same-epoch
  8. orphan past :data:`DRAIN_REQUEST_MAX_AGE_SECONDS`; re-writing refreshes it).
  9. Reading never raises: a malformed file reads as ``{}`` — still drain-active
  10. (fail-safe toward quiescing). Staleness rejects only on a *definite* verdict.
  11. """
  12. from __future__ import annotations
  13. import contextlib
  14. import functools
  15. import json
  16. import logging
  17. import re
  18. from datetime import datetime, timezone, UTC
  19. from pathlib import Path
  20. from typing import Any, Optional
  21. from gateway.memory_status import _parse_iso
  22. from hermes_constants import get_hermes_home
  23. from utils import atomic_json_write
  24. _log = logging.getLogger(__name__)
  25. # ``drain_reason`` a graceful restart publishes for its after-turn wait (#139041). The owner still
  26. # runs its in-flight turns, so a client may attach to answer a parked approval/clarify; new input is
  27. # committed queued for the next owner and never claimed here. ``stop()`` withdraws it.
  28. RESTART_WAIT_REASON = "restart_wait"
  29. def open_restart_wait(runner) -> None:
  30. """Publish the restart's after-turn wait (``drain_reason``) unless a drain already named its own."""
  31. descriptor = getattr(runner, "session_runtime_descriptor", None)
  32. if descriptor is not None and not descriptor.get("drain_reason"):
  33. descriptor["drain_reason"] = RESTART_WAIT_REASON
  34. def restart_wait_open(runner) -> bool:
  35. """True while *runner* is in a restart's after-turn wait (draining, not yet stopping)."""
  36. descriptor = getattr(runner, "session_runtime_descriptor", None) or {}
  37. return (bool(getattr(runner, "_draining", False)) and bool(getattr(runner, "_running", False))
  38. and descriptor.get("drain_reason") == RESTART_WAIT_REASON)
  39. # The requests a client needs during that wait to find, open and answer the conversation parked on
  40. # approval/clarify: the GET reads below (Desktop's session picker, the session detail and its
  41. # transcript pages, and the health probes) and one write, a Webapp tab's mint of the one-use ticket
  42. # its /api/ws dial-back needs (restart_wait_socket_ticket). Both only pass the drain's refusal; the
  43. # listener's auth gate still runs on them. Every other write and route keeps that refusal. The
  44. # prompt itself (approval.pending / *.respond) rides the WebSocket.
  45. _RESTART_WAIT_READS = re.compile(
  46. r"/api/(?:health|status|profiles/sessions(?:/sidebar)?"
  47. r"|sessions/[^/]+(?:/messages(?:/around)?|/timeline)?)")
  48. def restart_wait_read(runner, method, path) -> bool:
  49. """True for an allowlisted read request while *runner* is in a restart's after-turn wait."""
  50. return (method == "GET" and bool(_RESTART_WAIT_READS.fullmatch(path or ""))
  51. and restart_wait_open(runner))
  52. def restart_wait_socket_ticket(runner, method, path, query) -> bool:
  53. """True for a gateway-socket ticket mint while *runner* is in a restart's after-turn wait.
  54. Exempts the request from the drain's refusal only: it does not authenticate it."""
  55. from urllib.parse import parse_qs
  56. from hermes_cli.web_session_sockets import GATEWAY_SOCKET
  57. return (method == "POST" and path == "/api/webapp/socket-ticket"
  58. and parse_qs(query or "").get("socket") == [GATEWAY_SOCKET] and restart_wait_open(runner))
  59. # The WebSocket verbs an interactive client may still call during that wait: attach to and read the
  60. # parked session, answer its prompt (or interrupt the turn so the restart can finish) and commit
  61. # queued input for the next owner. Every other verb (config.set, session.save, rollback.restore,
  62. # process.kill, reload.*, ...) would run on an owner that is shutting down, so it is refused with
  63. # ``runtime_draining`` exactly like the REST writes above.
  64. RESTART_WAIT_WS_METHODS = frozenset({
  65. "ping", "runtime.describe", "session.resume", "session.detach", "session.events.since",
  66. "session.info", "session.list", "prompt.submit", "prompt.receipt", "session.interrupt",
  67. "approval.pending", "approval.received", "approval.respond", "clarify.respond", "clarify.lock",
  68. })
  69. def ws_method_refused_while_draining(runner, method) -> bool:
  70. """True when an interactive WebSocket verb must be refused because *runner* is draining."""
  71. return bool(getattr(runner, "_draining", False)) and method not in RESTART_WAIT_WS_METHODS
  72. _DRAIN_REQUEST_FILENAME = ".drain_request.json"
  73. # Drain-gated lifecycle actions complete in minutes; an hour bounds the wedge a leaked
  74. # marker can cause. Long drains refresh the marker instead of raising this.
  75. # Max-age fallback for a same-epoch orphaned marker (#85433). Long-running drains refresh the marker via
  76. # write_drain_request() (idempotent re-write bumps ``requested_at``) rather than raising this bound.
  77. DRAIN_REQUEST_MAX_AGE_SECONDS = 3600.0
  78. # Dedup for the expired-marker warning (the watcher re-reads every second); keyed by
  79. # ``requested_at`` so a keep-alive re-write that later expires logs again.
  80. _expiry_logged_for: Optional[str] = None
  81. @functools.lru_cache(maxsize=1)
  82. def current_instantiation_epoch() -> str:
  83. """Identity of THIS container / VM instantiation ("<boot_id>:<pid1_start>").
  84. Stable for the life of PID 1 (a gateway-only respawn keeps honouring an
  85. in-flight drain) but changes when the machine is recreated: boot_id on a VM
  86. reboot, PID 1's start time on ``docker restart``. ``""`` when neither is
  87. readable (non-Linux, no ``/proc``) disables the epoch check — never fail-closed.
  88. """
  89. boot_id = pid1_start = ""
  90. with contextlib.suppress(OSError):
  91. boot_id = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="utf-8").strip()
  92. with contextlib.suppress(OSError, IndexError):
  93. # "<pid> (<comm>) <state> ...": comm may contain spaces/parens, so split on the
  94. # LAST ')'. starttime is field 22 (1-indexed) = tail index 19.
  95. pid1_start = Path("/proc/1/stat").read_text(encoding="utf-8").rsplit(")", 1)[1].split()[19]
  96. return f"{boot_id}:{pid1_start}" if (boot_id or pid1_start) else ""
  97. def drain_request_path(home: Optional[Path] = None) -> Path:
  98. """Absolute path to the drain-request marker, respecting HERMES_HOME."""
  99. return Path(home if home is not None else get_hermes_home()) / _DRAIN_REQUEST_FILENAME
  100. def write_drain_request(
  101. *, principal: str = "drain-control", suppress_notification: bool = False, home: Optional[Path] = None
  102. ) -> dict[str, Any]:
  103. """Write the begin-drain marker atomically; returns the payload.
  104. Re-writing refreshes ``requested_at`` (keep-alive past the max-age).
  105. ``suppress_notification`` skips ONLY the home-channel "gateway shutting down"
  106. broadcast (the per-session interrupt ping is never suppressed); which drains
  107. are quiet is the caller's policy. Stamped with the instantiation epoch so a
  108. copy surviving a machine restart on the durable volume reads as stale.
  109. """
  110. payload = {
  111. "action": "drain", "requested_at": datetime.now(UTC).isoformat(), "principal": principal,
  112. "epoch": current_instantiation_epoch(), "suppress_notification": bool(suppress_notification),
  113. }
  114. atomic_json_write(drain_request_path(home), payload)
  115. return payload
  116. def clear_drain_request(*, home: Optional[Path] = None) -> bool:
  117. """Remove the drain marker (cancel-drain, idempotent). Returns True if one existed."""
  118. path = drain_request_path(home)
  119. try:
  120. path.unlink()
  121. return True
  122. except OSError as e:
  123. if not isinstance(e, FileNotFoundError):
  124. _log.warning("drain-control: failed to remove %s: %s", path, e)
  125. return False
  126. def _marker_is_expired(body: dict[str, Any]) -> bool:
  127. """True iff ``requested_at`` parses AND is older than the max-age.
  128. Missing/unparseable and future-dated (clock skew) timestamps are honoured.
  129. Logged once per marker, not per poll — the operator's breadcrumb for a leak.
  130. See #85433.
  131. """
  132. global _expiry_logged_for
  133. raw = body.get("requested_at")
  134. requested_at = _parse_iso(raw)
  135. if requested_at is None:
  136. return False
  137. age = (datetime.now(UTC) - requested_at).total_seconds()
  138. if age <= DRAIN_REQUEST_MAX_AGE_SECONDS:
  139. return False
  140. if _expiry_logged_for != raw:
  141. _expiry_logged_for = raw
  142. _log.warning(
  143. "drain-control: ignoring expired drain marker (requested_at=%s, age=%.0fs > max %.0fs, principal=%s) "
  144. "— the drain that wrote it was never cancelled; treating as stale so the gateway keeps accepting turns.",
  145. raw, age, DRAIN_REQUEST_MAX_AGE_SECONDS, body.get("principal"),
  146. )
  147. return True
  148. def _active_drain_body(home: Optional[Path]) -> Optional[dict[str, Any]]:
  149. """Marker body if present AND not stale (definite epoch mismatch or expired), else None."""
  150. body = read_drain_request(home=home)
  151. if body is None:
  152. return None
  153. current, marker_epoch = current_instantiation_epoch(), body.get("epoch")
  154. if (current and marker_epoch and marker_epoch != current) or _marker_is_expired(body):
  155. return None
  156. return body
  157. def drain_requested(*, home: Optional[Path] = None) -> bool:
  158. """True iff an active (present, same-epoch, unexpired) begin-drain marker exists.
  159. A marker whose ``epoch`` does not match the current instantiation epoch is treated as absent: it
  160. survived a container/VM restart (HERMES_HOME is a durable Fly volume on Hermes Cloud) and the lifecycle
  161. action that triggered the drain has already completed — honouring it would wedge the freshly-restarted
  162. gateway in ``draining`` (NS-570). A marker whose ``requested_at`` is older than
  163. :data:`DRAIN_REQUEST_MAX_AGE_SECONDS` is likewise treated as absent: it is a same-epoch orphan whose
  164. drain-gated action completed without a restart and was never cancelled (#85433). Both staleness checks
  165. are lenient (see :func:`_marker_epoch_is_stale` / :func:`_marker_is_expired`): a legacy/corrupt marker
  166. with no epoch and no timestamp, or an environment without ``/proc``, still reads as drain-active.
  167. """
  168. return _active_drain_body(home) is not None
  169. def drain_notification_suppressed(*, home: Optional[Path] = None) -> bool:
  170. """True iff an ACTIVE marker asks to suppress the shutdown broadcast.
  171. Same activeness rule as :func:`drain_requested`, so an orphan can never silence
  172. a fresh gateway; a marker without the field reads False (fail toward louder).
  173. "Active" means exactly what :func:`drain_requested` means — a marker present AND stamped with the
  174. current instantiation epoch AND not past its max-age. A stale (other-epoch) marker that survived a
  175. machine restart on the durable HERMES_HOME volume, or an expired same-epoch orphan (#85433), is ignored
  176. here just as it is for drain state (NS-570): we must never let an orphaned marker's flag silence a
  177. *fresh* gateway's legitimate shutdown broadcast.
  178. """
  179. body = _active_drain_body(home)
  180. return bool(body and body.get("suppress_notification"))
  181. def read_drain_request(*, home: Optional[Path] = None) -> Optional[dict[str, Any]]:
  182. """Return the marker payload, ``{}`` if present but unparseable, ``None`` if absent. Never raises."""
  183. path = drain_request_path(home)
  184. try:
  185. raw = path.read_text(encoding="utf-8-sig")
  186. except OSError as e:
  187. if not isinstance(e, FileNotFoundError):
  188. _log.warning("drain-control: failed to read %s: %s", path, e)
  189. return None
  190. try:
  191. data = json.loads(raw)
  192. except (ValueError, TypeError):
  193. return {}
  194. return data if isinstance(data, dict) else {}