scale_to_zero.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271
  1. """Scale-to-zero idle detection + dormant-quiesce for the gateway.
  2. Owns the *decision* to go idle, drives the relay transport's ``go_dormant()``, then SUSPENDS the
  3. machine via the local Fly Machines API socket; wake stays platform-side (autostart on wakeUrl).
  4. Self-suspend because Fly Proxy only sees INBOUND proxied connections: it would suspend mid-turn or
  5. before ``go_dormant()`` flipped the relay destination (buffered-event black hole). Enable is gated
  6. SOLELY by the NAS "Labs" toggle env stamp (not config); the idle timeout IS config.yaml. Quiesce
  7. uses ``go_dormant()`` (never disconnect/drain); ``mark_resume_pending`` is NOT called: suspend
  8. preserves RAM."""
  9. from __future__ import annotations
  10. import json
  11. import logging
  12. import os
  13. import socket
  14. import time
  15. import urllib.parse
  16. from pathlib import Path
  17. from typing import Any, Iterable, Optional
  18. logger = logging.getLogger(__name__)
  19. SCALE_TO_ZERO_ENV = "HERMES_SCALE_TO_ZERO" # stamped by NAS when the Labs toggle is on
  20. FLY_APP_NAME_ENV = "FLY_APP_NAME" # Fly-injected identity; both needed for self_suspend_available()
  21. FLY_MACHINE_ID_ENV = "FLY_MACHINE_ID"
  22. # Local flaps (Fly Machines API) socket; POST .../suspend freezes THIS machine.
  23. FLY_API_SOCKET = "/.fly/api"
  24. # NAS-brokered suspend, stamped where the guest cannot suspend itself. Carries
  25. # its own signed credential in the query string, like GATEWAY_RELAY_WAKE_URL.
  26. SLEEP_URL_ENV = "GATEWAY_RELAY_SLEEP_URL"
  27. _malformed_sleep_url_logged = False
  28. # Short is safe: real work always blocks the suspend, resume is sub-second; longer bills idle RAM.
  29. DEFAULT_IDLE_TIMEOUT_MINUTES = 2
  30. _TRUTHY = {"1", "true", "yes", "on"}
  31. # Dashboard-client liveness marker, touched by every /api/ws client socket on each
  32. # connect and inbound frame. Folded into the inbound clock so an open client holds the box awake —
  33. # otherwise it suspends under the client, whose reconnect re-pokes the wake URL and the instance
  34. # flaps every ~60s. Deliberately NO staleness cutoff: is_idle decides whether the mtime is recent.
  35. DASHBOARD_CLIENT_HEARTBEAT_REL = os.path.join("state", "dashboard_clients.heartbeat")
  36. def _env_str(env: Optional[dict], key: str) -> str:
  37. return str((os.environ if env is None else env).get(key, "")).strip()
  38. def scale_to_zero_enabled(environ: Optional[dict] = None) -> bool:
  39. """Whether the Labs toggle stamp is set. Absent/blank/falsey -> disabled."""
  40. return _env_str(environ, SCALE_TO_ZERO_ENV).lower() in _TRUTHY
  41. def parse_idle_timeout_seconds(cfg_value: Any,
  42. default_minutes: int = DEFAULT_IDLE_TIMEOUT_MINUTES) -> float:
  43. """Coerce ``scale_to_zero.idle_timeout_minutes`` to seconds. Non-numeric / non-positive
  44. degrades to the default (never <= 0: instant dormancy)."""
  45. try:
  46. minutes = float(cfg_value)
  47. except (TypeError, ValueError):
  48. minutes = 0.0
  49. return (float(default_minutes) if minutes <= 0 else minutes) * 60.0
  50. def messaging_is_relay_only_or_absent(platforms: Iterable[Any]) -> bool:
  51. """True iff the only connected platform is RELAY, or there is none. A directly-connected
  52. platform holds a live socket and cannot scale to zero. Compared by ``.value``/name so this
  53. module stays enum-import-free."""
  54. names = {str(getattr(p, "value", p)).strip().lower() for p in platforms}
  55. names.discard("relay")
  56. return not names
  57. def should_arm(*, enabled: bool, relay_only_or_absent: bool, wake_url: Optional[str]) -> bool:
  58. """Arm only if ALL hold: flag on, relay-only/absent messaging, wakeUrl registered
  59. (a suspended instance with no wake target is a black hole). Otherwise the watcher
  60. never starts, so a non-opted instance behaves exactly as before."""
  61. return bool(enabled) and bool(relay_only_or_absent) and bool(wake_url)
  62. def is_idle(*, active_work_count: int, seconds_since_last_inbound: float,
  63. idle_timeout_seconds: float, has_live_background_work: bool) -> bool:
  64. """Pure idle predicate: no active work, no inbound within the window, no live background work.
  65. ``active_work_count`` is the BROAD aggregate (agent turns + cron + API runs) — passing only
  66. ``len(_running_agents)`` reopens the mid-cron-job suspend hole. Callers that cannot read a
  67. work source must fail AWAKE (pass a positive sentinel), never fail to 0."""
  68. return (active_work_count <= 0 and not has_live_background_work
  69. and seconds_since_last_inbound >= idle_timeout_seconds)
  70. def dashboard_client_heartbeat_path(hermes_home: Optional[os.PathLike | str] = None):
  71. """Path of the dashboard-client liveness marker under the process's own HERMES_HOME.
  72. One marker per process, never per profile scope: the gateway's idle watcher reads it with no
  73. override, so a socket served in a secondary profile's scope must stamp the same file."""
  74. if hermes_home is None:
  75. from hermes_constants import get_process_hermes_home
  76. hermes_home = get_process_hermes_home()
  77. return Path(hermes_home) / DASHBOARD_CLIENT_HEARTBEAT_REL
  78. def touch_dashboard_client_heartbeat(path: Optional[os.PathLike | str] = None) -> bool:
  79. """Mark "a dashboard client is attached right now". Best-effort, never raises."""
  80. try:
  81. p = dashboard_client_heartbeat_path() if path is None else path
  82. os.makedirs(os.path.dirname(p), exist_ok=True)
  83. open(p, "a", encoding="utf-8").close()
  84. os.utime(p, None)
  85. return True
  86. except Exception:
  87. logger.debug("scale-to-zero: dashboard heartbeat touch failed", exc_info=True)
  88. return False
  89. def dashboard_client_last_seen(path: Optional[os.PathLike | str] = None, *,
  90. now: Optional[float] = None) -> Optional[float]:
  91. """Epoch seconds a dashboard client last sent a WS frame, or None if never. Missing marker ->
  92. None (steady state when nobody has the dashboard open — NOT fail-awake, or no instance would
  93. ever sleep). Unreadable marker -> ``now`` (fail-awake, as in ``is_idle``). Clamped to now: an
  94. NTP step-back can leave the mtime in the future."""
  95. current = time.time() if now is None else now
  96. p = dashboard_client_heartbeat_path() if path is None else path
  97. try:
  98. return min(os.stat(p).st_mtime, current)
  99. except FileNotFoundError:
  100. return None
  101. except OSError:
  102. return current
  103. def self_suspend_available(environ: Optional[dict] = None) -> bool:
  104. """True iff Fly machine identity is present AND the local Machines API socket exists.
  105. Off-Fly this is False; see ``suspend_available`` for whether some OTHER lever exists
  106. before concluding the watcher must abstain."""
  107. return bool(_env_str(environ, FLY_APP_NAME_ENV) and _env_str(environ, FLY_MACHINE_ID_ENV)
  108. and os.path.exists(FLY_API_SOCKET))
  109. def brokered_sleep_url(environ: Optional[dict] = None) -> Optional[str]:
  110. """The NAS sleep endpoint to POST, or None when this backend has no broker.
  111. Validated here rather than at POST time: a malformed value would otherwise let
  112. the watcher mark draining, hold the re-dial and flip the connector before
  113. urllib rejected it, quiescing for a suspend that could never happen.
  114. """
  115. env = environ if environ is not None else os.environ
  116. url = str(env.get(SLEEP_URL_ENV, "")).strip()
  117. if not url:
  118. return None
  119. parsed = urllib.parse.urlsplit(url)
  120. if parsed.scheme != "https" or not parsed.netloc:
  121. # Once per process: the watcher calls this every idle tick, and its own
  122. # no-lever latch sits AFTER this, so an unlatched warning here would
  123. # repeat for the life of a misconfigured deployment.
  124. global _malformed_sleep_url_logged
  125. if not _malformed_sleep_url_logged:
  126. _malformed_sleep_url_logged = True
  127. logger.warning(
  128. "scale-to-zero: ignoring malformed %s (want an absolute https URL)",
  129. SLEEP_URL_ENV,
  130. )
  131. return None
  132. return url
  133. def suspend_available(environ: Optional[dict] = None) -> bool:
  134. """Whether ANY suspend lever exists, in-guest or brokered.
  135. Quiescing without one is worse than not quiescing: the re-dial clears the flip.
  136. """
  137. env = environ if environ is not None else os.environ
  138. return self_suspend_available(env) or brokered_sleep_url(env) is not None
  139. # Must EXCEED the broker's own hard request ceiling (NAS route maxDuration = 30s),
  140. # or we give up while it is still working: the redial hold would be released, the
  141. # re-dial would clear the flip, and its stop would then land on a live destination.
  142. BROKERED_SUSPEND_TIMEOUT_S = 40.0
  143. # Flaps answers a suspend BEFORE the kernel freezes (suspend_self is documented
  144. # fire-and-forget), so the re-dial fence has to outlive the 2xx or a re-dial can
  145. # restore a live destination in the gap. Measured on Fly (gru, shared-cpu-4x/2GB,
  146. # 2026-09-03) across four suspends: the freeze landed 2.25s, 3.83s, 4.06s and
  147. # 4.19s after the 2xx. The gap is a RAM snapshot, so it grows with machine size —
  148. # 5.0s left only 0.81s of margin on the smallest instance and would be too short
  149. # on a larger one, which is the failure this fence exists to prevent.
  150. #
  151. # Sized generously because the fence is NOT paid on wake: the watcher slices it on
  152. # the wall clock (see _scale_to_zero_await_freeze_gap), so an overshoot costs only
  153. # the reconnect delay on the rare flaps-accepted-but-never-froze path, itself
  154. # capped by ws_transport.REDIAL_HOLD_MAX_S.
  155. FLY_FREEZE_GRACE_S = 15.0
  156. # Poll step for that wall-clock slice. Bounds how long after a resume the fence
  157. # lingers before the drain re-dial (the deadline is already past by then).
  158. FLY_FREEZE_GRACE_TICK_S = 0.25
  159. def request_brokered_suspend(
  160. url: str,
  161. *,
  162. timeout: float = BROKERED_SUSPEND_TIMEOUT_S,
  163. opener: Any = None,
  164. ) -> bool:
  165. """POST the NAS sleep URL so NAS stops this machine on our behalf.
  166. Same contract as ``suspend_self``: never raises, True only on 2xx, fail-awake.
  167. """
  168. import urllib.error
  169. import urllib.request
  170. # urllib sets Content-Length itself for a bytes body.
  171. request = urllib.request.Request(url, data=b"", method="POST")
  172. open_url = opener or urllib.request.urlopen
  173. try:
  174. with open_url(request, timeout=timeout) as response:
  175. status = int(getattr(response, "status", 0) or 0)
  176. except urllib.error.HTTPError as exc:
  177. # Not retried here: the watcher re-runs on its own interval.
  178. logger.warning(
  179. "scale-to-zero: brokered suspend rejected: %s %s",
  180. exc.code,
  181. exc.reason,
  182. )
  183. return False
  184. except (urllib.error.URLError, OSError, ValueError) as exc:
  185. logger.warning("scale-to-zero: brokered suspend request failed: %s", exc)
  186. return False
  187. ok = 200 <= status < 300
  188. if ok:
  189. logger.info("scale-to-zero: machine suspend accepted by NAS (%s)", status)
  190. else:
  191. logger.warning("scale-to-zero: brokered suspend returned %s", status)
  192. return ok
  193. def suspend_self(environ: Optional[dict] = None, *, socket_path: str = FLY_API_SOCKET,
  194. timeout: float = 10.0) -> bool:
  195. """POST /v1/apps/{app}/machines/{id}/suspend on the local flaps socket (the socket is the
  196. credential). Returns True when flaps accepted (2xx); the kernel then freezes this process
  197. shortly after, so treat as fire-and-forget. Never raises: a failed suspend leaves the machine
  198. running (fail-awake). stdlib-only on purpose — a plain unix-socket HTTP/1.1 request, no async
  199. plumbing to freeze mid-await."""
  200. app, machine_id = _env_str(environ, FLY_APP_NAME_ENV), _env_str(environ, FLY_MACHINE_ID_ENV)
  201. if not app or not machine_id:
  202. logger.warning("scale-to-zero: suspend_self called without Fly machine identity")
  203. return False
  204. request = (f"POST /v1/apps/{app}/machines/{machine_id}/suspend HTTP/1.1\r\n"
  205. "Host: flaps\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
  206. try:
  207. with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
  208. sock.settimeout(timeout)
  209. sock.connect(socket_path)
  210. sock.sendall(request.encode("ascii"))
  211. response = b""
  212. while len(response) < 65536 and (chunk := sock.recv(4096)):
  213. response += chunk
  214. except OSError as exc:
  215. logger.warning("scale-to-zero: flaps suspend request failed: %s", exc)
  216. return False
  217. status_line = response.split(b"\r\n", 1)[0].decode("ascii", "replace")
  218. parts = status_line.split()
  219. ok = len(parts) >= 2 and parts[1].isdigit() and 200 <= int(parts[1]) < 300
  220. if ok:
  221. logger.info("scale-to-zero: machine suspend accepted by flaps (%s)", status_line)
  222. else:
  223. body = response.split(b"\r\n\r\n", 1)[-1][:500].decode("utf-8", "replace")
  224. logger.warning("scale-to-zero: flaps suspend rejected: %s %s", status_line,
  225. json.dumps(body)[:500])
  226. return ok