systemd_notify.py 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119
  1. """Minimal, optional systemd ``sd_notify`` support for the gateway."""
  2. from __future__ import annotations
  3. import asyncio
  4. import contextlib
  5. import math
  6. import os
  7. import socket
  8. def _notify_socket() -> str:
  9. return os.environ.get("NOTIFY_SOCKET", "").strip() if hasattr(socket, "AF_UNIX") else ""
  10. def notify(message: str) -> bool:
  11. """Send an sd_notify datagram if systemd configured it; failures never block gateway startup."""
  12. if not (address := _notify_socket()) or not isinstance(message, str) or not message:
  13. return False
  14. try:
  15. with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sender:
  16. sender.setblocking(False) # a full receiver buffer must not stall the event loop
  17. # systemd's ``@abstract`` notation -> Python's leading-NUL address form
  18. sender.connect("\0" + address[1:] if address.startswith("@") else address)
  19. sender.send(message.encode("utf-8"))
  20. return True
  21. except (OSError, UnicodeError, ValueError):
  22. return False
  23. def watchdog_interval_seconds() -> float | None:
  24. try:
  25. interval = float(os.environ.get("WATCHDOG_USEC", "") if _notify_socket() else "") / 1e6
  26. except (TypeError, ValueError):
  27. return None
  28. return interval if math.isfinite(interval) and interval > 0 else None
  29. class SystemdWatchdog:
  30. """Feed systemd while the asyncio event loop continues to make progress."""
  31. def __init__(self, *, config_enabled: bool = True, lag_tolerance_seconds: float | None = None):
  32. self._config_enabled = bool(config_enabled)
  33. self.interval_seconds = watchdog_interval_seconds()
  34. self._lag_tolerance_seconds = lag_tolerance_seconds
  35. self._task: asyncio.Task[None] | None = None
  36. self._unhealthy = self._stopping = self._stopping_notified = False
  37. enabled = property(lambda self: self._config_enabled and self.interval_seconds is not None)
  38. unhealthy = property(lambda self: self._unhealthy)
  39. task = property(lambda self: self._task)
  40. def _lag_tolerance(self) -> float:
  41. default = max(0.1, (self.interval_seconds or 0.0) * 0.25)
  42. with contextlib.suppress(TypeError, ValueError):
  43. value = float(self._lag_tolerance_seconds)
  44. return max(0.0, value) if math.isfinite(value) else default
  45. return default
  46. def start(self) -> bool:
  47. if not self.enabled:
  48. return False
  49. if self._task is not None and not self._task.done():
  50. return True
  51. try:
  52. asyncio.get_running_loop()
  53. except RuntimeError:
  54. return False
  55. self._stopping = self._unhealthy = self._stopping_notified = False
  56. self._task = asyncio.create_task(self._run(), name="hermes-systemd-watchdog")
  57. return True
  58. def ready(self, status: str = "Gateway running") -> bool:
  59. safe_status = str(status or "Gateway running").replace("\n", " ")
  60. return self.enabled and notify(f"READY=1\nSTATUS={safe_status}")
  61. def record_tick(self, *, scheduled_at: float, now: float) -> bool:
  62. """Feed systemd only when the event loop woke within its lag budget."""
  63. if not self.enabled or self._stopping or self._unhealthy:
  64. return False
  65. try:
  66. lag = float(now) - float(scheduled_at)
  67. except (TypeError, ValueError):
  68. lag = float("inf")
  69. if not math.isfinite(lag) or lag > self._lag_tolerance():
  70. self._unhealthy = True
  71. notify("STATUS=watchdog unhealthy: event loop progress is late")
  72. return False
  73. notify("WATCHDOG=1")
  74. return True
  75. async def _run(self) -> None:
  76. if self.interval_seconds is None:
  77. return
  78. cadence = max(0.01, self.interval_seconds / 2.0)
  79. loop = asyncio.get_running_loop()
  80. scheduled_at = loop.time() + cadence
  81. with contextlib.suppress(asyncio.CancelledError):
  82. while not self._stopping and not self._unhealthy:
  83. await asyncio.sleep(max(0.0, scheduled_at - loop.time()))
  84. now = loop.time()
  85. if not self.record_tick(scheduled_at=scheduled_at, now=now):
  86. return
  87. scheduled_at += cadence
  88. if scheduled_at < now:
  89. scheduled_at = now + cadence
  90. async def stop(self) -> None:
  91. """Stop feeding systemd and emit ``STOPPING=1`` at most once."""
  92. self._stopping = True
  93. task = self._task
  94. if task is not None and task is not asyncio.current_task():
  95. task.cancel() # no-op on a finished task
  96. with contextlib.suppress(asyncio.CancelledError, Exception):
  97. await task
  98. self._task = None
  99. if self.enabled and not self._stopping_notified:
  100. notify("STOPPING=1")
  101. self._stopping_notified = True