resources.py 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. """Host resources the screen needs before it starts: memory.
  2. A headed desktop is a big allocation relative to a small gateway host: Xvnc + Xfce idle at ~220 MiB
  3. and the browser a human opens during a takeover adds 500 MiB to 1 GiB (measured in the official image:
  4. gateway 304 MiB → +Xfce 520 MiB → +Chromium on one page 1,073 MiB). On a 4 GB container the loser
  5. of that squeeze is Chromium, killed mid-login, or the gateway itself. So ``runtime.start()`` refuses
  6. while the host cannot spare ``bot_desktop.min_free_memory_mb`` and ``display.status`` says why, which
  7. the pane shows in place of the Start button.
  8. "Available" is the tighter of two numbers: what the cgroup still allows (a container limit; v2
  9. ``memory.max``/``memory.current``, v1 ``memory.limit_in_bytes``/``memory.usage_in_bytes``) and what
  10. the kernel reports free for the whole machine (``MemAvailable``) — a limit of 8 GB on a 4 GB host is
  11. not 8 GB.
  12. """
  13. from __future__ import annotations
  14. from dataclasses import dataclass
  15. from pathlib import Path
  16. from typing import Optional
  17. _CGROUP_V2 = Path("/sys/fs/cgroup")
  18. _CGROUP_V1 = Path("/sys/fs/cgroup/memory")
  19. _MEMINFO = Path("/proc/meminfo")
  20. _MIB = 1024 * 1024
  21. DEFAULT_MIN_FREE_MB = 1536
  22. @dataclass
  23. class MemoryInfo:
  24. available_mb: Optional[int] # None when nothing readable (non-Linux, odd sandboxes)
  25. limit_mb: Optional[int] # the cgroup limit when there is one, else the machine's MemTotal
  26. def _read_int(path: Path) -> Optional[int]:
  27. try:
  28. text = path.read_text(encoding="utf-8-sig").strip()
  29. except OSError:
  30. return None
  31. return int(text) if text.isdigit() else None # "max" → None
  32. def _meminfo() -> dict[str, int]:
  33. try:
  34. lines = _MEMINFO.read_text(encoding="utf-8-sig").splitlines()
  35. except OSError:
  36. return {}
  37. out: dict[str, int] = {}
  38. for line in lines:
  39. key, _, rest = line.partition(":")
  40. parts = rest.split()
  41. if parts and parts[0].isdigit():
  42. out[key] = int(parts[0]) * 1024 # kB
  43. return out
  44. def _stat_value(path: Path, key: str) -> Optional[int]:
  45. """One ``<key> <bytes>`` line out of a cgroup ``memory.stat``."""
  46. try:
  47. for line in path.read_text(encoding="utf-8-sig").splitlines():
  48. name, _, rest = line.partition(" ")
  49. if name == key:
  50. return int(rest.strip())
  51. except (OSError, ValueError):
  52. return None
  53. return None
  54. def _cgroup_limit_and_usage() -> tuple[Optional[int], Optional[int]]:
  55. """The cgroup's limit and its *working set* — usage minus reclaimable page cache.
  56. ``memory.current`` counts page cache, so a container reads several hundred MB above idle right after a
  57. desktop stops with every process gone. Charging that would make the gate tighten over uptime, so we
  58. subtract ``inactive_file``, the working-set convention kubelet uses.
  59. """
  60. limit = _read_int(_CGROUP_V2 / "memory.max")
  61. usage = _read_int(_CGROUP_V2 / "memory.current")
  62. cache = _stat_value(_CGROUP_V2 / "memory.stat", "inactive_file")
  63. if limit is None and usage is None:
  64. limit = _read_int(_CGROUP_V1 / "memory.limit_in_bytes")
  65. usage = _read_int(_CGROUP_V1 / "memory.usage_in_bytes")
  66. cache = _stat_value(_CGROUP_V1 / "memory.stat", "total_inactive_file")
  67. if limit is not None and limit >= 1 << 60: # v1 "unlimited" is a huge sentinel
  68. limit = None
  69. if usage is not None and cache:
  70. usage = max(usage - cache, 0)
  71. return limit, usage
  72. def memory_info() -> MemoryInfo:
  73. info = _meminfo()
  74. host_available = info.get("MemAvailable")
  75. host_total = info.get("MemTotal")
  76. limit, usage = _cgroup_limit_and_usage()
  77. candidates = [x for x in (host_available, (limit - usage) if limit is not None and usage is not None else None)
  78. if x is not None]
  79. available = min(candidates) if candidates else None
  80. limit_bytes = limit if limit is not None else host_total
  81. return MemoryInfo(available_mb=None if available is None else max(0, available // _MIB),
  82. limit_mb=None if limit_bytes is None else limit_bytes // _MIB)
  83. def min_free_mb() -> int:
  84. """``bot_desktop.min_free_memory_mb``; 0 disables the gate. A hosted deployment sets it in the
  85. instance's config.yaml (or the managed overlay), not an env var."""
  86. from hermes_cli.config import load_config_readonly
  87. cfg = load_config_readonly().get("bot_desktop") or {}
  88. try:
  89. return max(0, int(cfg.get("min_free_memory_mb", DEFAULT_MIN_FREE_MB)))
  90. except (TypeError, ValueError):
  91. return DEFAULT_MIN_FREE_MB
  92. def tight_headroom_mb(floor: Optional[int] = None) -> int:
  93. """Above the floor but below this, a start is allowed and logged: the desktop fits, a few browser tabs
  94. would not. Derived from the floor so raising the floor cannot silently retire the warning."""
  95. floor = min_free_mb() if floor is None else floor
  96. return floor + floor // 3
  97. def memory_blocker(info: Optional[MemoryInfo] = None, need: Optional[int] = None) -> Optional[str]:
  98. """Why the screen must not start now, or None. Unknown memory is not a blocker: a host we cannot
  99. read is not a host we know to be small. ``need`` lets a caller that also wants
  100. :func:`tight_headroom_mb` read the floor once instead of loading the config twice."""
  101. need = min_free_mb() if need is None else need
  102. if need == 0:
  103. return None
  104. info = info or memory_info()
  105. if info.available_mb is None or info.available_mb >= need:
  106. return None
  107. limit = f" of {info.limit_mb} MB" if info.limit_mb else ""
  108. return (f"Not enough free memory for a desktop: {info.available_mb} MB available{limit}, "
  109. f"{need} MB needed (bot_desktop.min_free_memory_mb)")