store_health.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301
  1. """Degraded state for an unwritable cron store (ENOSPC, EROFS, EACCES).
  2. A full disk or read-only mount fails every store write on every 60s tick. Instead of one warning
  3. per failing call site, each store directory gets ONE in-memory record: set by the first failed
  4. write, updated by later ones, cleared by the first jobs.json save that lands. While it is set,
  5. the tick skips the advance/claim work that can only fail and re-probes the store at most once a
  6. minute. ``probe_store`` is also what ``hermes cron status`` and ``hermes doctor`` use: they run in
  7. another process, so they cannot read this record or trust markers in a directory nobody can write.
  8. """
  9. from __future__ import annotations
  10. import contextlib
  11. import errno
  12. import logging
  13. import os
  14. import shutil
  15. import threading
  16. import time
  17. from dataclasses import dataclass, field
  18. from datetime import datetime
  19. from pathlib import Path
  20. from typing import Callable, Iterable, Optional
  21. logger = logging.getLogger("cron.jobs")
  22. PROBE_INTERVAL_SECONDS = 60.0
  23. # Store-write failures that degrade the store (the tick keeps running) instead of failing the tick.
  24. UNWRITABLE_ERRNOS = frozenset({errno.EROFS, errno.EACCES, errno.EPERM, errno.ENOSPC, errno.EDQUOT})
  25. # `hermes doctor` warns below this, before the store actually starts failing.
  26. LOW_FREE_BYTES = 100 << 20
  27. FIX_HINT = "free disk space, remount it read-write, or fix permissions on {store}"
  28. def describe_error(exc: OSError) -> str:
  29. code = errno.errorcode.get(exc.errno or 0, "")
  30. text = exc.strerror or str(exc)
  31. return f"{code}: {text}" if code else text
  32. def _report_fields(store: str, error: str, since: Optional[float]) -> dict:
  33. """The fields `cron status`, doctor and the home-channel notices print for an unwritable store."""
  34. since_text = "unknown" if since is None else datetime.fromtimestamp(since).astimezone().isoformat(timespec="seconds")
  35. return {"store": store, "error": error, "since": since_text, "fix": FIX_HINT.format(store=store)}
  36. @dataclass
  37. class StoreDegraded:
  38. store: str
  39. since: float # epoch seconds of the first failed write
  40. error: str
  41. recovered_at: Optional[float] = None # epoch seconds of the first write that landed again
  42. sites: set = field(default_factory=set)
  43. skipped: set = field(default_factory=set) # distinct (job id, scheduled instant) not run
  44. # Monotonic time of the last failed dispatch write or re-probe. None until a dispatch write
  45. # fails, so a due one-shot reaches its claim and settles its ``failed`` row once per probe.
  46. last_probe: Optional[float] = None
  47. @property
  48. def skipped_runs(self) -> int:
  49. return len(self.skipped)
  50. def notice_fields(self) -> dict:
  51. return dict(_report_fields(self.store, self.error, self.since), skipped=self.skipped_runs)
  52. _DISPATCH_SITES = frozenset({"advance", "claim"})
  53. _degraded: dict[str, StoreDegraded] = {}
  54. # Each store's just-ended outage, kept until a due scan's save lands (``outage_covers``): the save that
  55. # ends an outage is often another job's heartbeat, landing before the scan that fires the
  56. # one-shots the outage skipped.
  57. _recovered: dict[str, StoreDegraded] = {}
  58. _lock = threading.Lock()
  59. # The gateway consumes transitions; ``fn(event, record)`` with event "unwritable" or "recovered".
  60. _listener: Optional[Callable[[str, StoreDegraded], None]] = None
  61. def set_transition_listener(fn: Optional[Callable[[str, StoreDegraded], None]]) -> None:
  62. global _listener
  63. _listener = fn
  64. def _notify(event: str, record: StoreDegraded) -> None:
  65. listener = _listener
  66. if listener is None:
  67. return
  68. try: # a notice must never break the cron tick that reported the transition
  69. listener(event, record)
  70. except Exception:
  71. logger.debug("Cron store %s transition listener failed", event, exc_info=True)
  72. def _active_cron_dir() -> Path:
  73. from cron.jobs import _current_cron_store
  74. return _current_cron_store().cron_dir
  75. def _key(cron_dir: Optional[Path]) -> str:
  76. """Record key for a store: its resolved path (default: the active store). Callers pass both
  77. resolved (cron store) and unresolved (tick-lock dir, profile homes) paths for one store."""
  78. return os.path.realpath(cron_dir if cron_dir is not None else _active_cron_dir())
  79. def _run_keys(jobs: Iterable[dict]) -> set:
  80. return {(job.get("id"), job.get("next_run_at")) for job in jobs}
  81. def note_unwritable(exc: OSError, consequence: str, site: str, skipped_jobs: Iterable[dict] = (),
  82. cron_dir: Optional[Path] = None) -> None:
  83. """Record a failed write to ``cron_dir`` (default: the active store); WARN once, when the store
  84. enters the degraded state. Callers skip the dispatch that needed the write: no job runs
  85. without a durable advance/fire claim."""
  86. store = _key(cron_dir)
  87. with _lock:
  88. record = _degraded.get(store)
  89. entered = record is None
  90. if entered: # an ended outage not yet seen by a saved scan carries over: keep its start
  91. ended = _recovered.pop(store, None)
  92. since = ended.since if ended is not None else time.time()
  93. record = _degraded[store] = StoreDegraded(store, since, describe_error(exc))
  94. new_site = site not in record.sites
  95. record.sites.add(site)
  96. # A failed dispatch write throttles the next attempt; once dispatch has been tried, a failed
  97. # scan save is this tick's write attempt too, so the re-probe does not add a second one.
  98. if site in _DISPATCH_SITES or record.last_probe is not None:
  99. record.last_probe = time.monotonic()
  100. record.error = describe_error(exc)
  101. record.skipped |= _run_keys(skipped_jobs)
  102. if entered:
  103. logger.warning(
  104. "Cron store %s is unwritable (%s); %s. Scheduled jobs are skipped until it accepts writes "
  105. "again, then each due job fires once. Fix: %s.", store, exc, consequence,
  106. FIX_HINT.format(store=store))
  107. _notify("unwritable", record)
  108. elif new_site:
  109. logger.info("Cron store %s still unwritable (%s); %s", store, exc, consequence)
  110. def note_writable(cron_dir: Path) -> None:
  111. """A jobs.json save landed: clear the degraded state for that store."""
  112. if not _degraded:
  113. return
  114. with _lock:
  115. record = _degraded.pop(_key(cron_dir), None)
  116. if record is not None:
  117. record.recovered_at = time.time()
  118. _recovered[record.store] = record
  119. if record is None:
  120. return
  121. logger.warning("Cron store %s is writable again; %d skipped run(s), catching up once per job",
  122. record.store, record.skipped_runs)
  123. _notify("recovered", record)
  124. def degraded_record(cron_dir: Optional[Path] = None) -> Optional[StoreDegraded]:
  125. return _degraded.get(_key(cron_dir))
  126. def outage_covers(cron_dir: Path, due_at: float, grace: float) -> bool:
  127. """Whether a run due at ``due_at`` fell inside this store's current or just-ended outage."""
  128. return any(r is not None and r.since <= due_at + grace and (r.recovered_at is None or due_at <= r.recovered_at)
  129. for r in (_degraded.get(_key(cron_dir)), _recovered.get(_key(cron_dir))))
  130. def store_key(home) -> str:
  131. """Record key of a profile home's cron store (a symlinked cron/ resolves elsewhere)."""
  132. return _key(Path(home) / "cron")
  133. def forget_stores(stores) -> None:
  134. """Drop the state of ``stores`` (``store_key`` values, resolved while each home existed) whose
  135. profile this process no longer ticks, so a profile that left this gateway cannot keep the
  136. host-wide gauges at writable=0."""
  137. stores = set(stores)
  138. with _lock:
  139. for records in (_degraded, _recovered):
  140. for store in [s for s in records if s in stores]:
  141. del records[store]
  142. def degraded_records() -> list:
  143. with _lock:
  144. return list(_degraded.values())
  145. def dispatch_blocked(due_jobs: list) -> bool:
  146. """Whether this tick skips advance/claim for ``due_jobs``: the active store is known
  147. unwritable and the minute-throttled re-probe has not seen it accept a write. Skipped runs are
  148. recorded; once the probe passes, the dispatch's own save confirms recovery (or re-degrades)."""
  149. cron_dir = _active_cron_dir()
  150. record = _degraded.get(_key(cron_dir))
  151. # Entered in load/scan (last_probe None): dispatch has not been tried yet, so let it try.
  152. if record is None or record.last_probe is None:
  153. return False
  154. now = time.monotonic()
  155. throttled = now - record.last_probe < PROBE_INTERVAL_SECONDS
  156. error = None if throttled else probe_store(cron_dir) # file I/O stays outside the lock
  157. with _lock:
  158. if not throttled:
  159. record.last_probe = now
  160. if error is not None:
  161. record.error = describe_error(error)
  162. blocked = throttled or error is not None
  163. if blocked:
  164. record.skipped |= _run_keys(due_jobs)
  165. return blocked
  166. def end_recovery_window(cron_dir: Path) -> None:
  167. """A due scan's save landed after the outage ended: normal grace applies again."""
  168. if _recovered:
  169. with _lock:
  170. _recovered.pop(_key(cron_dir), None)
  171. def recheck_idle() -> None:
  172. """Idle tick (nothing due, so no save will land): re-probe a degraded store at most once a
  173. minute. The probe only creates an empty file, so a passing probe is confirmed by a real
  174. jobs.json save, whose ``note_writable`` ends the outage; metrics, notices and the one-shot
  175. grace gate then stop treating a recovered store as degraded."""
  176. if not _degraded:
  177. return
  178. cron_dir = _active_cron_dir()
  179. record = _degraded.get(_key(cron_dir))
  180. now = time.monotonic()
  181. if record is None or (record.last_probe is not None and now - record.last_probe < PROBE_INTERVAL_SECONDS):
  182. return
  183. record.last_probe = now
  184. error = probe_store(cron_dir)
  185. if error is None:
  186. from cron.jobs import _jobs_lock, load_jobs, save_jobs
  187. try:
  188. with _jobs_lock():
  189. save_jobs(load_jobs())
  190. return
  191. except OSError as exc: # e.g. EDQUOT or a read-only jobs.json in a writable dir
  192. error = exc
  193. with _lock:
  194. record.error = describe_error(error)
  195. def probe_report(cron_dir: Path) -> Optional[dict]:
  196. """Cross-process view for `cron status` / `doctor`: ``None`` when the store accepts writes,
  197. else the notice fields. ``since`` is jobs.json's mtime (the last write that landed);
  198. ``skipped`` is left to the caller, which counts the due jobs that have not fired."""
  199. error = probe_store(cron_dir)
  200. if error is None:
  201. return None
  202. try:
  203. since = (cron_dir / "jobs.json").stat().st_mtime
  204. except OSError:
  205. since = None
  206. return _report_fields(str(cron_dir), describe_error(error), since)
  207. def free_bytes(path: Path) -> Optional[int]:
  208. """Bytes this process can still write. Root may also fill the reserved blocks that
  209. ``disk_usage().free`` leaves out, so a 'full' disk is not full for root."""
  210. try:
  211. if hasattr(os, "geteuid") and os.geteuid() == 0:
  212. stats = os.statvfs(path)
  213. return stats.f_bfree * stats.f_frsize
  214. return shutil.disk_usage(path).free
  215. except OSError:
  216. return None
  217. def probe_store(cron_dir: Path) -> Optional[OSError]:
  218. """Write probe mirroring a jobs.json save: ``None`` when it would land (or the store does not
  219. exist yet), else the OSError it would hit. Stages a payload the size of the current jobs.json
  220. (plus a page) where the save stages it (beside a symlinked file's real target), so a full,
  221. over-quota or read-only target fails here the way the save would."""
  222. if not cron_dir.is_dir():
  223. return None
  224. from utils import mkstemp_beside
  225. jobs_file = cron_dir / "jobs.json"
  226. try:
  227. size = jobs_file.stat().st_size + 4096
  228. except OSError:
  229. size = 4096
  230. # A read-only jobs.json symlink target, or a tick lock the ticker cannot open (root-owned in a
  231. # writable dir: the tick skips every run while the dir itself still accepts writes).
  232. for target in (os.path.realpath(jobs_file), str(cron_dir / ".tick.lock")):
  233. if os.path.exists(target) and not os.access(target, os.W_OK):
  234. return OSError(errno.EACCES, os.strerror(errno.EACCES), target)
  235. tmp = None
  236. chunk = b"\0" * 65536
  237. try:
  238. fd, tmp = mkstemp_beside(jobs_file, prefix=".probe_")
  239. with os.fdopen(fd, "wb") as f:
  240. for offset in range(0, size, len(chunk)): # bounded memory for a large jobs.json
  241. f.write(chunk[:size - offset])
  242. f.flush()
  243. os.fsync(f.fileno())
  244. except OSError as exc:
  245. return exc
  246. finally:
  247. if tmp is not None:
  248. with contextlib.suppress(OSError):
  249. os.unlink(tmp)
  250. return None