hermes_state_lockowners.py 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124
  1. """Who holds the SQLite write lock on state.db right now (Linux ``/proc/locks``).
  2. The "database is locked for over Ns" failure names the victim but not the holder, and the
  3. open-descriptor scan (``hermes_state_holders``) cannot tell a reader from the one writer: every
  4. Hermes process has the DB open. SQLite's unix VFS takes ``fcntl`` byte-range locks whose offsets
  5. encode the lock kind, and the kernel exports them with the owning pid, so the holder can be named
  6. at the moment the deadline passes.
  7. """
  8. from __future__ import annotations
  9. import logging
  10. import os
  11. import sys
  12. from pathlib import Path
  13. from typing import Dict, List, Tuple
  14. logger = logging.getLogger(__name__)
  15. # sqlite/src/os_unix.c: UNIX_SHM_BASE = (22 + SQLITE_SHM_NLOCK) * 4 = 120; WAL lock i sits at BASE + i.
  16. _SHM_LOCK_NAMES = {
  17. 120: "WAL write",
  18. 121: "WAL checkpoint",
  19. 122: "WAL recover",
  20. 128: "shm dead-man switch",
  21. }
  22. # Rollback-journal / exclusive locks on the main file live in the pending-byte page (1 GiB).
  23. _PENDING_BYTE = 0x40000000
  24. _MAIN_LOCK_NAMES = {
  25. _PENDING_BYTE: "PENDING",
  26. _PENDING_BYTE + 1: "RESERVED",
  27. }
  28. def _describe_range(sidecar: str, start: int, end: int) -> str:
  29. if sidecar == "-shm":
  30. if start in _SHM_LOCK_NAMES and end == start:
  31. return _SHM_LOCK_NAMES[start]
  32. if 123 <= start <= 127 and end == start:
  33. return f"WAL read slot {start - 123}"
  34. return f"shm bytes {start}-{end}"
  35. if start == end and start in _MAIN_LOCK_NAMES:
  36. return _MAIN_LOCK_NAMES[start]
  37. if start == _PENDING_BYTE + 2:
  38. return "SHARED range"
  39. return f"db bytes {start}-{end}"
  40. def parse_proc_locks(text: str, inodes: dict[tuple[int, int], str]) -> list[tuple[int, str, str]]:
  41. """``(pid, lock kind, sidecar)`` for every WRITE lock on one of ``inodes``.
  42. ``inodes`` maps ``(st_dev, st_ino)`` to ``""`` (main file), ``"-wal"`` or ``"-shm"``. Read locks
  43. are dropped: they never block a writer in WAL mode. An OFD lock is reported with pid ``-1``
  44. (the kernel does not export its owner).
  45. """
  46. found: list[tuple[int, str, str]] = []
  47. for line in text.splitlines():
  48. fields = line.split()
  49. # "N: POSIX ADVISORY WRITE <pid> MAJ:MIN:INO <start> <end|EOF>"; a blocked waiter is "N: -> POSIX ...".
  50. if len(fields) < 8 or fields[1] == "->":
  51. continue
  52. _, _kind, _cls, access, pid_s, ident, start_s, end_s = fields[:8]
  53. if access != "WRITE":
  54. continue
  55. try:
  56. major_s, minor_s, ino_s = ident.split(":")
  57. dev = os.makedev(int(major_s, 16), int(minor_s, 16))
  58. key = (dev, int(ino_s))
  59. pid = int(pid_s)
  60. start = int(start_s)
  61. end = start if end_s == "EOF" else int(end_s)
  62. except ValueError:
  63. continue
  64. sidecar = inodes.get(key)
  65. if sidecar is None:
  66. continue
  67. found.append((pid, _describe_range(sidecar, start, end), sidecar))
  68. return found
  69. def state_db_write_lock_holders(db_path) -> list[str]:
  70. """Operator-facing lines naming the processes that hold a write-class lock on ``db_path``.
  71. Empty when nothing is held or the platform has no ``/proc/locks``.
  72. """
  73. if not sys.platform.startswith("linux"):
  74. return []
  75. base = os.path.realpath(os.fspath(db_path))
  76. inodes: dict[tuple[int, int], str] = {}
  77. for sidecar in ("", "-wal", "-shm"):
  78. try:
  79. st = os.stat(base + sidecar)
  80. except OSError:
  81. continue
  82. inodes[(st.st_dev, st.st_ino)] = sidecar
  83. # /proc/locks is host-wide and served over several read()s: lock churn in other processes
  84. # shifts it mid-read and can skip the holder. A skip rarely repeats, so union three passes.
  85. held: list[tuple[int, str, str]] = []
  86. for _ in range(3):
  87. try:
  88. with open("/proc/locks", encoding="ascii", errors="replace") as handle:
  89. text = handle.read()
  90. except OSError:
  91. return []
  92. held.extend(entry for entry in parse_proc_locks(text, inodes) if entry not in held)
  93. from hermes_state_holders import describe_holder_pid
  94. lines = []
  95. for pid, kind, sidecar in held:
  96. who = describe_holder_pid(pid) if pid > 0 else "OFD lock, owner pid not exported by the kernel"
  97. lines.append(f"{who} holds {kind} lock on {Path(base + sidecar).name}")
  98. return lines
  99. def log_write_lock_holders(db_path, patience_s: float) -> None:
  100. """One WARNING naming the write-lock holders at the moment a writer gave up waiting."""
  101. holders = state_db_write_lock_holders(db_path)
  102. if holders:
  103. detail = " | ".join(holders)
  104. else:
  105. detail = "no write-class lock held at the deadline (the holder released just before it)"
  106. logger.warning(
  107. "state.db write lock unavailable for %.0fs (%s): %s", patience_s, Path(db_path).name, detail
  108. )