runtime_ownership.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357
  1. """Nonblocking, all-or-nothing reservations of canonical profile homes.
  2. A held lock inode is never removed: unlinking it creates a second owner. The one exception is a
  3. stale inode this user cannot write (left by a gateway that ran as root, #42685) that a flock through
  4. a read-only descriptor proves unheld; it is replaced once, and every winner checks that the path
  5. still names the inode it locked.
  6. """
  7. from __future__ import annotations
  8. from contextlib import contextmanager
  9. import json
  10. import logging
  11. import os
  12. from pathlib import Path
  13. import stat
  14. import threading
  15. import time
  16. from hermes_dir_acl import make_user_dir
  17. logger = logging.getLogger(__name__)
  18. class OwnershipConflict(RuntimeError):
  19. pass
  20. class ReservationCancelled(OwnershipConflict):
  21. """The waiting reserve's caller gave up: nothing reserved, nothing left held."""
  22. def canonical_home(home: Path) -> Path:
  23. return Path(os.path.normcase(str(Path(home).expanduser().resolve())))
  24. def tighten_lock_mode(fd: int) -> None:
  25. """Validate an open ``gateway.lock`` descriptor and narrow an upgraded lock to 0600.
  26. Main's ``acquire_gateway_runtime_lock`` created the inode with ``open(path, "a+")`` under the
  27. umask (0644 on most hosts), and lock inodes are never replaced, so an upgraded home keeps that
  28. mode forever and discovery refuses it. Only a regular, single-link inode this user owns is
  29. chmodded, through the already-open NOFOLLOW descriptor (no path race); anything else is refused.
  30. """
  31. info = os.fstat(fd)
  32. if os.name == 'nt':
  33. if not stat.S_ISREG(info.st_mode):
  34. raise PermissionError('unsafe gateway lock owner or type')
  35. return
  36. if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid(): # windows-footgun: ok — POSIX branch
  37. raise PermissionError('unsafe gateway lock owner or type')
  38. if stat.S_IMODE(info.st_mode) & 0o077:
  39. if info.st_nlink != 1:
  40. raise PermissionError('unsafe gateway lock link count')
  41. os.fchmod(fd, 0o600) # windows-footgun: ok — POSIX branch
  42. _LOCK_FLAGS = os.O_RDWR | os.O_CREAT | getattr(os, 'O_NOFOLLOW', 0)
  43. def _open_checked(path: Path, flags: int) -> int:
  44. fd = os.open(path, flags, 0o600)
  45. try:
  46. tighten_lock_mode(fd)
  47. except BaseException:
  48. os.close(fd)
  49. raise
  50. return fd
  51. def _unlink_stale_lock(path: Path) -> bool:
  52. """Unlink a ``gateway.lock`` this user cannot write (another uid's) once a flock taken through a
  53. read-only descriptor proves it unheld. Symlinks, hardlinks, non-regular files, a home whose
  54. ``gateway.pid`` names a live process and anything not provably unheld are left in place (False).
  55. An inode this user cannot even open is refused with the recovery command: an unheld snapshot
  56. (``/proc/locks``) does not stop a holder locking it between the read and the unlink."""
  57. from gateway.status import _live_pid_from_record, _read_pid_record
  58. if os.name == 'nt':
  59. return False # a sharing violation is a live holder there
  60. info = os.lstat(path)
  61. if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
  62. return False
  63. if _live_pid_from_record(_read_pid_record(path.with_name('gateway.pid'))) is not None:
  64. return False
  65. try:
  66. probe = os.open(path, os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | os.O_NONBLOCK)
  67. except PermissionError:
  68. raise PermissionError(foreign_lock_recovery(path)) from None
  69. import fcntl
  70. try:
  71. if not _same_inode(os.fstat(probe), info):
  72. return False
  73. try:
  74. fcntl.flock(probe, fcntl.LOCK_EX | fcntl.LOCK_NB)
  75. except OSError:
  76. return False
  77. # Unlinked while still locked: a late locker of the old inode never shares our lock.
  78. return _unlink_if_same(path, info)
  79. finally:
  80. os.close(probe)
  81. def foreign_lock_recovery(path: Path) -> str:
  82. return (f'foreign_stale_lock: {path} is not readable by this user (left by a gateway run as root?) '
  83. f'and cannot be proven unheld. If `sudo lsof {path}` lists no process, run `sudo rm {path}`.')
  84. def unreadable_lock_recovery(path: Path) -> str | None:
  85. """The recovery a client prints for a ``gateway.lock`` it cannot even open read-only."""
  86. try:
  87. os.close(os.open(path, os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | os.O_NONBLOCK))
  88. except PermissionError:
  89. return foreign_lock_recovery(path)
  90. except OSError:
  91. pass
  92. return None
  93. def _unlink_if_same(path: Path, info) -> bool:
  94. if not _same_inode(os.lstat(path), info):
  95. return False
  96. os.unlink(path)
  97. return True
  98. def _same_inode(left, right) -> bool:
  99. return (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino)
  100. def _open_lock(path: Path) -> int:
  101. """Descriptor for *path*, replacing a provably stale inode this user cannot use once."""
  102. try:
  103. return _open_checked(path, _LOCK_FLAGS)
  104. except PermissionError:
  105. if not _unlink_stale_lock(path):
  106. raise
  107. # A racer may create the fresh inode first; both then contend for its flock as usual.
  108. return _open_checked(path, _LOCK_FLAGS)
  109. def _still_names(path: Path, fd: int) -> bool:
  110. """True when *path* is still the inode *fd* locked (a stale-lock replacement unlinks it)."""
  111. if os.name == 'nt':
  112. return True
  113. try:
  114. return _same_inode(os.lstat(path), os.fstat(fd))
  115. except FileNotFoundError:
  116. return False
  117. class ProfileOwnership:
  118. def __init__(self):
  119. self._handles: dict[Path, object] = {}
  120. self._writers: list[threading.Thread] = []
  121. self._mutex = threading.RLock()
  122. @property
  123. def homes(self) -> tuple[Path, ...]:
  124. with self._mutex:
  125. return tuple(self._handles)
  126. def reserve(self, homes, *, purpose: str | None = None) -> None:
  127. """``purpose`` marks a maintenance reservation in the lock record, so a contender can tell
  128. a short maintenance holder from a gateway runtime owner (:func:`live_runtime_owner`)."""
  129. from gateway.status import _try_acquire_file_lock, _build_pid_record
  130. with self._mutex:
  131. added = []
  132. try:
  133. for home in sorted({canonical_home(p) for p in homes}):
  134. if home in self._handles:
  135. continue
  136. make_user_dir(home, mode=0o700, parents=True, exist_ok=True)
  137. path = home / 'gateway.lock'
  138. fd = _open_lock(path)
  139. handle = os.fdopen(fd, 'r+', encoding='utf-8') # windows-footgun: ok — write-only
  140. try:
  141. if not _try_acquire_file_lock(handle) or not _still_names(path, handle.fileno()):
  142. raise OwnershipConflict(f'Gateway runtime already owns profile {home}')
  143. record = {**_build_pid_record(), 'hermes_home': str(home)}
  144. if purpose:
  145. record['purpose'] = purpose
  146. handle.seek(0)
  147. handle.truncate()
  148. json.dump(record, handle)
  149. handle.flush()
  150. os.fsync(handle.fileno())
  151. except BaseException:
  152. handle.close()
  153. raise
  154. self._handles[home] = handle
  155. added.append(home)
  156. except BaseException:
  157. for home in reversed(added):
  158. self.release(home)
  159. raise
  160. def owns(self, home: Path) -> bool:
  161. with self._mutex:
  162. return canonical_home(home) in self._handles
  163. def release(self, home: Path) -> None:
  164. from gateway.status import _release_file_lock
  165. with self._mutex:
  166. handle = self._handles.pop(canonical_home(home), None)
  167. if handle is not None:
  168. _release_file_lock(handle)
  169. handle.close()
  170. def start_writer(self, thread: threading.Thread) -> None:
  171. """Register before starting so partial startup cannot forget a live writer."""
  172. with self._mutex:
  173. self._writers.append(thread)
  174. thread.start()
  175. def close(self) -> None:
  176. with self._mutex:
  177. # A timed-out daemon can still write during finally/atexit. Keep the
  178. # handles alive; process exit releases them atomically in the OS.
  179. if any(thread.is_alive() for thread in self._writers):
  180. return
  181. self._writers.clear()
  182. for home in reversed(self.homes):
  183. self.release(home)
  184. process_ownership = ProfileOwnership()
  185. _maintenance = threading.local()
  186. def maintenance_holder(home: Path, purpose: str) -> dict | None:
  187. """The live holder record of *home*'s ``gateway.lock`` when it is a ``purpose`` maintenance
  188. reservation (see :meth:`ProfileOwnership.reserve`), else None. Advisory only: it names the
  189. holder for a refusal or a short wait, it never grants the lock."""
  190. from gateway.status import _live_pid_from_record, _read_gateway_lock_record
  191. record = _read_gateway_lock_record(canonical_home(home) / 'gateway.lock')
  192. if not record or record.get('purpose') != purpose or _live_pid_from_record(record) is None:
  193. return None
  194. return record
  195. def lock_record_unpublished(home: Path) -> bool:
  196. """True when *home*'s ``gateway.lock`` record names no live holder yet: empty, torn, or a dead
  197. previous holder's. :meth:`ProfileOwnership.reserve` takes the flock BEFORE it writes the record,
  198. so a contender that loses the flock in that window cannot tell maintenance from a gateway; it
  199. must retry (bounded) instead of reporting a runtime owner. A record naming the CALLER is the
  200. caller's own rolled-back attempt (an all-or-nothing reserve writes each home's record before a
  201. later home refuses), so whoever holds the flock now has not published either. Advisory only,
  202. never grants the lock."""
  203. from gateway.status import _live_pid_from_record, _read_gateway_lock_record
  204. record = _read_gateway_lock_record(canonical_home(home) / 'gateway.lock')
  205. return not record or _live_pid_from_record(record) in (None, os.getpid())
  206. # A client's discovery probe (gateway.status._probe_lock_file) holds a gateway.lock flock for an
  207. # instant; the next attempt after this pause normally finds it gone.
  208. PROBE_RETRY_BACKOFF_S = 0.015
  209. TRANSIENT_HOLDER_POLL_S = 0.1
  210. def _conflicting_homes(homes):
  211. """The homes whose gateway.lock another holder has right now: each probed alone with its own open
  212. file description and released at once (the all-or-nothing reserve does not say which failed).
  213. A home freed meanwhile is simply not listed; the next reserve attempt takes it."""
  214. conflicting = []
  215. for home in homes:
  216. probe = ProfileOwnership()
  217. try:
  218. probe.reserve([home])
  219. except OwnershipConflict:
  220. conflicting.append(home)
  221. finally:
  222. probe.close()
  223. return conflicting
  224. def reserve_waiting_out_transient_holders(ownership, homes, *, wait_s: float,
  225. cancel: threading.Event | None = None) -> None:
  226. """``ownership.reserve(homes)``, retried until *wait_s* passes while every contender is
  227. transient: a client's lock probe (gone by the re-check), a browse reconcile, or a holder that has
  228. not published its record yet (flock taken, record unwritten). A positively identified live owner
  229. on any home refuses at once. Every grant is a real ``reserve``: the flock is never bypassed.
  230. Shared by the gateway boot and the running multiplexer's hot-serve of a new profile.
  231. *cancel* set (a worker thread's caller gave up) ends the wait at once and releases a grant
  232. that landed meanwhile, raising :class:`ReservationCancelled`: the caller may already have run
  233. its ownership cleanup, so nobody would ever release a home taken after it."""
  234. from hermes_state_reconcile import BROWSE_RECONCILE_PURPOSE
  235. cancel = threading.Event() if cancel is None else cancel
  236. deadline = time.monotonic() + wait_s
  237. logged = False
  238. while not cancel.is_set():
  239. before = set(ownership.homes)
  240. try:
  241. ownership.reserve(homes)
  242. except OwnershipConflict:
  243. # Only the homes that actually conflict decide: a real owner on any of them refuses at
  244. # once, even while another home is mid-browse (multiplexer boot, all-or-nothing).
  245. conflicting = _conflicting_homes(homes)
  246. holders = [h for h in conflicting
  247. if maintenance_holder(h, BROWSE_RECONCILE_PURPOSE) or lock_record_unpublished(h)]
  248. if len(holders) < len(conflicting) or time.monotonic() >= deadline:
  249. raise
  250. if not conflicting:
  251. cancel.wait(PROBE_RETRY_BACKOFF_S) # the holder already let go: a lock probe, never an owner
  252. continue
  253. if not logged:
  254. logger.info("Profile %s is held by a short session-store schema catch-up; waiting up to %.0fs",
  255. holders[0], wait_s)
  256. logged = True
  257. cancel.wait(TRANSIENT_HOLDER_POLL_S)
  258. continue
  259. if not cancel.is_set():
  260. return
  261. for home in {canonical_home(p) for p in homes} - before:
  262. ownership.release(home)
  263. raise ReservationCancelled(f'Reservation of {", ".join(map(str, homes))} cancelled')
  264. def live_runtime_owner(home: Path) -> int | None:
  265. """PID of a live runtime owner of *home* (a gateway, or a reservation that is not marked as
  266. maintenance), else None. A purpose-marked maintenance holder is never an owner."""
  267. from gateway.status import _live_pid_from_record, _read_gateway_lock_record, get_running_pid
  268. home = canonical_home(home)
  269. record = _read_gateway_lock_record(home / 'gateway.lock')
  270. live = _live_pid_from_record(record) if record else None
  271. if live is not None and record.get('purpose'):
  272. return None # the flock holder is maintenance, so no gateway can hold it too
  273. return get_running_pid(home / 'gateway.pid', cleanup_stale=False) or live
  274. @contextmanager
  275. def exclusive_maintenance(homes, *, purpose: str | None = None):
  276. """Reserve the authority's exact lock before touching managed state.
  277. This is not an owner-presence check: the reservation remains held through
  278. publication, excluding startup even before its PID/control socket exists.
  279. Only nested synchronous maintenance in this thread may reuse a reservation;
  280. being the runtime owner (even in this process) never grants maintenance.
  281. Lock inodes must not be restored, removed, or replaced by callers.
  282. """
  283. state = getattr(_maintenance, 'state', None)
  284. outermost = state is None or state[0] != os.getpid()
  285. owner = state[1] if state is not None and not outermost else ProfileOwnership()
  286. previous = set(owner.homes)
  287. try:
  288. owner.reserve(homes, purpose=purpose)
  289. if outermost:
  290. _maintenance.state = (os.getpid(), owner)
  291. yield
  292. except OwnershipConflict as exc:
  293. raise OwnershipConflict(
  294. f'Exclusive maintenance refused: {exc}. Drain and stop the gateway, then retry.'
  295. ) from exc
  296. finally:
  297. for home in reversed(owner.homes):
  298. if home not in previous:
  299. owner.release(home)
  300. if outermost:
  301. _maintenance.state = None