browser_launcher_holders.py 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211
  1. """Cross-process holders of a shared-profile Chromium (``tools/browser_launcher.py``).
  2. A persistent profile (``browser.user_data_dir``, the Bot Desktop profile, the real-profile copy) has ONE
  3. Chromium, launched by whichever Hermes process got there first; every other process attaches to it. Each
  4. attacher leaves a lease beside the launch record in the profile's private state dir (per OS user, so every
  5. Hermes home pinning the dir sees the same leases: ``browser_launcher._shared_profiles_root``). The process holding
  6. the record (the launcher, or whoever it was handed to) stops the browser on exit only when no other live
  7. Hermes process holds a lease; otherwise it hands the record to one of them, so the LAST holder stops it. A
  8. holder that crashed is dropped once its pid and start time no longer match, never on the pid alone.
  9. A pid means something only inside the PID namespace and boot that wrote it (a ``HERMES_HOME`` shared by a
  10. container gateway and a host CLI): every record, lease and throwaway-dir name carries the writer's
  11. :func:`ns_identity`, and one from another namespace or boot, or one lacking it, is never judged dead. A lease is
  12. keyed by the record of the browser it holds, whichever namespace wrote that record; only its holder part carries
  13. the lease writer's tag, so a launcher finds every holder whatever namespace it attached from.
  14. Every decision runs under the records dir's lock: a lease written while another process decides cannot be
  15. missed. Siblings late-import the facade (its ``_pid_alive`` / ``_start_time`` are the patched seams).
  16. """
  17. from __future__ import annotations
  18. import contextlib
  19. import errno
  20. import functools
  21. import hashlib
  22. import json
  23. import logging
  24. import os
  25. import time
  26. from pathlib import Path
  27. from typing import Iterator, Optional
  28. logger = logging.getLogger(__name__)
  29. _LOCK_NAME = ".holders.lock"
  30. @functools.cache
  31. def ns_identity() -> Optional[str]:
  32. """This process's PID namespace and boot (Linux: the pid-namespace inode + the kernel boot id); None where
  33. neither is readable (no ``/proc``), which then matches only another None."""
  34. try:
  35. ino = os.stat("/proc/self/ns/pid").st_ino
  36. boot = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip()
  37. except OSError:
  38. return None
  39. return f"{ino}:{boot}"
  40. def ns_tag() -> str:
  41. """The name suffix of this namespace's records, leases and throwaway dirs (``""`` without an identity)."""
  42. identity = ns_identity()
  43. return f"-n{hashlib.sha256(identity.encode()).hexdigest()[:12]}" if identity else ""
  44. def same_namespace(data: dict) -> bool:
  45. """A record or lease written in this PID namespace and boot: only then do its pids name processes here."""
  46. return data.get("ns") == ns_identity()
  47. def held_here(data: dict, pid_field: str) -> bool:
  48. return data.get(pid_field) == os.getpid() and same_namespace(data)
  49. # msvcrt.locking reports a lock another process holds as EDEADLOCK (LK_LOCK, after its own ~10 s of retries)
  50. # or EACCES (LK_NBLCK); anything else is a real error.
  51. _NT_LOCK_CONTENDED = frozenset({errno.EACCES, errno.EDEADLK, getattr(errno, "EDEADLOCK", errno.EDEADLK)})
  52. _NT_LOCK_TIMEOUT_S = 30.0
  53. def nt_lock_failure(exc: OSError, records: Path, expired: bool) -> Optional[OSError]:
  54. """What a failed ``msvcrt.locking`` attempt means: None to retry (the lock is held and time is left), else
  55. the error to raise. Bounded, so a lock leaked by a hung process fails the caller instead of hanging it."""
  56. if exc.errno not in _NT_LOCK_CONTENDED:
  57. return exc
  58. if expired:
  59. return TimeoutError(f"browser launcher state {records} stayed locked for {_NT_LOCK_TIMEOUT_S:.0f} s")
  60. return None
  61. def _lock_nt(fd: int, records: Path, blocking: bool) -> bool:
  62. import msvcrt
  63. deadline = time.monotonic() + _NT_LOCK_TIMEOUT_S
  64. while True:
  65. try:
  66. msvcrt.locking(fd, msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK, 1)
  67. return True
  68. except OSError as exc:
  69. failure = nt_lock_failure(exc, records, not blocking or time.monotonic() >= deadline)
  70. if failure is exc:
  71. raise
  72. if not blocking:
  73. return False
  74. if failure is not None:
  75. raise failure from exc
  76. def _lock_posix(fd: int, blocking: bool) -> bool:
  77. import fcntl # windows-footgun: ok — POSIX only (records_lock's nt branch takes _lock_nt)
  78. try:
  79. fcntl.flock(fd, fcntl.LOCK_EX if blocking else fcntl.LOCK_EX | fcntl.LOCK_NB)
  80. except BlockingIOError:
  81. return False
  82. return True
  83. @contextlib.contextmanager
  84. def records_lock(records: Path, *, blocking: bool = True) -> Iterator[bool]:
  85. """Exclusive, cross-process lock on ``records`` (a private launcher state dir); yields whether it is held
  86. (always True when ``blocking``). Never nested: two locks on one file in one process are two open file
  87. descriptions, and the second would wait forever."""
  88. flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)
  89. fd = os.open(records / _LOCK_NAME, flags, 0o600)
  90. held = False
  91. try:
  92. held = _lock_nt(fd, records, blocking) if os.name == "nt" else _lock_posix(fd, blocking)
  93. yield held
  94. finally:
  95. if held and os.name == "nt":
  96. import msvcrt
  97. with contextlib.suppress(OSError):
  98. msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
  99. os.close(fd)
  100. def _lease_path(record: Path, holder_pid: int) -> Path:
  101. return record.with_name(f"{record.stem}@{holder_pid}{ns_tag()}.lease")
  102. def _load(path: Path) -> Optional[dict]:
  103. try:
  104. data = json.loads(path.read_text(encoding="utf-8-sig"))
  105. except (OSError, ValueError):
  106. data = None
  107. if isinstance(data, dict):
  108. return data
  109. path.unlink(missing_ok=True)
  110. return None
  111. def process_alive(pid, start_time) -> bool:
  112. """``pid`` is still the process that wrote a record or lease: alive, and with the recorded start time
  113. (a recycled pid has another one)."""
  114. from tools import browser_launcher as launcher
  115. if not isinstance(pid, int) or not launcher._pid_alive(pid):
  116. return False
  117. return start_time is None or launcher._start_time(pid) == start_time
  118. def write_lease(record: Path, browser) -> None:
  119. """This process holds ``browser`` (caller holds :func:`records_lock`)."""
  120. from tools import browser_launcher as launcher
  121. from utils import atomic_json_write
  122. me = os.getpid()
  123. atomic_json_write(_lease_path(record, me), {
  124. "holder_pid": me, "holder_start_time": launcher._start_time(me), "pid": browser.pid,
  125. "start_time": browser.start_time, "user_data_dir": browser.user_data_dir, "ns": ns_identity()}, indent=None)
  126. def drop_lease(record: Path) -> None:
  127. _lease_path(record, os.getpid()).unlink(missing_ok=True)
  128. def _holder_gone(lease: dict) -> bool:
  129. """Proven gone: written in this namespace and boot, and its pid + start time no longer match."""
  130. return same_namespace(lease) and not process_alive(lease.get("holder_pid"), lease.get("holder_start_time"))
  131. def other_live_holders(record: Path, browser_start_time) -> list[dict]:
  132. """Leases of OTHER live processes on the browser ``record`` names (caller holds :func:`records_lock`).
  133. A lease of a dead holder, or one naming an earlier browser on the same pid (another start time, from whichever
  134. namespace that knew it), is removed; one from another namespace counts as live (its holder cannot be checked
  135. from here)."""
  136. holders = []
  137. for path in sorted(record.parent.glob(f"{record.stem}@*.lease")):
  138. lease = _load(path)
  139. if lease is None or held_here(lease, "holder_pid"):
  140. continue
  141. other_launch = lease.get("start_time") != browser_start_time and (
  142. same_namespace(lease) or lease.get("start_time") is not None)
  143. if other_launch or _holder_gone(lease):
  144. path.unlink(missing_ok=True)
  145. continue
  146. holders.append(lease)
  147. return holders
  148. def drop_dead_leases(records: Path) -> None:
  149. """Leases whose holder is gone (caller holds :func:`records_lock`): a lease outlives its launch record when
  150. the browser died while its holder crashed."""
  151. for path in records.glob("*.lease"):
  152. lease = _load(path)
  153. if lease is not None and _holder_gone(lease):
  154. path.unlink(missing_ok=True)
  155. def successor(holders: list[dict]) -> Optional[dict]:
  156. """The live holder a record can be handed to: one in this namespace (a pid from another one names nobody
  157. here, so a record handed there could never be stopped nor reaped from here)."""
  158. return next((h for h in holders if same_namespace(h)), None)
  159. def hand_over(record: Path, data: dict, holder: dict) -> None:
  160. """Make ``holder`` the record's owner: every reaper now sees a live owner, and the holder stops the
  161. browser when it is the last one out."""
  162. from utils import atomic_json_write
  163. atomic_json_write(record, {**data, "owner_pid": holder["holder_pid"],
  164. "owner_start_time": holder.get("holder_start_time")}, indent=None)
  165. logger.info("browser: Chromium pid %s is still used by Hermes pid %s; handed it over instead of stopping it",
  166. data.get("pid"), holder["holder_pid"])