_early_recovery_zip.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289
  1. """Settles a ZIP ``hermes update`` swap whose owner died: its journal, owner lock and recovery.
  2. Split out of ``_early_recovery`` (stdlib-only, like it). ``_early_recovery.restore_interrupted_pull``
  3. imports this only when the journal exists, so the git-marker repair and its published closure
  4. (``RECOVERY_CLOSURE``) never need it.
  5. """
  6. from __future__ import annotations
  7. import contextlib
  8. import os
  9. import re
  10. import stat
  11. import sys
  12. import time
  13. from pathlib import Path
  14. from hermes_cli._early_recovery import (
  15. _CLAIM_HELD_ERRNOS, ZIP_SWAP_JOURNAL, _keep_aside, _lock_fd, _project_root, _pytest_owns_live_checkout,
  16. write_durable_text)
  17. # A journal in the install root naming every entry the swap stages/renames and whether it existed
  18. # before, plus an OS lock its owner holds for the whole stage+swap (the kernel drops it with the
  19. # owner: liveness without pids or ages).
  20. _ZIP_STAGING_SUFFIX, _ZIP_OLD_SUFFIX = ".hermes-update-staging", ".hermes-update-old"
  21. class ZipSwapLock:
  22. """``zip_swap_owner_lock``'s verdict: truthy only while this process holds the kernel lock.
  23. ``reason`` names a refusal: ``busy`` (a live owner holds it) or why no lock could be taken at all.
  24. """
  25. __slots__ = ("owned", "reason")
  26. def __init__(self, owned: bool, reason: str = "") -> None:
  27. self.owned, self.reason = owned, reason
  28. def __bool__(self) -> bool:
  29. return self.owned
  30. @contextlib.contextmanager
  31. def zip_swap_owner_lock(root: Path, *, wait: float = 0.0):
  32. """Yields a truthy ``ZipSwapLock`` while this process owns the ZIP swap lock, a falsy one otherwise.
  33. The lock file is a stable sidecar: never unlinked, so every process locks the same inode (an
  34. unlink lets a waiter keep the old inode while a newcomer locks a fresh one). No lock, no admission:
  35. a root where the file cannot be opened or locked refuses with the reason (fail closed)."""
  36. lock_path = Path(root) / (ZIP_SWAP_JOURNAL + ".lock")
  37. try:
  38. fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o644)
  39. except OSError as exc:
  40. yield ZipSwapLock(False, f"cannot open the ZIP swap lock {lock_path}: {exc.strerror or exc}")
  41. return
  42. try:
  43. deadline = time.monotonic() + wait
  44. while True:
  45. try:
  46. if sys.platform == "win32":
  47. import msvcrt
  48. os.lseek(fd, 0, os.SEEK_SET)
  49. msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
  50. else:
  51. import fcntl
  52. fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
  53. break
  54. except OSError as exc:
  55. if exc.errno not in _CLAIM_HELD_ERRNOS:
  56. yield ZipSwapLock(False, f"cannot lock the ZIP swap lock {lock_path}: {exc.strerror or exc}")
  57. return
  58. if time.monotonic() >= deadline:
  59. yield ZipSwapLock(False, "busy")
  60. return
  61. time.sleep(0.05)
  62. try:
  63. yield ZipSwapLock(True)
  64. finally:
  65. _lock_fd(fd, False)
  66. finally:
  67. os.close(fd)
  68. def write_zip_swap_journal(root: Path, phase: str, entries: list, gen: str, temps: dict | None = None) -> None:
  69. """Publish the swap journal: ``entries`` are ``[name, existed, staged_id, live_id]`` (ids from
  70. ``zip_entry_identity``, "" while unknown); ``gen`` is the run's random tag (its backup temps);
  71. ``temps`` maps each backup temp the swap created (its file name) to its ``filling`` identity."""
  72. import json
  73. write_durable_text(Path(root) / ZIP_SWAP_JOURNAL, json.dumps(
  74. {"pid": os.getpid(), "gen": gen, "phase": phase, "entries": entries, "temps": temps or {}}))
  75. def zip_entry_identity(path, *, filling: bool = False) -> str:
  76. """The entry's own identity (lstat, never its target); "" when absent or unidentifiable.
  77. The ZIP journal's provenance: recovery deletes only an entry whose identity it recorded when the
  78. swap made it (a staging copy, or the live entry its backup links/renames), never a lookalike. A
  79. non-directory adds size and mtime: a file deleted and recreated often gets the same inode back.
  80. A directory, or a file still being written (``filling``), is ``dev:ino:type`` only: its own size
  81. and mtime move while it is being filled."""
  82. try:
  83. st = os.lstat(path)
  84. except OSError:
  85. return ""
  86. if not st.st_ino:
  87. return ""
  88. base = f"{st.st_dev}:{st.st_ino}:{stat.S_IFMT(st.st_mode):o}"
  89. return base if filling or stat.S_ISDIR(st.st_mode) else f"{base}:{st.st_size}:{st.st_mtime_ns}"
  90. def _drop_path(path: Path) -> None:
  91. if path.is_dir() and not path.is_symlink():
  92. import shutil
  93. try:
  94. shutil.rmtree(path)
  95. except OSError:
  96. # A staged copy keeps its source's modes: a read-only directory refuses to give up its
  97. # entries. These trees are the update's own copies, so make their directories writable and
  98. # retry. Files keep their modes: a grafted artifact is a hardlink to the LIVE file, and POSIX
  99. # deletion needs only the parent's write bit. Windows refuses to delete a read-only file, so
  100. # there only a file no other link shares has its read-only bit cleared.
  101. rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
  102. os.chmod(path, os.stat(path).st_mode | rwx)
  103. for dirpath, dirnames, files in os.walk(path):
  104. for name in dirnames:
  105. child = os.path.join(dirpath, name)
  106. if not os.path.islink(child):
  107. os.chmod(child, os.stat(child).st_mode | rwx)
  108. for name in files if sys.platform == "win32" else ():
  109. child = os.path.join(dirpath, name)
  110. st = os.lstat(child)
  111. if st.st_nlink == 1 and not stat.S_ISLNK(st.st_mode):
  112. os.chmod(child, st.st_mode | stat.S_IWUSR)
  113. shutil.rmtree(path)
  114. elif path.exists() or path.is_symlink():
  115. path.unlink()
  116. def _parse_zip_swap_journal(raw: str) -> tuple[str, str, list[tuple[str, bool, str, str]], dict[str, str]] | None:
  117. """``(phase, gen, [(entry name, existed, staged_id, live_id)], temps)`` from a journal this code wrote,
  118. else None. A journal without ``temps`` (an older writer's) vouches for no backup temp."""
  119. import json
  120. try:
  121. data = json.loads(raw)
  122. except ValueError:
  123. return None
  124. if not isinstance(data, dict) or data.get("phase") not in ("staging", "swapping", "committed"):
  125. return None
  126. entries, gen = data.get("entries"), data.get("gen")
  127. if not isinstance(entries, list) or not isinstance(gen, str) or not re.fullmatch(r"[0-9a-f]{12}", gen):
  128. return None
  129. if not all(isinstance(e, list) and len(e) == 4 and all(isinstance(v, str) for v in (e[0], e[2], e[3]))
  130. and isinstance(e[1], bool) and e[0] not in ("", ".", "..") and "/" not in e[0] and "\\" not in e[0]
  131. for e in entries):
  132. return None
  133. temps = data.get("temps", {})
  134. if not isinstance(temps, dict) or not all(isinstance(v, str) for v in temps.values()):
  135. return None
  136. return data["phase"], gen, [(name, existed, staged, live) for name, existed, staged, live in entries], temps
  137. def _discard_unless_owned(path: Path, identity: str, kept: list[Path]) -> None:
  138. """Delete ``path`` only when it is provably the entry the swap recorded; anything else (a later
  139. user file at the suffix, a foreign journal's victim) is renamed aside, never deleted (review Z1)."""
  140. if not os.path.lexists(path):
  141. return
  142. if identity and zip_entry_identity(path) == identity:
  143. _drop_path(path)
  144. else:
  145. kept.append(_keep_aside(path))
  146. def _settle_zip_entry(root: Path, phase: str, gen: str, entry: tuple[str, bool, str, str], temps: dict[str, str],
  147. kept: list[Path]) -> bool:
  148. """Finish or roll back one journaled entry; True when the live tree changed.
  149. Presence is the ENTRY's (lexists/lstat), never its target's: a dangling symlink backup is the only
  150. copy of a tracked symlink, not "no backup" (review Z3). Restoring a backup never deletes bytes, so
  151. it needs no provenance; every deletion does (``_discard_unless_owned``)."""
  152. name, existed, staged_id, live_id = entry
  153. here = os.path.lexists
  154. dst = root / name
  155. staging, old = Path(f"{dst}{_ZIP_STAGING_SUFFIX}"), Path(f"{dst}{_ZIP_OLD_SUFFIX}")
  156. changed = False
  157. if phase == "swapping":
  158. # Staging finished before this phase, so a backup now is the one the swap made from the old
  159. # entry: it existed, whatever an older journal recorded.
  160. existed = existed or here(old)
  161. if existed and here(old) and not (live_id and zip_entry_identity(dst) == live_id):
  162. if (old.is_file() and not old.is_symlink() and staged_id and zip_entry_identity(dst) == staged_id
  163. and not dst.is_dir()):
  164. os.replace(old, dst) # a file entry never goes missing, not even here
  165. else:
  166. _discard_unless_owned(dst, staged_id, kept)
  167. os.rename(old, dst) # moves a symlink itself, never its target
  168. changed = True
  169. elif not existed and here(dst) and not here(staging):
  170. _discard_unless_owned(dst, staged_id, kept)
  171. changed = True
  172. elif existed and not here(dst) and here(old):
  173. os.rename(old, dst) # the backup is the only copy left
  174. changed = True
  175. for leftover, identity in ((staging, staged_id), (old, live_id)):
  176. _discard_unless_owned(leftover, identity, kept)
  177. # ``<old>.<gen>.tmp``: a backup copy of the live file killed before its rename (no-hardlink file
  178. # systems). Neither the tag nor the bytes prove a file there is that copy (F78/F78-R): only the
  179. # identity the swap journaled the moment it created the temp does. A file system that hands a
  180. # recreated name its old inode (FAT, where this copy runs) would let that record vouch for a
  181. # later file too, so the bytes must also be what a killed copy is: a prefix of the live file.
  182. tmp = Path(f"{old}.{gen}.tmp")
  183. if _is_killed_backup_copy(tmp, dst, temps.get(tmp.name, "")):
  184. _drop_path(tmp)
  185. elif here(tmp):
  186. kept.append(_keep_aside(tmp))
  187. return changed
  188. def _is_killed_backup_copy(tmp: Path, dst: Path, identity: str) -> bool:
  189. """The identity matches the regular file the swap recorded (type included), so the bytes are readable."""
  190. return (bool(identity) and zip_entry_identity(tmp, filling=True) == identity
  191. and dst.is_file() and not dst.is_symlink() and dst.read_bytes().startswith(tmp.read_bytes()))
  192. def restore_interrupted_zip_swap(project_root: Path | None = None) -> bool:
  193. """Finish or roll back a ZIP swap whose owner died; True when the tree changed (relaunch).
  194. ``committed``: every rename landed, only backups remain -> drop them (finish). ``swapping``: put
  195. each moved-aside entry back and remove entries the swap added (roll back to the old tree, which
  196. the venv was built for; ``hermes update`` redoes it). ``staging``: nothing live moved -> drop the
  197. staging copies. Either way no ``*.hermes-update-staging``/``-old`` sibling is left to wedge the
  198. next run's free-space or dirty-tree checks. A live owner (lock held) is never second-guessed.
  199. Provenance (review Z1): a sibling or entry is deleted only when its identity is the one the journal
  200. recorded; anything else is kept aside under a ``.hermes-update-kept`` name and reported.
  201. """
  202. root = _project_root() if project_root is None else Path(project_root)
  203. journal = root / ZIP_SWAP_JOURNAL
  204. if not journal.is_file() or _pytest_owns_live_checkout(root):
  205. return False
  206. with zip_swap_owner_lock(root) as owned:
  207. if not owned and owned.reason != "busy":
  208. print(f"⚠ An interrupted ZIP update cannot be settled: {owned.reason}.", file=sys.stderr)
  209. if not owned or not journal.is_file():
  210. return False
  211. try:
  212. raw = journal.read_text(encoding="utf-8-sig")
  213. except OSError as exc: # transient (AV, sharing violation): the journal is still the record
  214. print(f"⚠ Could not read the interrupted ZIP update's journal ({exc}); the next launch retries.",
  215. file=sys.stderr)
  216. return False
  217. parsed = _parse_zip_swap_journal(raw)
  218. if parsed is None:
  219. # Never guess which live entry to move, and never retire the only record of a mixed tree.
  220. print(f"⚠ The interrupted ZIP update's journal {journal} is unreadable or from another version; "
  221. "it and every `*.hermes-update-old` backup were kept. Put back what each backup replaced "
  222. "(or reinstall), then delete the journal.", file=sys.stderr)
  223. return False
  224. phase, gen, entries, temps = parsed
  225. changed = False
  226. failed = False
  227. kept: list[Path] = []
  228. for entry in reversed(entries):
  229. try:
  230. changed = _settle_zip_entry(root, phase, gen, entry, temps, kept) or changed
  231. except OSError as exc:
  232. failed = True
  233. print(f"⚠ Could not settle {entry[0]} after an interrupted ZIP update: {exc}", file=sys.stderr)
  234. if kept:
  235. print("⚠ An interrupted ZIP update's recovery found entries it could not prove were its own and "
  236. f"kept them aside instead of deleting them: {', '.join(map(str, kept))}. Delete each once "
  237. "you know it is not yours.", file=sys.stderr)
  238. # Retire the journal only on a verified terminal state, not an exception-free loop: no sibling
  239. # (staging copy, backup, backup temp) left that only this journal could still explain.
  240. siblings = (_ZIP_STAGING_SUFFIX, _ZIP_OLD_SUFFIX, f"{_ZIP_OLD_SUFFIX}.{gen}.tmp")
  241. unsettled = [e[0] for e in entries if any(os.path.lexists(f"{root / e[0]}{x}") for x in siblings)]
  242. if failed or unsettled:
  243. if unsettled and not failed:
  244. print(f"⚠ The interrupted ZIP update left {', '.join(unsettled)} unsettled; its journal "
  245. f"{journal} stays and the next launch retries.", file=sys.stderr)
  246. return changed # the journal stays: the next launch retries
  247. journal.unlink(missing_ok=True)
  248. if changed:
  249. print("⚠ A previous ZIP `hermes update` was interrupted mid-swap; the old install was put back. "
  250. "`hermes update` updates it again.", file=sys.stderr)
  251. return changed