_early_recovery.py 71 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467
  1. """Startup requests for PM recovery and rescue of orphaned launchers."""
  2. from __future__ import annotations
  3. import contextlib
  4. import errno
  5. import os
  6. import re
  7. import subprocess
  8. import sys
  9. import time
  10. from pathlib import Path, PurePosixPath
  11. # Older installers left renamed launchers behind on failure. The startup
  12. # orphan sweep still restores them. Generation installs never rename live shims.
  13. QUARANTINE_RESTORE_BACKOFF_MS: tuple[int, ...] = (0, 100, 250, 500, 1000)
  14. def restore_quarantined_shims(
  15. moved: list[tuple[Path, Path]], *, stream=None,
  16. backoff_ms: tuple[int, ...] = QUARANTINE_RESTORE_BACKOFF_MS,
  17. ) -> list[tuple[Path, Path]]:
  18. """Rename quarantined shims back, retrying a lock instead of giving up.
  19. A pair is not a failure when ``original`` already exists or ``quarantined`` has gone: the
  20. installer wrote a fresh shim, or a concurrent sweep won the race. Both are silent, so two
  21. processes sweeping the same orphan cannot produce a spurious error.
  22. """
  23. if stream is None:
  24. stream = sys.stderr
  25. failed: list[tuple[Path, Path]] = []
  26. for original, quarantined in moved:
  27. last_exc: OSError | None = None
  28. for delay_ms in backoff_ms:
  29. try:
  30. if os.path.exists(original) or not os.path.exists(quarantined):
  31. last_exc = None
  32. break
  33. if delay_ms:
  34. time.sleep(delay_ms / 1000.0)
  35. os.rename(quarantined, original)
  36. last_exc = None
  37. break
  38. except OSError as exc:
  39. last_exc = exc
  40. if last_exc is None:
  41. continue
  42. failed.append((original, quarantined))
  43. name = os.path.basename(str(original))
  44. stem = name[:-4] if name.lower().endswith(".exe") else name
  45. print(
  46. f" ✖ FAILED to restore {name} "
  47. f"({last_exc.__class__.__name__}) — it is still quarantined "
  48. f"as {os.path.basename(str(quarantined))}.\n"
  49. f" `{stem}` will NOT be on PATH until it is put back. Run this, "
  50. f"then re-run the update:\n"
  51. f' move "{quarantined}" "{original}"',
  52. file=stream,
  53. )
  54. return failed
  55. # Set only when this process successfully finishes a deferred core install for an ``update``
  56. # invocation. The CLI import that follows must not resolve external secret sources: a configured
  57. # source can map cryptography._rust and immediately recreate the self-lock marker this fresh
  58. # process just consumed. Process-local on purpose so children do not inherit the exception.
  59. _UPDATE_RETRY_RECOVERED = False
  60. def _should_skip_external_secret_sources() -> bool:
  61. """True inside any ``hermes update`` process (and its import probes).
  62. Every dotenv load in the process — ``hermes_cli.main``, ``run_agent``, ``cli`` — consults
  63. this, so the updater never resolves external secret sources: on Windows they map
  64. ``cryptography._rust.pyd`` into the process replacing that venv, and everywhere a slow
  65. ``op``/``bws``/command helper (up to 120s per source) would run inside the updater's
  66. 120s critical-module import probe and be reported as an import-health timeout.
  67. Profile flags are stripped before ``hermes_cli.main`` loads dotenv, so ``argv[1]`` is
  68. the authoritative subcommand.
  69. """
  70. return _UPDATE_RETRY_RECOVERED or sys.argv[1:2] == ["update"]
  71. def _project_root() -> Path:
  72. return Path(__file__).resolve().parent.parent
  73. def _read_marker_attempts(marker_path: Path) -> int:
  74. """Attempt counter from a marker's opportunistic JSON body; corrupt/missing → 0."""
  75. try:
  76. raw = marker_path.read_text(encoding="utf-8-sig", errors="replace").strip()
  77. except OSError:
  78. return 0
  79. if not raw:
  80. return 0
  81. try:
  82. import json
  83. attempts = json.loads(raw).get("attempts", 0)
  84. if type(attempts) is not int: # only our writers' ints count: Infinity/NaN/true never reach int()
  85. raise TypeError(attempts)
  86. return max(0, attempts)
  87. except (ValueError, AttributeError, TypeError):
  88. for line in reversed(raw.splitlines()):
  89. key, separator, value = line.partition("=")
  90. if separator and key.strip() == "attempts":
  91. try:
  92. return max(0, int(value))
  93. except ValueError:
  94. return 0
  95. return 0
  96. def _process_state(pid: int) -> str | None:
  97. """Single-letter process state (``ps`` style), or ``None`` when unknowable.
  98. ``os.kill(pid, 0)`` also succeeds for a ZOMBIE — a process that exited but
  99. whose parent has not reaped it yet. Reading the state lets callers treat a
  100. zombie as dead, so a crashed update stage lingering under an un-reaping
  101. parent cannot keep a stale update marker "live" for the whole age ceiling
  102. (#77259, #120635, #125932). Best-effort and stdlib-only: on any failure the
  103. answer is ``None`` and callers keep their signal-0 verdict.
  104. """
  105. if sys.platform == "linux":
  106. try:
  107. with open(f"/proc/{pid}/stat", "rb") as fh:
  108. stat = fh.read()
  109. except OSError:
  110. return None
  111. # Field 3 is the state, but comm may contain spaces/parens: anchor on
  112. # the closing paren of comm instead of splitting on whitespace.
  113. comm_end = stat.rfind(b")")
  114. if comm_end < 0:
  115. return None
  116. return stat[comm_end + 2 : comm_end + 3].decode("ascii", "replace") or None
  117. if sys.platform == "darwin":
  118. try:
  119. out = subprocess.run(
  120. ["ps", "-o", "stat=", "-p", str(pid)],
  121. capture_output=True, text=True, encoding="utf-8", errors="replace",
  122. timeout=5, check=False,
  123. ).stdout
  124. except (OSError, subprocess.SubprocessError):
  125. return None
  126. return out.strip()[:1] or None
  127. return None
  128. def _pid_is_running(pid: int) -> bool:
  129. """Best-effort stdlib-only process liveness probe.
  130. ``os.kill(pid, 0)`` is not a no-op on Windows, so use the Win32 process handle API there. An
  131. access-denied result counts as live: racing an elevated updater is worse than postponing
  132. recovery for one launch. A zombie (exited, un-reaped) counts as dead — see
  133. :func:`_process_state`.
  134. """
  135. if pid <= 0:
  136. return False
  137. if sys.platform == "win32":
  138. try:
  139. import ctypes
  140. synchronize = 0x00100000
  141. kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
  142. kernel32.OpenProcess.argtypes = [ctypes.c_ulong, ctypes.c_int, ctypes.c_ulong]
  143. kernel32.OpenProcess.restype = ctypes.c_void_p
  144. kernel32.WaitForSingleObject.argtypes = [ctypes.c_void_p, ctypes.c_ulong]
  145. kernel32.WaitForSingleObject.restype = ctypes.c_ulong
  146. kernel32.CloseHandle.argtypes = [ctypes.c_void_p]
  147. kernel32.CloseHandle.restype = ctypes.c_int
  148. handle = kernel32.OpenProcess(synchronize, False, pid)
  149. if not handle:
  150. return ctypes.get_last_error() == 5 # ERROR_ACCESS_DENIED
  151. try:
  152. return kernel32.WaitForSingleObject(handle, 0) == 258
  153. finally:
  154. kernel32.CloseHandle(handle)
  155. except Exception:
  156. return True
  157. try:
  158. os.kill(pid, 0) # windows-footgun: ok — Windows returns above
  159. except ProcessLookupError:
  160. return False
  161. except PermissionError:
  162. return True
  163. except OSError:
  164. return False
  165. state = _process_state(pid)
  166. if state is not None and state.upper().startswith("Z"):
  167. return False # exited, unreaped — not a live owner
  168. return True
  169. def _marker_owner_is_live(marker: Path) -> bool:
  170. """True when a legacy update marker names a process still running."""
  171. try:
  172. body = marker.read_text(encoding="utf-8-sig", errors="replace")
  173. except OSError:
  174. return False
  175. for line in body.splitlines():
  176. key, separator, value = line.partition("=")
  177. if separator and key.strip() == "pid":
  178. try:
  179. return _pid_is_running(int(value.strip()))
  180. except ValueError:
  181. return False
  182. return False
  183. # ``hermes update`` writes this into the git dir right before git moves the checkout and removes it
  184. # once git has exited (a kill is the only exit that keeps it). Git rewrites the tree file by file and
  185. # moves HEAD last, so an update killed in between leaves HEAD on the old commit with a prefix of the
  186. # files already new; that mixed tree fails at import in every entry point, ``hermes update`` included.
  187. INTERRUPTED_PULL_MARKER = "hermes-update-pull"
  188. # A fast-forward takes seconds; past this a "live" owner pid is a recycled one.
  189. _INTERRUPTED_PULL_MAX_AGE_SECONDS = 10 * 60
  190. # The user (or a killed updater) is mid-operation: its own state files own the tree.
  191. _GIT_OPERATION_IN_PROGRESS = ("MERGE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD", "rebase-merge", "rebase-apply")
  192. _REGULAR_FILE_MODES = ("100644", "100755")
  193. # What git writes into the tree itself: files and symlinks (a gitlink, 160000, is a submodule's own checkout).
  194. _WORKTREE_BLOB_MODES = (*_REGULAR_FILE_MODES, "120000")
  195. def git_operation_in_progress(root: Path) -> str | None:
  196. """Return the active Git operation currently controlling *root*, if any."""
  197. git_dir = _git_dir(root)
  198. for marker in _GIT_OPERATION_IN_PROGRESS:
  199. state = git_dir / marker
  200. if not state.exists():
  201. continue
  202. if marker == "rebase-apply":
  203. return "am" if (state / "applying").exists() else "rebase"
  204. if marker == "rebase-merge":
  205. return "rebase"
  206. return marker.removesuffix("_HEAD").lower().replace("_", "-")
  207. return None
  208. def _git_dir(root: Path) -> Path:
  209. """``root``'s git dir: ``.git`` itself, or where a linked worktree's ``.git`` file points."""
  210. dot_git = root / ".git"
  211. if dot_git.is_file():
  212. text = dot_git.read_text(encoding="utf-8-sig").strip()
  213. if text.startswith("gitdir:"):
  214. return root / text[len("gitdir:"):].strip()
  215. return dot_git
  216. def interrupted_pull_marker(root: Path) -> Path:
  217. return _git_dir(root) / INTERRUPTED_PULL_MARKER
  218. # The repair's own code is in the tree a killed move tears (``hermes_bootstrap``, this module, the
  219. # package initializer, the lock/custody modules). ``update_cmd_commit.arm_tree_move`` publishes this
  220. # closure, as committed at the marker's ``pre``, beside the marker before git writes; a minted
  221. # launcher whose checkout import fails runs the repair from there, reading the same files from git's
  222. # objects first when no updater published them (``_launchers._CLOSURE_REPAIR``).
  223. # The modules import only the stdlib and each other; the package initializer is published empty.
  224. RECOVERY_CLOSURE_DIR = "hermes-update-recovery"
  225. RECOVERY_CLOSURE = ("hermes_cli/_early_recovery.py", "hermes_cli/update_lock.py", "hermes_cli/update_custody.py")
  226. # ``<blob id> <path>`` per published file (``RECOVERY_CLOSURE`` + the empty package initializer):
  227. # a closure is used only when every file hashes to its id (a power loss can leave the renamed dir
  228. # with files git's objects never had), else it is rebuilt from git's objects.
  229. RECOVERY_CLOSURE_MANIFEST = "MANIFEST"
  230. RECOVERY_CLOSURE_INIT = "hermes_cli/__init__.py"
  231. def recovery_closure_dir(root: Path, pre: str) -> Path:
  232. if not is_object_id(pre): # never a revision (``HEAD``) or a path (``../x``) under the git dir
  233. raise ValueError(f"not a commit id: {pre!r}")
  234. return _git_dir(root) / RECOVERY_CLOSURE_DIR / pre
  235. def is_object_id(value: str) -> bool:
  236. return bool(re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", value or ""))
  237. def blob_id(data: bytes, like: str) -> str:
  238. """Git's object id of a blob holding ``data``, in ``like``'s object format (SHA-1 or SHA-256)."""
  239. import hashlib
  240. return hashlib.new("sha1" if len(like) == 40 else "sha256", b"blob %d\0" % len(data) + data).hexdigest()
  241. def recovery_closure_verified(closure: Path, pre: str) -> bool:
  242. """Every closure file present with exactly the bytes its manifest names, and nothing missing."""
  243. try:
  244. listed = dict(line.split(" ", 1)[::-1] for line in
  245. (closure / RECOVERY_CLOSURE_MANIFEST).read_text(encoding="utf-8-sig").splitlines())
  246. return (set(listed) == {*RECOVERY_CLOSURE, RECOVERY_CLOSURE_INIT}
  247. and listed[RECOVERY_CLOSURE_INIT] == blob_id(b"", pre)
  248. and all(blob_id((closure / rel).read_bytes(), pre) == oid for rel, oid in listed.items()))
  249. except (OSError, ValueError):
  250. return False
  251. def _git_executable(recorded_by_updater: str = "") -> str:
  252. """The git ``hermes update`` runs (``_subprocess_compat.expose_pm_git``), without installing it.
  253. The absolute git the killed updater recorded in its marker first: finding PM's copy needs
  254. ``pm``, which imports ``hermes_constants``, and a merge killed while writing that module (or
  255. anything else ``pm`` imports) leaves only this path to the repair. Then PATH's git. A Windows install whose only git is the one install.ps1 staged in PM's store
  256. has none on PATH until the updater exposes it, so this falls back to that copy: PM's recorded
  257. entry, or the lockfile's pinned entry the installer extracted without recording it. A bare
  258. ``git`` there dies with WinError 2 and the torn tree this repair exists for stays torn.
  259. """
  260. import shutil
  261. if recorded_by_updater and os.path.isfile(recorded_by_updater) and _is_git(recorded_by_updater):
  262. return recorded_by_updater
  263. found = shutil.which("git")
  264. if found:
  265. return found
  266. with contextlib.suppress(Exception): # no PM, no store, no Windows git package: PATH's answer stands
  267. import pm
  268. from pm import paths
  269. from pm.lock import Lockfile
  270. recorded = pm.installed_package("git", allow_outdated=True)
  271. if recorded is not None and recorded.binary is not None and recorded.binary.is_file():
  272. return str(recorded.binary)
  273. package, target = pm.get_package("git"), pm.current_target()
  274. version = Lockfile(paths.lockfile_path()).version("git")
  275. staged = package.binary(paths.store_root() / package.store_entry(version, target), target)
  276. if staged is not None and staged.is_file():
  277. return str(staged)
  278. return "git"
  279. def _is_git(path: str) -> bool:
  280. """``path`` still runs as git: ``--version`` exits 0 and says ``git version`` (m5). A recorded
  281. git that decayed into anything else (gone executable, a stub that exits 0 silently) must not
  282. answer the repair's questions: its empty ``rev-parse`` would read as "HEAD moved"."""
  283. try:
  284. probe = subprocess.run([path, "--version"], capture_output=True, text=True, encoding="utf-8",
  285. errors="replace", timeout=30, stdin=subprocess.DEVNULL,
  286. creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), check=False)
  287. except (OSError, ValueError, subprocess.SubprocessError):
  288. return False
  289. return probe.returncode == 0 and probe.stdout.startswith("git version")
  290. # A full object name (SHA-1 or SHA-256): what `git rev-parse --verify HEAD` must print (m5).
  291. _OID = re.compile(r"[0-9a-f]{40}(?:[0-9a-f]{24})?")
  292. def _trees_git_could_write(git, pre: str, target: str) -> tuple[list[str], set[str]]:
  293. """The trees the killed git was moving the checkout to, and the paths whose new content is unknowable.
  294. A fast-forward or ``reset --hard`` writes ``target``. On a custom branch the updater runs
  295. ``git merge``, whose files are the merge of both sides: ``merge-tree`` computes the same tree. A
  296. conflicted path (its markers carry other labels) and, on git < 2.38 (no ``--write-tree``), every
  297. path both sides changed count as git's whatever their content.
  298. """
  299. base = git("merge-base", pre, target).stdout.strip()
  300. if not base or base == pre: # fast-forward, or unrelated histories (only a reset can land those)
  301. return [target], set()
  302. merged = git("merge-tree", "--write-tree", "-z", "--name-only", "--no-messages", pre, target)
  303. if merged.returncode in (0, 1): # 1: conflicts
  304. tree, *conflicted = merged.stdout.split("\0")
  305. return [target, tree], set(filter(None, conflicted))
  306. changed = [set(filter(None, git("diff", "--name-only", "-z", "--no-renames", base, side).stdout.split("\0")))
  307. for side in (pre, target)]
  308. return [target], changed[0] & changed[1]
  309. def _hash_worktree(git, paths: list[str]) -> dict[str, str]:
  310. """Blob ids of the checkout's files, through the repo's clean filters, like ``git add`` would store."""
  311. listed = [p for p in paths if "\n" not in p] # --stdin-paths is newline-delimited
  312. blobs = {}
  313. if listed:
  314. hashed = git("hash-object", "--stdin-paths", stdin="\n".join(listed) + "\n")
  315. if hashed.returncode != 0:
  316. raise subprocess.SubprocessError(hashed.stderr.strip())
  317. blobs.update(zip(listed, hashed.stdout.split()))
  318. for path in set(paths) - set(listed):
  319. blobs[path] = git("hash-object", "--", path).stdout.strip()
  320. return blobs
  321. def _modes_git_wrote(git, root: Path, entries: dict, links: set[str]) -> set[str]:
  322. """Paths whose new entry keeps the ``pre`` blob under another mode, already in that mode.
  323. Equal bytes cannot tell a regular file holding ``original`` from a symlink to ``original`` (one
  324. blob id), nor 644 from 755: only the entry's mode can (review N07). The worktree shows a link
  325. (lstat), except under ``core.symlinks=false`` where git checks a link out as a plain file holding
  326. its target, and the executable bit, except on Windows. The index shows the mode git staged where
  327. the worktree cannot; git writes it after the files, so it covers a git killed after a whole tree.
  328. """
  329. wanted = {path: {m for m, b in new if b == old_blob and m != old_mode}
  330. for path, (old_mode, old_blob, new) in entries.items() if old_blob is not None}
  331. wanted = {path: modes for path, modes in wanted.items() if modes}
  332. if not wanted:
  333. return set()
  334. symlinks = git("config", "--type=bool", "core.symlinks").stdout.strip() != "false"
  335. staged = git("ls-files", "-s", "-z", "--", *wanted)
  336. if staged.returncode != 0:
  337. raise subprocess.SubprocessError(staged.stderr.strip())
  338. index = {path: meta.split()[0] for meta, _tab, path in
  339. (record.partition("\t") for record in staged.stdout.split("\0")) if path}
  340. written = set()
  341. for path, modes in wanted.items():
  342. if path in links:
  343. shown = {"120000"}
  344. elif not (root / path).is_file():
  345. shown = set() # missing: the caller judges it by absence
  346. else:
  347. shown = set(_REGULAR_FILE_MODES) if sys.platform == "win32" else \
  348. {"100755" if (root / path).stat().st_mode & 0o100 else "100644"}
  349. if not symlinks:
  350. shown.add("120000")
  351. if (entries[path][0] not in shown and shown & modes) or index.get(path) in modes:
  352. written.add(path)
  353. return written
  354. def _paths_git_wrote(git, root: Path, pre: str, target: str) -> tuple[list[str], list[str], set[str], set[str]] | None:
  355. """Paths the killed git already touched on the way to ``target``: (restore from HEAD, remove as added,
  356. directories git may have created for its added files, the added ones to keep aside, not delete).
  357. Git rewrites a file as unlink, create, write, so a kill leaves it missing, empty or cut short:
  358. all of those count as git's, like the full new blob. Content that matches neither side and is not
  359. the start of a new blob is the user's own edit (e.g. a re-applied stash) and is left alone. An
  360. added path holding less than a whole new blob may equally be the user's own file created there
  361. after the kill (a ``touch``, a first line): it leaves the tree, but renamed aside, never deleted.
  362. ``None``: git no longer knows ``target``.
  363. """
  364. if git("rev-parse", "-q", "--verify", f"{target}^{{commit}}").returncode != 0:
  365. return None
  366. trees, unknown = _trees_git_could_write(git, pre, target)
  367. entries = {} # path -> (pre mode, pre blob or None when git adds it, [(new mode, new blob or None)])
  368. for tree in trees:
  369. diff = git("diff", "--raw", "-z", "--no-renames", "--no-abbrev", pre, tree)
  370. if diff.returncode != 0:
  371. raise subprocess.SubprocessError(diff.stderr.strip())
  372. parts = diff.stdout.split("\0")
  373. for meta, path in zip(parts[::2], parts[1::2]):
  374. old_mode, new_mode, old_blob, new_blob, status = meta.lstrip(":").split()
  375. if (old_mode if status == "D" else new_mode) not in _WORKTREE_BLOB_MODES:
  376. continue
  377. entry = entries.setdefault(path, (old_mode, None if status == "A" else old_blob, []))
  378. entry[2].append((new_mode, None if status == "D" else new_blob))
  379. # A symlink's blob is its target text, read without following it; core.symlinks=false checks a
  380. # link out as a plain file holding that text, which hash-object already matches.
  381. links = {path for path in entries if os.path.islink(root / path)}
  382. worktree_blob = _hash_worktree(git, [path for path in entries if path not in links and (root / path).is_file()])
  383. worktree_blob.update({path: blob_id(os.fsencode(os.readlink(root / path)), pre) for path in links})
  384. retyped = _modes_git_wrote(git, root, entries, links)
  385. restore, added, kept = [], [], set()
  386. for path, (old_mode, old_blob, new) in entries.items():
  387. file, blobs = root / path, {blob for _mode, blob in new if blob}
  388. if path not in worktree_blob:
  389. written = old_blob is not None # unlinked (or deleted), not yet recreated
  390. elif worktree_blob[path] == old_blob: # only the entry's mode tells whether git got here
  391. written = path in retyped
  392. elif worktree_blob[path] in blobs or path in unknown:
  393. written = True
  394. elif path in links: # git creates a symlink whole: any other target is the user's
  395. written = False
  396. else: # git's own file cut short starts one of the new blobs
  397. content = file.read_bytes()
  398. written = False
  399. for blob in blobs:
  400. shown = git("cat-file", "--filters", f"--path={path}", blob, text=False)
  401. if shown.returncode != 0:
  402. raise subprocess.SubprocessError(shown.stderr.decode(errors="replace").strip())
  403. written = written or shown.stdout.startswith(content)
  404. if written:
  405. (added if old_blob is None else restore).append(path)
  406. if old_blob is None and worktree_blob.get(path) not in blobs: # a whole blob is in git's objects
  407. kept.add(path)
  408. new_dirs = {str(parent) for path, (_m, old_blob, _n) in entries.items() if old_blob is None
  409. for parent in PurePosixPath(path).parents if parent.parts}
  410. if new_dirs:
  411. new_dirs -= set(git("ls-tree", "-r", "-d", "--name-only", "-z", pre).stdout.split("\0"))
  412. return restore, added, new_dirs, kept
  413. # Launches that start together after a killed update (a restarting gateway or Desktop backend next to
  414. # the user's CLI) restore one at a time: the claim holder owns the git index, the rest wait for it and
  415. # relaunch from its tree. An OS lock, not a pid file: the kernel drops it with its owner, so a dead
  416. # launch's claim is broken without a check-then-unlink race.
  417. _RESTORE_CLAIM = "hermes-update-pull.claim"
  418. _RESTORE_CLAIM_WAIT_SECONDS = 10.0
  419. _merge_advice_shown = False
  420. # flock's EWOULDBLOCK and msvcrt LK_NBLCK's EACCES/EDEADLOCK: another process holds the claim.
  421. _CLAIM_HELD_ERRNOS = frozenset({errno.EAGAIN, errno.EWOULDBLOCK, errno.EACCES, errno.EDEADLK})
  422. def _lock_fd(fd: int, lock: bool) -> bool:
  423. """False only while another launch holds the claim.
  424. A filesystem that cannot lock at all (ENOLCK on NFS without lockd, EOPNOTSUPP/EINVAL on some SMB
  425. shares) proceeds unguarded like a read-only git dir: counting it as "held" would stop every launch
  426. from ever repairing a checkout the restore handled fine before claims existed.
  427. """
  428. try:
  429. if sys.platform == "win32":
  430. import msvcrt
  431. os.lseek(fd, 0, os.SEEK_SET) # msvcrt locks from the file position
  432. msvcrt.locking(fd, msvcrt.LK_NBLCK if lock else msvcrt.LK_UNLCK, 1)
  433. else:
  434. import fcntl
  435. fcntl.flock(fd, (fcntl.LOCK_EX | fcntl.LOCK_NB) if lock else fcntl.LOCK_UN)
  436. except OSError as exc:
  437. return not (lock and exc.errno in _CLAIM_HELD_ERRNOS)
  438. return True
  439. @contextlib.contextmanager
  440. def _restore_claim(git_dir: Path):
  441. """Yields True while this launch holds the restore claim, False when another held it past the wait."""
  442. try:
  443. fd = os.open(git_dir / _RESTORE_CLAIM, os.O_RDWR | os.O_CREAT, 0o644)
  444. except OSError: # read-only git dir: no restore can write there either, the attempt reports why
  445. yield True
  446. return
  447. try:
  448. deadline = time.monotonic() + _RESTORE_CLAIM_WAIT_SECONDS
  449. while not _lock_fd(fd, True):
  450. if time.monotonic() > deadline:
  451. yield False
  452. return
  453. time.sleep(0.05)
  454. try:
  455. os.ftruncate(fd, 0)
  456. os.write(fd, f"pid={os.getpid()}\n".encode())
  457. except OSError:
  458. pass
  459. try:
  460. yield True
  461. finally:
  462. _lock_fd(fd, False)
  463. finally:
  464. os.close(fd)
  465. class _Holder(str):
  466. """A process that may own a git lock, e.g. ``pid 4242 (git commit)``: truthy, and never ``is True``."""
  467. # Git subcommands that only read: none takes ``index.lock`` and then waits with its fd CLOSED. Every
  468. # other git (``commit``/``merge``/``rebase``/``am``/``stash``... in the editor or a hook, a dashed
  469. # ``git-commit`` from git-core, an alias, a wrapper, a third-party ``git-<tool>``, a git whose
  470. # subcommand cannot be read) may hold ``index.lock`` with no fd naming it, so it counts as a holder.
  471. # A reader (a paged ``git log``, ``cat-file --batch``, a background ``fetch``) counts only while its
  472. # fd is on the lock.
  473. _READER_GIT = frozenset({
  474. "annotate", "blame", "cat-file", "check-attr", "check-ignore", "check-mailmap", "check-ref-format",
  475. "cherry", "count-objects", "describe", "diff", "diff-files", "diff-index", "diff-tree",
  476. "for-each-ref", "fsmonitor--daemon", "grep", "help", "log", "ls-files", "ls-remote", "ls-tree",
  477. "merge-base", "name-rev", "range-diff", "rev-list", "rev-parse", "shortlog", "show", "show-branch",
  478. "show-ref", "status", "var", "verify-commit", "verify-tag", "version", "whatchanged",
  479. "credential", "credential-cache", "credential-store",
  480. # Transfers write objects and refs, never the index.
  481. "fetch", "fetch-pack", "http-fetch", "index-pack", "pack-objects", "remote-http", "remote-https",
  482. "upload-pack",
  483. })
  484. # Global options that take the NEXT argument as their value (``git -C <dir> -c k=v commit``).
  485. _GIT_VALUE_OPTIONS = frozenset({
  486. "-C", "-c", "--git-dir", "--work-tree", "--namespace", "--config-env", "--super-prefix", "--attr-source",
  487. })
  488. # What points a git at a repository other than its cwd.
  489. _GIT_LOCATION_ENV = ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR")
  490. def _git_program(arg0: str) -> str | None:
  491. """``git`` or ``git-<sub>`` when ``arg0`` (an argv[0], a comm, a psutil name) runs git, else None.
  492. Any directory (``/usr/lib/git-core/git-commit``), either separator, ``.exe`` and case folded:
  493. Windows runs ``git.exe`` and ``git-commit.exe``."""
  494. name = re.split(r"[\\/]", arg0.strip())[-1].lower()
  495. name = name.removesuffix(".exe")
  496. return name if name == "git" or name.startswith("git-") else None
  497. def _git_argv(cmdline: bytes | list[str]) -> list[str]:
  498. if isinstance(cmdline, bytes):
  499. return [a.decode("utf-8", "replace") for a in cmdline.split(b"\0") if a]
  500. return [str(a) for a in cmdline if a]
  501. def _git_subcommand_of(cmdline: bytes | list[str]) -> str | None:
  502. """The subcommand a git argv runs: ``commit`` for ``git -C x -c k=v commit`` and for a dashed
  503. ``/usr/lib/git-core/git-commit`` (``git-commit.exe``); None when argv[0] is not git or names none."""
  504. args = _git_argv(cmdline)
  505. program = _git_program(args[0]) if args else None
  506. if program is None:
  507. return None
  508. if program != "git":
  509. return program[len("git-"):]
  510. it = iter(args[1:])
  511. for arg in it:
  512. if arg in _GIT_VALUE_OPTIONS:
  513. next(it, None)
  514. elif not arg.startswith("-"):
  515. return arg
  516. return None
  517. def _checkout_places(*paths: Path | str | None) -> tuple[str, ...]:
  518. """The checkout's own paths as given AND resolved, so another process's path need not be resolved."""
  519. places = {os.path.normcase(os.path.normpath(os.path.abspath(p))) for p in paths if p is not None}
  520. places |= {os.path.normcase(os.path.realpath(p)) for p in paths if p is not None}
  521. return tuple(sorted(places))
  522. def _path_within(path: str, places: tuple[str, ...]) -> bool:
  523. """``path`` is one of ``places`` or under it, by PATH COMPONENTS (``/x/hermes-backup`` is not in ``/x/hermes``).
  524. Lexical: another process's cwd is already the kernel's resolved path, and resolving a stranger's
  525. path would touch whatever it names."""
  526. if not os.path.isabs(path):
  527. return False
  528. lexical = os.path.normcase(os.path.normpath(path))
  529. for place in places:
  530. try:
  531. if os.path.commonpath([lexical, place]) == place:
  532. return True
  533. except ValueError: # another drive
  534. continue
  535. return False
  536. def git_works_in(args: list[str], cwd: str | None, env: dict[str, str] | None, places: tuple[str, ...]) -> bool:
  537. """A git with this argv/cwd/environment works in ``places``: its cwd, a path argument (absolute,
  538. or relative to its cwd, ``--opt=<path>`` too) or a ``GIT_DIR``-style variable is inside one."""
  539. def inside(value: str) -> bool:
  540. if not value:
  541. return False
  542. if not os.path.isabs(value):
  543. if not cwd:
  544. return False
  545. value = os.path.join(cwd, value)
  546. return _path_within(value, places)
  547. if cwd and _path_within(cwd, places):
  548. return True
  549. for arg in args[1:]:
  550. if inside(arg.partition("=")[2] if arg.startswith("-") else arg):
  551. return True
  552. return any(inside((env or {}).get(key, "")) for key in _GIT_LOCATION_ENV)
  553. def _could_write(uid: int | None, git_dir: Path) -> bool:
  554. """A process running as ``uid`` may write ``git_dir`` (root, its owner, or a group/world-writable dir)."""
  555. try:
  556. st = os.stat(git_dir)
  557. except OSError:
  558. return True
  559. return uid is None or uid in (0, st.st_uid) or bool(st.st_mode & 0o022)
  560. def _held_open(path: Path, root: Path | None = None, *, any_git: bool = False) -> _Holder | bool | None:
  561. """Whether a running process may still own ``path`` (a git lock): the holder, False, or None (unknowable).
  562. The holder is a truthy :class:`_Holder` naming it (``pid 4242 (git commit)``). False is proof, not
  563. a guess: Linux reads every process's fds through /proc and, with ``root``, also counts any live git
  564. working in the checkout (cwd, a path argument or ``GIT_DIR``) that is not a pure reader
  565. (:data:`_READER_GIT`; ``any_git`` counts readers too), whatever form runs it: ``git commit``
  566. waiting in the editor has CLOSED its lock fd, and a dashed ``git-commit``, an alias or ``git.exe``
  567. is the same git. A git that could write the git dir but whose cwd or environment cannot be read
  568. makes the answer None. macOS/BSD ask ``lsof`` for open fds and ``ps`` for any such git (``ps``
  569. cannot say where it works, so any counts). Without either check the answer is None and the caller
  570. never deletes the lock. Windows needs no answer here: it refuses to unlink a file another process
  571. has open, so the caller's unlink is the probe.
  572. """
  573. if (Path("/proc") / "self" / "fd").is_dir():
  574. return _held_open_proc(path, root, any_git)
  575. return _held_open_lsof(path, root, any_git)
  576. # Daemons a git spawns that outlive it, often with the checkout as cwd, and never take the index
  577. # lock: even ``any_git`` ignores them, or one cached credential would keep a dead lock for hours.
  578. _NEVER_LOCKS = frozenset({"credential-cache--daemon", "fsmonitor--daemon"})
  579. def _counts_as_holder(args: list[str], any_git: bool) -> bool:
  580. sub = _git_subcommand_of(args)
  581. return sub not in _NEVER_LOCKS and (any_git or sub not in _READER_GIT)
  582. def _exited(pid_dir: Path) -> bool:
  583. """The process has released its memory (exiting, or a zombie) or is gone.
  584. The kernel drops its fds before its cwd, and refuses its cwd/environ once it is exiting, so an
  585. unreadable cwd there is not a hidden git: counting it made any git exiting anywhere on a busy
  586. machine postpone the repair. A live process keeps a non-empty argv."""
  587. try:
  588. return not (pid_dir / "cmdline").read_bytes()
  589. except OSError:
  590. return not pid_dir.exists()
  591. def _held_open_proc(path: Path, root: Path | None, any_git: bool) -> _Holder | bool | None:
  592. proc = Path("/proc")
  593. target = os.path.realpath(path)
  594. places = _checkout_places(root, path.parent) if root is not None else None
  595. unknowable = False
  596. def name(args: list[str], pid_dir: Path) -> str:
  597. if args:
  598. return " ".join([os.path.basename(args[0]), *args[1:3]])
  599. try:
  600. return (pid_dir / "comm").read_bytes().decode("ascii", "replace").strip() # /proc: Linux only
  601. except OSError:
  602. return "?"
  603. for pid_dir in proc.glob("[0-9]*"):
  604. try:
  605. if any(os.readlink(entry.path) == target for entry in os.scandir(pid_dir / "fd")):
  606. try:
  607. args = _git_argv((pid_dir / "cmdline").read_bytes())
  608. except OSError:
  609. args = []
  610. return _Holder(f"pid {pid_dir.name} ({name(args, pid_dir)})")
  611. except OSError:
  612. pass
  613. if places is None:
  614. continue
  615. try:
  616. uid = pid_dir.stat().st_uid
  617. except OSError:
  618. continue # exited
  619. try:
  620. args = _git_argv((pid_dir / "cmdline").read_bytes())
  621. comm = (pid_dir / "comm").read_bytes().decode("ascii", "replace").strip()
  622. except OSError:
  623. if pid_dir.exists() and _could_write(uid, path.parent):
  624. unknowable = True # hidden from us (hidepid) yet able to write here: it may be a git
  625. continue
  626. if not args:
  627. continue # a zombie or a kernel thread holds nothing
  628. if _git_program(args[0]) is None and _git_program(comm) is None:
  629. continue
  630. if int(pid_dir.name) == os.getpid() or not _counts_as_holder(args, any_git):
  631. continue
  632. try:
  633. cwd = os.readlink(pid_dir / "cwd").removesuffix(" (deleted)")
  634. env = dict(entry.decode("utf-8", "replace").partition("=")[::2]
  635. for entry in (pid_dir / "environ").read_bytes().split(b"\0") if entry)
  636. except OSError:
  637. if _exited(pid_dir):
  638. continue # exited since its argv was read: no cwd/environ left, and it holds nothing
  639. if git_works_in(args, None, None, places):
  640. return _Holder(f"pid {pid_dir.name} ({name(args, pid_dir)})")
  641. if pid_dir.exists() and _could_write(uid, path.parent):
  642. unknowable = True # a lock-keeping git we cannot place may be working here
  643. continue
  644. if git_works_in(args, cwd, env, places):
  645. return _Holder(f"pid {pid_dir.name} ({name(args, pid_dir)})")
  646. return None if unknowable else False
  647. def _held_open_lsof(path: Path, root: Path | None, any_git: bool) -> _Holder | bool | None:
  648. import shutil
  649. lsof = shutil.which("lsof") or next((p for p in ("/usr/sbin/lsof", "/usr/bin/lsof") if os.path.isfile(p)), None)
  650. if lsof is None:
  651. return None
  652. try:
  653. found = subprocess.run([lsof, "-F", "pc", "--", str(path)], capture_output=True, text=True, encoding="utf-8",
  654. errors="replace", timeout=20,
  655. stdin=subprocess.DEVNULL, check=False)
  656. except (OSError, subprocess.SubprocessError):
  657. return None
  658. pids = [line[1:] for line in found.stdout.splitlines() if line.startswith("p")]
  659. if pids:
  660. names = [line[1:] for line in found.stdout.splitlines() if line.startswith("c")]
  661. return _Holder(f"pid {pids[0]} ({names[0] if names else '?'})")
  662. if found.returncode not in (0, 1): # lsof exits 1 when nothing has the file open
  663. return None
  664. return False if root is None else _ps_git_holder(lsof, path.parent, root, any_git)
  665. def _lsof_cwds(lsof: str, pids: list[str]) -> dict[str, str] | None:
  666. """Each pid's cwd from ``lsof -d cwd`` (macOS has no /proc); None when lsof cannot answer."""
  667. try:
  668. out = subprocess.run([lsof, "-a", "-d", "cwd", "-F", "pn", "-p", ",".join(pids)], capture_output=True,
  669. text=True, encoding="utf-8", errors="replace", timeout=20, stdin=subprocess.DEVNULL, check=False)
  670. except (OSError, subprocess.SubprocessError):
  671. return None
  672. if out.returncode not in (0, 1):
  673. return None
  674. cwds: dict[str, str] = {}
  675. pid = None
  676. for line in out.stdout.splitlines():
  677. if line.startswith("p"):
  678. pid = line[1:]
  679. elif line.startswith("n") and pid is not None:
  680. cwds[pid] = line[1:]
  681. return cwds
  682. def _still_running(pid: int) -> bool:
  683. """POSIX only (the lsof branch never runs on Windows); stdlib, as this launch-time repair must be."""
  684. try:
  685. os.kill(pid, 0) # windows-footgun: ok — lsof/ps branch is macOS/BSD only, Windows never reaches it
  686. except ProcessLookupError:
  687. return False
  688. except OSError: # EPERM: alive, another user's
  689. return True
  690. return True
  691. def _ps_git_holder(lsof: str, git_dir: Path, root: Path, any_git: bool) -> _Holder | bool | None:
  692. """An empty ``lsof`` is no proof: ``git commit`` waiting in the editor has closed its lock fd.
  693. Every git (by executable name, dashed forms included) that could write ``git_dir``, is not a pure
  694. reader, and works in the checkout (its cwd from ``lsof -d cwd``, or a path argument) keeps the
  695. lock; one whose cwd cannot be read makes the answer None. Gits elsewhere on the machine do not
  696. count, or any commit open in another repository would block every launch-time repair."""
  697. import shutil
  698. # The launcher can run with a PATH that has neither git nor ps (a Windows-style install, a
  699. # stripped service env): ps lives at /bin/ps on macOS and every BSD.
  700. ps_bin = shutil.which("ps") or next((p for p in ("/bin/ps", "/usr/bin/ps") if os.path.isfile(p)), None)
  701. if ps_bin is None:
  702. return None
  703. def ps(*columns: str) -> list[list[str]] | None:
  704. try:
  705. out = subprocess.run([ps_bin, "-A", *(f"-o{c}=" for c in columns)], capture_output=True,
  706. text=True, encoding="utf-8", errors="replace",
  707. timeout=20, stdin=subprocess.DEVNULL, check=False)
  708. except (OSError, subprocess.SubprocessError):
  709. return None
  710. if out.returncode != 0:
  711. return None
  712. return [line.split(None, len(columns) - 1) for line in out.stdout.splitlines() if line.strip()]
  713. named = ps("pid", "uid", "comm") # comm: the executable (``/usr/libexec/git-core/git-commit``)
  714. commands = ps("pid", "command")
  715. if named is None or commands is None:
  716. return None
  717. argv = {row[0]: row[1].split() for row in commands if len(row) == 2}
  718. candidates: dict[str, list[str]] = {}
  719. for row in named:
  720. if len(row) != 3 or not (row[0].isdigit() and row[1].isdigit()) or int(row[0]) == os.getpid():
  721. continue
  722. if _git_program(row[2]) is None and not (argv.get(row[0]) and _git_program(argv[row[0]][0])):
  723. continue
  724. args = argv.get(row[0]) or [row[2]]
  725. if _git_program(args[0]) is None:
  726. args = [row[2]] # an argv[0] with spaces: no subcommand, so it counts
  727. if _could_write(int(row[1]), git_dir) and _counts_as_holder(args, any_git):
  728. candidates[row[0]] = args
  729. if not candidates:
  730. return False
  731. cwds = _lsof_cwds(lsof, sorted(candidates))
  732. if cwds is None:
  733. return None
  734. places = _checkout_places(root, git_dir)
  735. for pid, args in candidates.items():
  736. cwd = cwds.get(pid)
  737. if cwd is None and _still_running(int(pid)):
  738. return None # a lock-keeping git we cannot place may be working here
  739. if git_works_in(args, cwd, None, places):
  740. return _Holder(f"pid {pid} ({' '.join([os.path.basename(args[0]), *args[1:3]])})")
  741. return False
  742. def _index_lock_holder(git_dir: Path, root: Path | None) -> str:
  743. """Who keeps ``index.lock`` for the user-facing message: ``pid N (name)`` when it can be named."""
  744. held = None if sys.platform == "win32" else _held_open(git_dir / "index.lock", root)
  745. return f"a running git ({held})" if isinstance(held, _Holder) else "a running git"
  746. def _release_dead_lock(lock: Path, root: Path | None = None, *, any_git: bool = False) -> bool:
  747. """Drop a killed git's ``lock`` (``index.lock`` refuses every git command) once its owner is PROVEN gone.
  748. False while a live git may hold it, or when this platform cannot prove it dead: the caller then
  749. keeps the interrupted-pull marker, so the next launch tries again instead of a rollback being lost.
  750. ``any_git``: every git working in ``root`` keeps it, readers included (:func:`_held_open`).
  751. """
  752. deadline = time.monotonic() + 5
  753. while lock.exists():
  754. held = None if sys.platform == "win32" else _held_open(lock, root, any_git=any_git)
  755. if held is False or sys.platform == "win32":
  756. try:
  757. lock.unlink()
  758. return True
  759. except FileNotFoundError:
  760. return True
  761. except PermissionError: # Windows: open in a live process
  762. pass
  763. elif held is None:
  764. return False
  765. if time.monotonic() > deadline:
  766. return False
  767. time.sleep(0.1)
  768. return True
  769. # The ZIP update's equivalent of the interrupted-pull marker (``_early_recovery_zip``): a journal in the
  770. # install root naming every entry the swap stages/renames. Its name stays here: the launch fast path
  771. # stats it without importing the ZIP code.
  772. ZIP_SWAP_JOURNAL = ".hermes-update-zip-swap"
  773. def write_durable_text(path: Path, text: str) -> None:
  774. """``text`` at ``path`` as one durable record, never a half-written one.
  775. The temp is an unpredictable name created exclusively (no-follow) beside ``path``: a pre-existing
  776. name there is never written through (its symlink or hardlink would carry the record onto another
  777. file) and never deleted (it may be a user's file: review Z5). fsync before the rename: an empty
  778. record over a mixed tree is no record.
  779. """
  780. flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
  781. while True:
  782. tmp = path.with_name(f"{path.name}.{os.urandom(6).hex()}.tmp")
  783. try:
  784. fd = os.open(tmp, flags, 0o644)
  785. break
  786. except FileExistsError:
  787. continue # 48 random bits taken: draw again, never reuse the name
  788. try:
  789. with os.fdopen(fd, "w", encoding="utf-8") as handle:
  790. handle.write(text)
  791. handle.flush()
  792. os.fsync(handle.fileno())
  793. os.replace(tmp, path)
  794. except BaseException:
  795. tmp.unlink(missing_ok=True)
  796. raise
  797. def restore_interrupted_pull(project_root: Path | None = None, *, after_failure: bool = False) -> bool:
  798. """Put back the files a killed ``hermes update`` had half-moved to the new commit.
  799. ``after_failure``: the updater itself calls this when git exited non-zero mid-move (a locked or
  800. read-only file): same restore, and the marker stays whenever the tree is not verified whole.
  801. Returns True when the tree changed under this process: modules it already imported may be the
  802. half-written ones, so the caller must relaunch (``relaunch_after_restore``).
  803. Fast path (no marker) is one or two ``stat`` calls. Acts only when the marker's owner is gone,
  804. HEAD is still the pre-pull commit and no merge/rebase is in progress; then every path git wrote
  805. (the target's content, or torn on the way there) returns to HEAD (the commit the venv was built
  806. for), so the install is whole again and ``hermes update`` redoes the update from the start. Local
  807. edits are never touched; the updater's autostash (if any) stays in ``git stash list``. Concurrent
  808. launches take turns (``_restore_claim``); a launch that waited out another's restore relaunches.
  809. A torn ``hermes_bootstrap.py``, ``hermes_cli/__init__.py`` or recovery/lock/custody module fails
  810. before (or inside) the checkout's copy of this code: a minted launcher then runs the copy the
  811. updater published beside the marker (``RECOVERY_CLOSURE``). Limits, by design: other entry points
  812. (``python -m``, the ``hermes-agent`` hook's ``agent/__init__.py``) have no such fallback, and a
  813. launcher minted before that fallback existed has none either. A file git also changes that the user deleted, emptied or cut to a prefix of git's version
  814. looks exactly like git's own half-written file and is restored too, as is a user edit to a
  815. conflicted path or, on git < 2.38, to a path both sides of a custom-branch merge changed.
  816. """
  817. try:
  818. root = _project_root() if project_root is None else project_root
  819. marker = interrupted_pull_marker(root)
  820. if not marker.is_file() and not (Path(root) / ZIP_SWAP_JOURNAL).is_file():
  821. return False # fast path: nothing to repair, no lock taken
  822. if _pytest_owns_live_checkout(root):
  823. return False
  824. # A live `hermes update` (or its build/completion/git, after its updater died) owns the
  825. # checkout: its own transaction settles the tree, and repairing under it races its git.
  826. busy_note = "⚠ Not repairing the checkout now: {}. Launch again once it finishes."
  827. if (Path(root) / ZIP_SWAP_JOURNAL).is_file():
  828. with _checkout_custody(Path(root)) as busy:
  829. if busy:
  830. print(busy_note.format(busy), file=sys.stderr)
  831. return False
  832. from hermes_cli._early_recovery_zip import restore_interrupted_zip_swap
  833. if restore_interrupted_zip_swap(root):
  834. return True
  835. if not marker.is_file():
  836. return False
  837. with _restore_claim(marker.parent) as claimed:
  838. if not claimed:
  839. if after_failure:
  840. return False # the marker stays: the updater reports the move as not settled
  841. # The tree is still torn (the marker is there): importing checkout code now would run
  842. # the half-written files. Fail closed like unprovable custody does, as one line and
  843. # exit 1 (SystemExit's message), never a traceback (F20).
  844. raise SystemExit("hermes: another Hermes launch is finishing an interrupted `hermes update`; "
  845. "launch again in a moment.")
  846. if not marker.is_file():
  847. return True # another launch finished while this one started: rerun from its tree
  848. # The claim orders launches; the checkout lock keeps out an update tree. Claim first, so a
  849. # launch that waited out another's repair reruns from its tree without contending.
  850. with _checkout_custody(Path(root)) as busy:
  851. if busy:
  852. print(busy_note.format(busy), file=sys.stderr)
  853. return False
  854. return _restore_holding_claim(root, marker, after_failure=after_failure)
  855. except (OSError, subprocess.SubprocessError, ValueError) as exc:
  856. # Never block launch: the import that follows surfaces any real breakage.
  857. print(f"⚠ Could not check for an interrupted `hermes update`: {exc}", file=sys.stderr)
  858. return False
  859. @contextlib.contextmanager
  860. def _checkout_custody(root: Path):
  861. """Hold the checkout kernel lock (``hermes_cli.update_lock``) for a repair, like an updater (R2).
  862. Yields ``""`` while this process holds or joined it (the updater's own ``after_failure``
  863. settle and its children join), else the busy reason: a live update tree owns the checkout and
  864. the repair leaves the tree alone. The restore's mutating git children inherit the lock (POSIX)
  865. or join this process's kill-on-close job (Windows) through ``update_custody.run_git``, so a
  866. launch killed mid-restore keeps the checkout locked until its git is gone.
  867. A torn lock module cannot prove custody: an already-running updater retains its imported
  868. copy and may still be writing this checkout. Refuse launch rather than race that writer.
  869. """
  870. try:
  871. from hermes_cli import update_lock
  872. except Exception as exc:
  873. raise RuntimeError(
  874. "Cannot safely repair the interrupted update: checkout custody is unavailable "
  875. f"({type(exc).__name__}: {exc}). The recovery marker was kept. "
  876. "Wait for any running update to finish; if this persists, repair the checkout "
  877. "before launching Hermes again."
  878. ) from exc
  879. holder = update_lock._acquire_checkout(Path(root))
  880. if holder is not None and not update_lock.checkout_lock_held(Path(root)):
  881. # Refused, then free by the probe: its holder just exited. Take it now rather than run the
  882. # repair unguarded in that window (m10).
  883. holder = update_lock._acquire_checkout(Path(root))
  884. if holder is not None:
  885. if not update_lock.checkout_lock_held(Path(root)):
  886. # Not a holder but no lock at all (a git dir without working locks, e.g. NFS without
  887. # lockd): no updater can hold it either (they fail closed), so the repair runs unguarded.
  888. yield ""
  889. return
  890. yield update_lock.describe_holder(holder) or "another update holds the checkout lock"
  891. return
  892. try:
  893. yield ""
  894. finally:
  895. update_lock._release_checkout()
  896. def _claim_owner_alive(fields: dict[str, str], marker: Path) -> bool:
  897. """The updater that wrote ``marker`` is still running and the marker is fresh: hands off."""
  898. try:
  899. owner = int(fields.get("pid", ""))
  900. except ValueError:
  901. owner = -1
  902. # Our own pid is never the owner: this runs at startup, and containers hand a retry the
  903. # killed updater's pid.
  904. return (owner != os.getpid() and _pid_is_running(owner)
  905. and time.time() - marker.stat().st_mtime < _INTERRUPTED_PULL_MAX_AGE_SECONDS)
  906. def _resume_killed_rollback(git, fields: dict[str, str], git_dir: Path, root: Path,
  907. pre: str, rollback: str, *, after_failure: bool) -> str | None:
  908. """Redo a killed syntax rollback's HEAD move to ``pre``; the new HEAD, or None (marker kept)."""
  909. # Only the ref the rollback left HEAD on may be rewound: a branch the user checked out at
  910. # ``target`` since is theirs (and the update's branch would stay on the broken commit).
  911. named = git("symbolic-ref", "-q", "HEAD")
  912. if "ref" not in fields or (named.stdout.strip() if named.returncode == 0 else "") != fields["ref"].strip():
  913. # The marker stays: switching back lets the next launch resume. Never a reset recipe (F73):
  914. # a reset --hard would wipe whatever the user changed on the branch they checked out.
  915. ref = fields.get("ref", "").strip().removeprefix("refs/heads/")
  916. back = (f"`git -C {root} checkout {ref}`" if ref
  917. else f"`git -C {root} checkout --detach {fields.get('target', '').strip()[:10] or pre[:10]}`")
  918. print(f"⚠ An interrupted `hermes update` rollback to {pre[:10]} was not resumed: HEAD now names "
  919. f"another branch. Switch back with {back}, then launch `hermes` again to finish it.",
  920. file=sys.stderr)
  921. return None
  922. # A syntax rollback killed before it moved HEAD back: redo that step (HEAD and index, no file),
  923. # so the restore lands on ``pre`` (the code the update started from), not the broken tree.
  924. # Every step is checked, and the killed step's ``index.lock`` goes first, only once its git is
  925. # proven gone: until HEAD is on ``pre`` this marker is the rollback's only record.
  926. reason = _redo_rollback_head(git, git_dir, root, pre, rollback, after_failure=after_failure)
  927. if reason is not None:
  928. print(f"⚠ An interrupted `hermes update` rollback to {pre[:10]} cannot resume yet ({reason}); "
  929. "the next launch retries.", file=sys.stderr)
  930. return None
  931. return _read_head(git)
  932. def _advise_killed_merge(git_dir: Path, root: Path, target: str, stash: str) -> None:
  933. """Once per process, tell the user to abort the killed updater's own unfinished merge."""
  934. global _merge_advice_shown
  935. merge_head = git_dir / "MERGE_HEAD"
  936. if (not _merge_advice_shown and merge_head.is_file()
  937. and merge_head.read_text(encoding="utf-8-sig").strip() == target):
  938. # The killed updater's own merge: its conflict markers may sit in startup modules.
  939. _merge_advice_shown = True
  940. print(f"⚠ A killed `hermes update` left its merge unfinished. Run `git -C {root} merge --abort`, "
  941. "then launch again." + (f" Your local changes are in its stash ({stash})." if stash else ""),
  942. file=sys.stderr)
  943. def _custody_git(root: Path, recorded: str):
  944. """The restore's ``git(*args)``: the binary the updater recorded, every run under custody (``run_git``)."""
  945. executable = _git_executable(recorded)
  946. try:
  947. from hermes_cli.update_custody import run_git
  948. except Exception as exc:
  949. raise RuntimeError(
  950. "Cannot safely repair the interrupted update: child custody is unavailable "
  951. f"({type(exc).__name__}: {exc}). The recovery marker was kept. "
  952. "Repair the checkout before launching Hermes again."
  953. ) from exc
  954. def git(*args: str, stdin: str | None = None, text: bool = True) -> subprocess.CompletedProcess:
  955. base = [executable, "--literal-pathspecs", "-C", str(root)]
  956. kwargs = dict(input=stdin, cwd=str(root), capture_output=True, timeout=120,
  957. stdin=None if stdin is not None else subprocess.DEVNULL,
  958. **({"text": True, "encoding": "utf-8", "errors": "replace"} if text else {}))
  959. return run_git(base, list(args), **kwargs)
  960. return git
  961. def _judge_index_lock(fields: dict[str, str], marker: Path, root: Path, *,
  962. after_failure: bool) -> tuple[bool, bool, bool]:
  963. """``(foreign_lock, known_foreign, index_free)`` for the git dir's ``index.lock``, before any git runs.
  964. A killed git's lock goes first (an unresolvable or failing git would otherwise strand it, and it
  965. refuses every later git command). Proven-dead only; after a git that EXITED (``after_failure``) a
  966. lock now is another git's, never ours to drop."""
  967. git_dir = marker.parent
  968. lock = git_dir / "index.lock"
  969. foreign_lock = after_failure and lock.exists()
  970. if foreign_lock and _lock_predates_move(fields, lock):
  971. # Only the generation that was there before our git ran is another git's for sure. A lock
  972. # that appeared during the move can be our own SIGKILLed git's: never judged here (this
  973. # process cannot tell), but a later launch reclaims it once no git holds it (C15).
  974. _remember_foreign_lock(marker, lock)
  975. # The lock generation judged foreign when our git exited is never a killed git's to reclaim later:
  976. # a live `git commit` in the editor holds it with its fd closed, which only Linux can still see.
  977. known_foreign = bool(fields.get("foreign_lock", "").strip()) and \
  978. fields["foreign_lock"].strip() == _lock_identity(lock)
  979. index_free = foreign_lock or (not known_foreign and _release_dead_lock(lock, root))
  980. return foreign_lock, known_foreign, index_free
  981. def _read_head(git) -> str | None:
  982. # Only a full object name is an answer: an empty or garbled one keeps the marker (m5).
  983. head = git("rev-parse", "HEAD")
  984. oid = head.stdout.strip() if head.returncode == 0 else ""
  985. if _OID.fullmatch(oid):
  986. return oid
  987. detail = head.stderr.strip() or f"git printed {oid!r}, exit {head.returncode}"
  988. print(f"⚠ Could not read HEAD to repair an interrupted `hermes update` ({detail}); "
  989. "the next launch retries.", file=sys.stderr)
  990. return None
  991. def _retire_unattributable(git, git_dir: Path, root: Path, marker: Path, pre: str, target: str) -> None:
  992. """``target`` is gone (a gc or re-clone): nothing left to compare the tree against.
  993. Unattributable bytes are not proof of a whole tree: only a clean tracked tree at ``pre`` retires
  994. the record, rollback or not (review G2)."""
  995. if not _rollback_verified(git, git_dir, pre):
  996. print(f"⚠ An interrupted `hermes update` left tracked files that differ from {pre[:10]} and commit "
  997. f"{target[:10]} is gone; the marker was kept. Inspect `git -C {root} status`, then "
  998. f"`git -C {root} checkout {pre[:10]} -- <file>` for each file the update wrote.", file=sys.stderr)
  999. return
  1000. marker.unlink()
  1001. print(f"⚠ Ignoring a stale interrupted-update marker: commit {target[:10]} is gone.", file=sys.stderr)
  1002. def _restore_holding_claim(root: Path, marker: Path, *, after_failure: bool = False) -> bool:
  1003. git_dir = marker.parent
  1004. fields = dict(line.partition("=")[::2] for line in marker.read_text(encoding="utf-8-sig").splitlines())
  1005. if _claim_owner_alive(fields, marker):
  1006. return False
  1007. pre, target = fields.get("pre", "").strip(), fields.get("target", "").strip()
  1008. stash = fields.get("stash", "").strip()
  1009. git = _custody_git(root, fields.get("git", "").strip())
  1010. foreign_lock, known_foreign, index_free = _judge_index_lock(fields, marker, root, after_failure=after_failure)
  1011. rollback = fields.get("rollback", "").strip()
  1012. rollback = rollback if rollback in ("branch", "detach") else ""
  1013. head = _read_head(git)
  1014. if head is None:
  1015. return False
  1016. if rollback and pre and target and head == target:
  1017. head = _resume_killed_rollback(git, fields, git_dir, root, pre, rollback,
  1018. after_failure=after_failure or known_foreign)
  1019. if head is None:
  1020. return False
  1021. if not pre or not target or head != pre:
  1022. marker.unlink() # git finished (HEAD moved) or the marker is unusable
  1023. return False
  1024. if any((git_dir / name).exists() for name in _GIT_OPERATION_IN_PROGRESS):
  1025. _advise_killed_merge(git_dir, root, target, stash)
  1026. return False
  1027. # A killed claim holder's own git child can still be writing; scanning under it reads half a tree.
  1028. # After a git that EXITED (``after_failure``) no git of ours is left: a lock now is another git's
  1029. # (often the very reason ours failed), never ours to drop. The scan below only reads.
  1030. if not index_free:
  1031. print(f"⚠ {_index_lock_holder(git_dir, root)} holds the index after an interrupted `hermes update`; "
  1032. "the next launch finishes the restore.", file=sys.stderr)
  1033. return False
  1034. written = _paths_git_wrote(git, root, pre, target)
  1035. if written is None:
  1036. _retire_unattributable(git, git_dir, root, marker, pre, target)
  1037. return False
  1038. return _put_back_written(git, root, marker, written, pre, stash, rollback,
  1039. foreign_lock=foreign_lock, after_failure=after_failure)
  1040. def _put_back_written(git, root: Path, marker: Path, written, pre: str, stash: str, rollback: str, *,
  1041. foreign_lock: bool, after_failure: bool) -> bool:
  1042. """Return every path git wrote (``_paths_git_wrote``) to ``pre`` and retire the marker once verified."""
  1043. restore, added, new_dirs, kept = written
  1044. if (restore or added) and foreign_lock:
  1045. print("⚠ Another git holds the index, so the files the update already wrote cannot be put back "
  1046. "now; the next launch restores them.", file=sys.stderr)
  1047. return False
  1048. if restore or added:
  1049. print(("⚠ git stopped partway through writing the new code — " if after_failure else
  1050. "⚠ A previous `hermes update` was killed while git was writing the new code — ")
  1051. + f"restoring the checkout to {pre[:10]}...", file=sys.stderr)
  1052. failed = _put_back_paths(git, root, restore, added, kept)
  1053. if failed:
  1054. # No manual recipe: a reset would also wipe the edits this restore keeps, and the
  1055. # marker stays so the next launch retries.
  1056. reason = (failed.stderr.strip().splitlines() or ["git failed"])[-1]
  1057. print(f" ✗ Could not restore it automatically ({reason}); the next launch retries.",
  1058. file=sys.stderr)
  1059. return False
  1060. for rel in sorted(new_dirs, key=lambda d: d.count("/"), reverse=True):
  1061. with contextlib.suppress(OSError):
  1062. (root / rel).rmdir() # only when empty: an untracked file inside keeps it
  1063. if rollback and not _rollback_verified(git, marker.parent, pre, owned=set(restore)):
  1064. # Only HEAD on ``pre`` with every path the update wrote back at ``pre`` is a finished rollback;
  1065. # anything less keeps its only record. Other tracked edits (made after the kill, or autocrlf /
  1066. # filemode noise) are not the rollback's to judge, and no reset is advised: it would wipe them.
  1067. print(f" ✗ The rollback to {pre[:10]} is not verified yet (files the update wrote still differ); "
  1068. f"the next launch retries. Inspect `git -C {root} status`.", file=sys.stderr)
  1069. return bool(restore or added)
  1070. marker.unlink()
  1071. if not restore and not added:
  1072. return False # the killed git never reached the tree: nothing to put back
  1073. print(" ✓ Checkout restored; `hermes update` updates it again.", file=sys.stderr)
  1074. if stash:
  1075. print(f" Your local changes are still in the update's stash ({stash}).", file=sys.stderr)
  1076. return True
  1077. def _lock_identity(lock: Path) -> str:
  1078. """``inode:mtime_ns`` of a lock file, "" when there is none: a recreated lock is a new generation."""
  1079. try:
  1080. st = lock.stat()
  1081. except OSError:
  1082. return ""
  1083. return f"{st.st_ino}:{st.st_mtime_ns}"
  1084. def _lock_predates_move(fields: dict[str, str], lock: Path) -> bool:
  1085. """``lock`` is the generation ``arm_tree_move`` saw before the move's git ran. A marker without
  1086. the record (an older updater's, hand-written) cannot say: every lock counts as foreign."""
  1087. if "index_lock" not in fields:
  1088. return True
  1089. before = fields["index_lock"].strip()
  1090. return bool(before) and before == _lock_identity(lock)
  1091. def _remember_foreign_lock(marker: Path, lock: Path) -> None:
  1092. """Record ``lock``'s identity in the marker (atomically: the marker is the restore's only record)."""
  1093. identity = _lock_identity(lock)
  1094. if not identity:
  1095. return
  1096. lines = [line for line in marker.read_text(encoding="utf-8-sig").splitlines()
  1097. if not line.startswith("foreign_lock=")]
  1098. write_durable_text(marker, "\n".join([*lines, f"foreign_lock={identity}"]) + "\n")
  1099. def _redo_rollback_head(git, git_dir: Path, root: Path, pre: str, mode: str, *, after_failure: bool) -> str | None:
  1100. """Move HEAD (and the index) back to ``pre`` for a killed syntax rollback; None, or why not yet.
  1101. The killed step's ``index.lock`` is reclaimed first, and only when its git is proven gone. After a
  1102. git that exited in this very process (``after_failure``) a lock is never ours to judge.
  1103. """
  1104. lock = git_dir / "index.lock"
  1105. if lock.exists() and (after_failure or not _release_dead_lock(lock, root)):
  1106. return f"{_index_lock_holder(git_dir, root)} may still hold .git/index.lock"
  1107. if mode == "detach":
  1108. moved = git("update-ref", "--no-deref", "HEAD", pre)
  1109. if moved.returncode != 0:
  1110. return f"git update-ref failed: {(moved.stderr.strip().splitlines() or ['?'])[-1]}"
  1111. reset = git("reset", "-q", pre)
  1112. if reset.returncode != 0:
  1113. return f"git reset failed: {(reset.stderr.strip().splitlines() or ['?'])[-1]}"
  1114. head = git("rev-parse", "HEAD")
  1115. if head.returncode != 0 or head.stdout.strip() != pre:
  1116. return "HEAD did not move back"
  1117. return None
  1118. def _rollback_verified(git, git_dir: Path, pre: str, owned: set[str] | None = None) -> bool:
  1119. """The rollback landed: HEAD is ``pre``, no index lock, and no tracked change among ``owned`` (the
  1120. paths the update wrote), or anywhere when they are unknown (``target`` gone)."""
  1121. head = git("rev-parse", "HEAD")
  1122. if head.returncode != 0 or head.stdout.strip() != pre or (git_dir / "index.lock").exists():
  1123. return False
  1124. status = git("status", "--porcelain", "-z", "--untracked-files=no", "--no-renames")
  1125. changed = {entry[3:] for entry in status.stdout.split("\0") if entry}
  1126. return status.returncode == 0 and not (changed if owned is None else changed & owned)
  1127. def _put_back_paths(git, root: Path, restore: list[str], added: list[str],
  1128. kept: set[str]) -> subprocess.CompletedProcess | None:
  1129. """Return ``restore`` to HEAD and drop ``added``, renaming those in ``kept`` aside; the failed git run, or None."""
  1130. if restore:
  1131. run = git("restore", "--source=HEAD", "--staged", "--worktree", "--pathspec-from-file=-",
  1132. "--pathspec-file-nul", stdin="\0".join(restore))
  1133. if run.returncode:
  1134. return run
  1135. if added:
  1136. run = git("rm", "-q", "--cached", "--ignore-unmatch", "--pathspec-from-file=-",
  1137. "--pathspec-file-nul", stdin="\0".join(added))
  1138. for rel in added:
  1139. if rel in kept:
  1140. print(f" Kept {rel} as {_keep_aside(root / rel).name}: it may be your own file.", file=sys.stderr)
  1141. else:
  1142. (root / rel).unlink(missing_ok=True)
  1143. if run.returncode:
  1144. return run
  1145. return None
  1146. def _keep_aside(file: Path) -> Path:
  1147. """Rename ``file`` to a free ``<name>.hermes-update-kept[-N]`` beside it (never importable, never clobbered)."""
  1148. n = 1
  1149. while os.path.lexists(aside := file.with_name(f"{file.name}.hermes-update-kept" + (f"-{n}" if n > 1 else ""))):
  1150. n += 1
  1151. os.rename(file, aside)
  1152. return aside
  1153. def relaunch_after_restore() -> None:
  1154. """Re-run this command from the restored tree; never returns.
  1155. Everything imported so far (this package, ``hermes_bootstrap``, ``hermes_cli.main`` itself) may
  1156. be the killed git's new files, and they would run against the restored old tree.
  1157. """
  1158. argv = [sys.executable, *sys.orig_argv[1:]]
  1159. sys.stdout.flush()
  1160. sys.stderr.flush()
  1161. if sys.platform == "win32":
  1162. # os.execv on Windows spawns and exits, detaching the console's wait on us.
  1163. sys.exit(subprocess.call(argv)) # windows-footgun: ok — interactive child keeps our console
  1164. os.execv(sys.executable, argv)
  1165. def _pytest_owns_live_checkout(root: Path) -> bool:
  1166. """Keep lifecycle tests from repairing the checkout that runs the suite.
  1167. Copied installations in tmp_path remain eligible for real recovery tests.
  1168. """
  1169. return "PYTEST_CURRENT_TEST" in os.environ and root == Path(__file__).resolve().parent.parent
  1170. def _missing_environment(root: Path) -> bool:
  1171. """True when PM recorded an environment whose site-packages is gone."""
  1172. from pm.environments import runtime_facts_path, selected_venv, site_packages
  1173. if not runtime_facts_path(root).is_file():
  1174. return False
  1175. try:
  1176. return not site_packages(selected_venv(root)).is_dir()
  1177. except RuntimeError:
  1178. return True # PM validates the recorded state before rebuilding.
  1179. def _count_failed_attempt(marker: Path) -> None:
  1180. """Bump ``marker``'s attempt count in its own format so the retry limit can trip."""
  1181. import json
  1182. # Best-effort: an unwritable marker only means the limit trips later.
  1183. with contextlib.suppress(OSError):
  1184. attempts = _read_marker_attempts(marker) + 1
  1185. # Our own record: undecodable bytes must not raise UnicodeDecodeError on every launch.
  1186. body = marker.read_text(encoding="utf-8-sig", errors="replace")
  1187. if any(line.startswith("pid=") for line in body.splitlines()):
  1188. lines = [line for line in body.splitlines() if not line.startswith("attempts=")]
  1189. body = "\n".join([*lines, f"attempts={attempts}"]) + "\n"
  1190. else:
  1191. body = json.dumps({"attempts": attempts})
  1192. marker.write_text(body, encoding="utf-8")
  1193. def recover_if_needed(project_root: Path | None = None, argv: list[str] | None = None, *, explicit: bool = False) -> bool:
  1194. """Ask PM to restore dependencies before activation; leave failed requests retryable."""
  1195. global _UPDATE_RETRY_RECOVERED
  1196. root = _project_root() if project_root is None else Path(project_root).resolve()
  1197. if not explicit and _pytest_owns_live_checkout(root):
  1198. return False
  1199. from hermes_cli._parser import command_argv
  1200. args = command_argv(sys.argv[1:] if argv is None else argv)
  1201. if not explicit and args[:1] == ["pm"]:
  1202. return False # PM's command boundary owns the explicit repair.
  1203. from pm.environments import install_state_dir
  1204. missing_marker = install_state_dir(root) / ".repair-incomplete"
  1205. marker_paths = (root / ".update-incomplete", root / ".lazy-refresh-incomplete", missing_marker)
  1206. markers = [path for path in marker_paths if path.is_file()]
  1207. if not (root / "pyproject.toml").is_file() or not (explicit or markers or _missing_environment(root)):
  1208. return False
  1209. lock = _claim_recovery_lock(root)
  1210. if lock is None:
  1211. return False
  1212. try:
  1213. # Recheck after locking: another launch can finish between discovery and claim.
  1214. markers = [path for path in marker_paths if path.is_file()]
  1215. if not markers:
  1216. if not explicit and not _missing_environment(root):
  1217. return False
  1218. missing_marker.write_text('{"attempts": 0}', encoding="utf-8")
  1219. markers = [missing_marker]
  1220. if any(_marker_owner_is_live(marker) for marker in markers):
  1221. return False
  1222. if not explicit and any(_read_marker_attempts(marker) >= _EARLY_CORE_INSTALL_MAX_ATTEMPTS for marker in markers):
  1223. print("hermes: automatic dependency repair retry limit reached; run `hermes pm repair`", file=sys.stderr)
  1224. return False
  1225. from pm.recovery import repair_dependencies
  1226. print("hermes: repairing the recorded dependency environment...", file=sys.stderr)
  1227. repair_dependencies(root)
  1228. for marker in markers:
  1229. marker.unlink(missing_ok=True)
  1230. _UPDATE_RETRY_RECOVERED = args[:1] == ["update"]
  1231. print("hermes: dependency environment repaired", file=sys.stderr)
  1232. return True
  1233. except Exception as exc:
  1234. from pm.environments import install_state_permission_message
  1235. if isinstance(exc, PermissionError) and install_state_permission_message(root, exc):
  1236. raise # The bootstrap or PM CLI reports the access error once.
  1237. for marker in markers:
  1238. _count_failed_attempt(marker)
  1239. print(f"hermes: dependency repair failed: {exc}; run `hermes pm repair`", file=sys.stderr)
  1240. return False
  1241. finally:
  1242. os.close(lock)
  1243. # A failed network or build must not retry on every launch forever.
  1244. _EARLY_CORE_INSTALL_MAX_ATTEMPTS = 3
  1245. def _claim_recovery_lock(root: Path) -> int | None:
  1246. """Hold a kernel lock in writable state; process exit releases it."""
  1247. from pm.environments import install_state_dir
  1248. from hermes_cli.runtime_state import _lock
  1249. state = install_state_dir(root)
  1250. state.mkdir(parents=True, exist_ok=True)
  1251. fd = os.open(state / ".recovery.lock", os.O_CREAT | os.O_RDWR, 0o600)
  1252. try:
  1253. if _lock(fd, wait=False):
  1254. return fd
  1255. except BaseException:
  1256. os.close(fd)
  1257. raise
  1258. os.close(fd)
  1259. return None