update_custody.py 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835
  1. """Checkout custody for the updater's children (R2): the ONE spawn policy for every git (and the
  2. Node build) an update starts.
  3. The checkout kernel lock (``update_lock``) must stay held while any process of the update tree
  4. can still write the checkout, and must NOT leak into processes that outlive the update:
  5. * Every updater git call carries ``-c gc.autoDetach=false -c maintenance.auto=false``: git never
  6. forks a detached gc/maintenance child that would inherit (POSIX) or outlive (Windows) the lock.
  7. * POSIX: the lock fd is inherited ONLY by git commands that mutate the worktree, index or refs
  8. locally (:data:`LOCAL_MUTATORS`, run with ``core.fsmonitor=false`` so no fsmonitor daemon
  9. starts under them, with no credential helper and no repository hooks). Network/credential commands (fetch,
  10. ls-remote, credential) and readers run without it: a ``git credential-cache--daemon`` they
  11. start never holds the checkout. A partial clone's mutator would lazily fetch the objects a move
  12. needs as a child holding the fd (and start the daemon under it), so before a move the objects
  13. are fetched without the fd and with the user's helpers (:func:`_prefetch_for_move`); a lazy
  14. fetch the prefetch missed still runs, just without a helper. A killed fetch that leaves
  15. ``*.lock`` ref files or ``shallow.lock`` is recovered by the stale-lock rules (``gitlock.release_dead_fetch_locks``
  16. once its update is proven dead, ``gitlock.clear_stale_git_locks`` by age otherwise).
  17. * Windows has no fd inheritance: while this process holds (or joined) the checkout lock, every
  18. child started here is created SUSPENDED, assigned to the update's kill-on-close job and only
  19. then resumed, so it and everything it spawns die with the lock owner. The Node build goes
  20. through ``pm.progress.run_contained`` (whose Popen it never sees): :func:`contained_command`
  21. wraps it in a stdlib launcher that joins the job before it starts node and, once node exits,
  22. terminates and waits out everything node left running (C3): a normal release never frees the
  23. checkout under a build descendant.
  24. * Windows fails CLOSED (D2): a child the job refuses is never run. The msvcrt byte lock belongs
  25. to the process that took it — an inherited handle does not keep it, the kernel unlocks it when
  26. the owner exits — so nothing else could fence a writer outside the job: after the owner's
  27. death the lock would be free while it still wrote. A refused bind kills the still-suspended
  28. child; a refused launcher join exits before it starts node; either raises
  29. :class:`CustodyRefused`, a clear refusal of that step instead of an unfenced writer.
  30. Every updater git runner (``update_cmd._git_run``, ``update_cmd_git._git_run``,
  31. ``update_cmd_stash``, ``update_cmd_check``, ``gitlock``, ``update_cmd_commit``,
  32. ``_early_recovery``'s restore) calls :func:`run_git`.
  33. Not covered, on purpose: the read-only release/check readers (``source_releases``,
  34. ``source_check``: ``ls-remote``, ``cherry``, ``rev-parse``) run their own probes; they never get
  35. the lock fd and write nothing, so they cannot leak or outlive custody of the checkout.
  36. """
  37. from __future__ import annotations
  38. import contextlib
  39. import logging
  40. import os
  41. import subprocess
  42. import sys
  43. from collections.abc import Sequence
  44. import itertools
  45. logger = logging.getLogger(__name__)
  46. # No detached child: `git gc --auto` / `maintenance run --auto --detach` would otherwise fork a
  47. # daemonized repack after any command that writes objects (commit, merge, fetch, stash).
  48. GIT_NO_DETACH = ("-c", "gc.autoDetach=false", "-c", "maintenance.auto=false")
  49. # Local mutators only (they hold the lock fd): never start an fsmonitor daemon under it, and no
  50. # credential helper — a promisor lazy fetch under a mutator would start `git
  51. # credential-cache--daemon` holding the fd for its lifetime (900 s by default, m3). No repository
  52. # hooks either (F1): a hook inherits the fd, and one that backgrounds a process (a post-merge
  53. # daemon) kept a completed update's checkout locked until it exited. Per command only: the
  54. # repository's own hook configuration is untouched.
  55. _MUTATOR_CONFIG = ("-c", "core.fsmonitor=false", "-c", "credential.helper=", "-c", f"core.hooksPath={os.devnull}")
  56. # git subcommands that write the worktree, the index or refs on THIS machine. Only these inherit
  57. # the checkout lock fd: if the updater dies mid-command, the checkout stays locked until git exits.
  58. # No `pull` (fetch + merge: its network half must not hold the fd; the updater fetches, then
  59. # merges). `gc` packs refs (and its repack writes the object store); `pack-objects` is the pack
  60. # tidy's merge, whose output the update moves into the object store.
  61. LOCAL_MUTATORS = frozenset({
  62. "add", "am", "apply", "checkout", "checkout-index", "cherry-pick", "clean", "commit", "gc",
  63. "merge", "mv", "pack-objects", "read-tree", "rebase", "reset", "restore", "revert", "rm", "stash",
  64. "switch", "update-index", "update-ref", "symbolic-ref", "tag", "branch", "worktree",
  65. })
  66. # Mutators that move the checkout to another commit: what they read from a promisor remote is
  67. # fetched first, without the fd (_prefetch_for_move).
  68. _MOVES = frozenset({"checkout", "merge", "read-tree", "reset", "switch"})
  69. # Global options before the subcommand that take a separate value argument.
  70. _GLOBAL_WITH_VALUE = frozenset({"-c", "-C", "--git-dir", "--work-tree", "--namespace", "--exec-path",
  71. "--config-env", "--super-prefix", "--attr-source", "--list-cmds"})
  72. _CREATE_SUSPENDED = 0x00000004
  73. class CustodyRefused(OSError):
  74. """Windows would not put an update child in the update's kill-on-close job, so the child was
  75. never run (D2): outside the job it could outlive a killed update and write the checkout after
  76. the checkout lock is free."""
  77. def __init__(self, argv: Sequence[str], cause: object) -> None:
  78. import os
  79. name = os.path.basename(str(argv[0])) if argv else "child"
  80. self.reason = (
  81. f"Windows would not put `{name}` in this update's process job ({cause}), so it was not "
  82. "run: a checkout writer outside the job could outlive a killed update and write the "
  83. "checkout after its lock is released.")
  84. super().__init__(f"{self.reason} {_RETRY.format(command='the command')}")
  85. _RETRY = ("Run {command} again from a regular terminal, outside any sandbox or tool that confines "
  86. "the processes it starts.")
  87. # This run's custody (m2): whether an update child already ran in custody, and the first refusal.
  88. # Many readers swallow an OSError (a reader's None, a best-effort probe), so a refusal can end
  89. # the update as a misleading downstream error; the command's failure path prints
  90. # :func:`refusal_notice` instead.
  91. _RUN: dict = {"ran": False, "refused": None}
  92. def _refuse(argv: Sequence[str], cause: object, *, receipted: bool = False) -> CustodyRefused:
  93. """The :class:`CustodyRefused` for ``argv``: logged, noted in the receipt (unless the caller
  94. already did) and kept as this run's first refusal."""
  95. argv = list(argv) if isinstance(argv, (list, tuple)) else [argv]
  96. exc = CustodyRefused(argv, cause)
  97. detail = f"{exc.reason}: {' '.join(str(arg) for arg in argv[:2])}"
  98. if not receipted:
  99. logger.warning("Refused update child: %s", detail)
  100. with contextlib.suppress(Exception):
  101. from hermes_cli.update_receipt import record_step
  102. record_step("update_custody", False, detail)
  103. if _RUN["refused"] is None:
  104. _RUN["refused"] = (exc, not _RUN["ran"] and not _past_commit())
  105. return exc
  106. def _past_commit() -> bool:
  107. """The update reached its commit point: its receipt records the ``apply`` stage (the
  108. completion child resumes that receipt). No receipt is open before an update begins, nor in
  109. the parent after its completion child returned, which starts no update child."""
  110. try:
  111. from hermes_cli.update_receipt import _current
  112. current = _current.get()
  113. except Exception: # health: allow BLE001 -- fail closed: unknown = past commit, never "nothing changed"
  114. return True
  115. return current is not None and any(stage.get("name") == "apply" for stage in current.data.get("stages") or ())
  116. def refusal_notice(command: str = "hermes update") -> str | None:
  117. """What the command's failure path prints when this run refused an update child (m2), in
  118. place of whatever generic error the refusal turned into downstream; ``None`` otherwise."""
  119. if _RUN["refused"] is None:
  120. return None
  121. exc, unchanged = _RUN["refused"]
  122. outcome = ("Nothing was changed: no update step had run yet." if unchanged else
  123. "Update steps before it had already run.")
  124. return f"✗ `{command}` stopped: {exc.reason}\n {outcome}\n {_RETRY.format(command=f'`{command}`')}"
  125. def git_subcommand(args: Sequence[str]) -> str | None:
  126. """The subcommand in ``args`` (everything after the git executable), skipping global options."""
  127. it = iter(args)
  128. for arg in it:
  129. if arg in _GLOBAL_WITH_VALUE:
  130. next(it, None)
  131. elif not arg.startswith("-"):
  132. return arg
  133. return None
  134. def is_local_mutator(args: Sequence[str]) -> bool:
  135. return git_subcommand(args) in LOCAL_MUTATORS
  136. def git_argv(git_cmd: Sequence[str], args: Sequence[str]) -> list[str]:
  137. """``git_cmd + args`` with the custody config inserted before the subcommand."""
  138. git_cmd, args = list(git_cmd), list(args)
  139. extra = GIT_NO_DETACH + (_MUTATOR_CONFIG if is_local_mutator(git_cmd[1:] + args) else ())
  140. return [*git_cmd, *extra, *args]
  141. def _held() -> dict | None:
  142. try:
  143. from hermes_cli import update_lock
  144. except Exception: # health: allow BLE001 -- a torn tree's half-written module raises anything; no updater runs
  145. return None
  146. return update_lock._HELD
  147. def _death_signal_preexec():
  148. """Linux: a ``preexec_fn`` making the child die (SIGKILL) with the updater thread that waits
  149. on it. For the git children that do NOT hold the lock fd (fetch, ls-remote, readers): a
  150. killed owner must not leave a fetch rewriting refs under the next lock owner. The ruling's
  151. stale-lock rules recover a fetch killed mid-write. ``None`` elsewhere (Windows: the job;
  152. macOS has no parent-death signal: :func:`run_git` puts a fetch under :func:`_owner_watch`)."""
  153. if not sys.platform.startswith("linux"):
  154. return None
  155. import ctypes
  156. import os
  157. prctl = ctypes.CDLL(None, use_errno=True).prctl # resolved before fork: the child only calls
  158. parent = os.getpid()
  159. def _arm():
  160. prctl(1, 9, 0, 0, 0) # PR_SET_PDEATHSIG, SIGKILL
  161. if os.getppid() != parent: # the owner died between fork and prctl
  162. os._exit(137)
  163. return _arm
  164. def _custody_kwargs(inherit_lock: bool, kwargs: dict) -> dict:
  165. fds = _lock_fds(inherit_lock)
  166. if fds:
  167. kwargs["pass_fds"] = tuple(dict.fromkeys((*kwargs.get("pass_fds", ()), *fds)))
  168. elif not inherit_lock and _held() is not None and "preexec_fn" not in kwargs:
  169. arm = _death_signal_preexec()
  170. if arm is not None:
  171. kwargs["preexec_fn"] = arm
  172. return kwargs
  173. def _lock_fds(inherit_lock: bool) -> tuple[int, ...]:
  174. if not inherit_lock or sys.platform == "win32":
  175. return ()
  176. from hermes_cli.update_lock import custody_spawn_kwargs
  177. return tuple(custody_spawn_kwargs().get("pass_fds", ()))
  178. def _bind_suspended(proc: subprocess.Popen) -> None:
  179. """Assign a CREATE_SUSPENDED child to the update's kill-on-close job, then resume it.
  180. Suspended until bound, so nothing it spawns can escape the job. A refused bind kills the
  181. child before it ran a single instruction and raises :class:`CustodyRefused` (D2: never an
  182. unfenced writer); a failed resume kills the child."""
  183. from hermes_cli.update_lock import _bind_to_kill_on_close_job, resume_suspended_child
  184. try:
  185. _bind_to_kill_on_close_job(proc)
  186. except OSError as exc:
  187. proc.kill()
  188. proc.wait()
  189. raise _refuse(proc.args, exc) from exc
  190. resume_suspended_child(proc)
  191. _RUN["ran"] = True
  192. def popen(argv: Sequence[str], *, inherit_lock: bool = False, **kwargs) -> subprocess.Popen:
  193. """``subprocess.Popen`` in the update tree's custody (see the module doc)."""
  194. if sys.platform == "win32" and _held() is not None:
  195. kwargs["creationflags"] = kwargs.get("creationflags", 0) | _CREATE_SUSPENDED
  196. proc = subprocess.Popen(list(argv), **kwargs)
  197. try:
  198. _bind_suspended(proc)
  199. except BaseException:
  200. with contextlib.suppress(OSError):
  201. proc.kill()
  202. raise
  203. return proc
  204. return subprocess.Popen(list(argv), **_custody_kwargs(inherit_lock, kwargs))
  205. def run(argv: Sequence[str], *, inherit_lock: bool = False, **kwargs) -> subprocess.CompletedProcess:
  206. """``subprocess.run`` in the update tree's custody. Off Windows (or outside an update) it IS
  207. ``subprocess.run`` with the caller's kwargs untouched, plus the lock fd for mutators (or the
  208. parent-death signal for the rest); on Windows inside an update it is the same contract over
  209. :func:`popen`, so the child is job-bound before it runs."""
  210. if not (sys.platform == "win32" and _held() is not None):
  211. return subprocess.run(list(argv), **_custody_kwargs(inherit_lock, kwargs)) # noqa: PLW1510 -- forwarding wrapper: callers pass subprocess kwargs via **kwargs (check may arrive through it); an explicit check=False would raise TypeError
  212. input, timeout = kwargs.pop("input", None), kwargs.pop("timeout", None)
  213. check = kwargs.pop("check", False)
  214. if input is not None:
  215. kwargs["stdin"] = subprocess.PIPE
  216. if kwargs.pop("capture_output", False):
  217. kwargs["stdout"] = kwargs["stderr"] = subprocess.PIPE
  218. with popen(argv, **kwargs) as proc:
  219. try:
  220. stdout, stderr = proc.communicate(input, timeout=timeout)
  221. except subprocess.TimeoutExpired as exc:
  222. # git.exe's git-remote-https inherits the pipes: kill the tree, not git.exe alone (and
  223. # never the job: it also holds the update's other children).
  224. from hermes_cli._subprocess_compat import kill_and_drain
  225. drained = kill_and_drain(proc, _DRAIN_SECONDS)
  226. if drained is not None:
  227. exc.stdout, exc.stderr = drained
  228. raise
  229. except BaseException:
  230. proc.kill()
  231. raise
  232. code = proc.poll()
  233. if check and code:
  234. raise subprocess.CalledProcessError(code, proc.args, output=stdout, stderr=stderr)
  235. return subprocess.CompletedProcess(proc.args, code, stdout, stderr)
  236. def popen_post_commit(argv: Sequence[str], *, label: str, **kwargs) -> subprocess.Popen:
  237. """``subprocess.Popen`` for a checkout writer that runs after the update committed (the
  238. historical takeover's ``update_finish``). POSIX: the lock fd. Windows inside an update:
  239. created suspended and bound to the update's kill-on-close job before it runs. A refused bind
  240. must not fail a committed update, so the child still runs, never silently (printed and
  241. receipted) and never unfenced: it joins the checkout lock holding its own lease (R5b)."""
  242. if not (sys.platform == "win32" and _held() is not None):
  243. return subprocess.Popen(list(argv), **_custody_kwargs(True, kwargs))
  244. from hermes_cli.update_lock import bind_child_to_update_tree, resume_suspended_child
  245. kwargs["creationflags"] = kwargs.get("creationflags", 0) | _CREATE_SUSPENDED
  246. proc = subprocess.Popen(list(argv), **kwargs)
  247. try:
  248. refusal = bind_child_to_update_tree(proc)
  249. if refusal is not None:
  250. detail = (f"the update's job would not take the {label} ({refusal}), so it runs outside "
  251. "the job, holding its own checkout lease")
  252. print(f" ⚠ {detail}", flush=True)
  253. with contextlib.suppress(Exception):
  254. from hermes_cli.update_receipt import record_step
  255. record_step("update_custody", False, detail)
  256. resume_suspended_child(proc)
  257. except BaseException:
  258. proc.kill()
  259. proc.wait()
  260. raise
  261. return proc
  262. # How long a timed-out child's output may still drain after its tree was killed.
  263. _DRAIN_SECONDS = 5
  264. def run_git(git_cmd: Sequence[str], args: Sequence[str], **kwargs) -> subprocess.CompletedProcess:
  265. """THE updater git runner: custody config in argv, the lock fd only into local mutators,
  266. Windows job binding inside an update. ``kwargs`` are ``subprocess.run``'s."""
  267. argv = git_argv(git_cmd, args)
  268. mutator = is_local_mutator(argv[1:])
  269. if mutator:
  270. _prefetch_for_move(list(git_cmd), list(args), kwargs)
  271. if sys.platform != "win32" and _held() is not None and "preexec_fn" not in kwargs \
  272. and _death_signal_preexec() is None:
  273. # No parent-death signal (macOS): the tree runner owns git AND the helpers it starts. A
  274. # local mutator holds the lock fd and finishes its write if the owner dies, as on Linux.
  275. return _run_owner_watched(argv, _custody_kwargs(mutator, kwargs), watch_owner=not mutator)
  276. return run(argv, inherit_lock=mutator, **kwargs)
  277. # git without the lock fd (a fetch's network half must not hold it, m3) dies with its owner: on
  278. # Linux by the parent-death signal; where there is none (macOS) by this watchdog (R3), or a fetch
  279. # would go on rewriting refs and leave live `*.lock` files under the next owner.
  280. # The watchdog: stdin is a pipe only the owner writes. A byte = the child finished; EOF with no
  281. # byte = the owner died (the kernel closed its end), so SIGKILL the child's whole process group
  282. # (git and the helpers it started: git-remote-https, index-pack, pack-objects), as
  283. # PR_SET_PDEATHSIG would for the child alone. It holds no lock fd and runs in its own session.
  284. _OWNER_WATCH = (
  285. "import os, signal, sys\n"
  286. "if not os.read(0, 1):\n"
  287. " try:\n"
  288. " os.killpg(int(sys.argv[1]), signal.SIGKILL)\n" # windows-footgun: ok -- POSIX only (run_git gates win32 out)
  289. " except OSError:\n"
  290. " pass\n"
  291. )
  292. def _owner_watch(pid: int):
  293. """Start the parent-death stand-in for child ``pid``; returns the callable that dismisses it
  294. once the child is done, or ``None`` when it could not start (logged: the fetch still runs, as
  295. it did before; only the owner's death mid-fetch is then unfenced)."""
  296. import os
  297. read_end, write_end = os.pipe() # non-inheritable: no other child keeps the owner's end open
  298. try:
  299. watcher = subprocess.Popen([sys.executable, "-I", "-S", "-c", _OWNER_WATCH, str(pid)], stdin=read_end,
  300. # detached-spawn: ok — the owner-death watchdog itself: it must outlive the owner to kill the fetch
  301. stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True)
  302. except OSError as exc:
  303. os.close(write_end)
  304. logger.warning("Could not watch update child %s for its owner's death: %s", pid, exc)
  305. return None
  306. finally:
  307. os.close(read_end)
  308. def dismiss() -> None:
  309. with contextlib.suppress(OSError): # the watcher already gone: nothing left to tell it
  310. os.write(write_end, b"x")
  311. os.close(write_end)
  312. with contextlib.suppress(subprocess.TimeoutExpired): # it exits on the byte; never block on it
  313. watcher.wait(timeout=5)
  314. return dismiss
  315. def _kill_tree(proc: subprocess.Popen) -> None:
  316. """SIGKILL ``proc``'s process group: the child leads it (``start_new_session``), and while it
  317. is unreaped its pid, hence the group id, cannot be reused. Once reaped (``communicate`` reaps
  318. on a Ctrl-C inside its wait window) the id may already name another session, so it is never
  319. signalled; ``returncode`` is read, not ``poll()``ed, since polling would reap a live zombie
  320. and skip the helpers a timed-out git leaves running."""
  321. import os
  322. import signal
  323. if proc.returncode is not None:
  324. return
  325. with contextlib.suppress(OSError): # already gone
  326. os.killpg(proc.pid, signal.SIGKILL) # windows-footgun: ok -- POSIX only (run_git gates win32 out)
  327. def _run_owner_watched(argv: Sequence[str], kwargs: dict, *, watch_owner: bool = True) -> subprocess.CompletedProcess:
  328. """``subprocess.run`` for updater git where no parent-death signal exists, under
  329. :func:`_owner_watch` while it runs. git leads a session of its own, so a timeout, a cancel or
  330. the owner's death ends its helpers with it, not git's pid alone (#144439)."""
  331. input, timeout = kwargs.pop("input", None), kwargs.pop("timeout", None)
  332. check = kwargs.pop("check", False)
  333. if input is not None:
  334. kwargs["stdin"] = subprocess.PIPE
  335. if kwargs.pop("capture_output", False):
  336. kwargs["stdout"] = kwargs["stderr"] = subprocess.PIPE
  337. # detached-spawn: ok — awaited; _owner_watch kills a fetch's group on owner death, and a mutator must finish its write
  338. with subprocess.Popen(list(argv), start_new_session=True, **kwargs) as proc:
  339. dismiss = _owner_watch(proc.pid) if watch_owner else None
  340. try:
  341. stdout, stderr = proc.communicate(input, timeout=timeout)
  342. except subprocess.TimeoutExpired as exc:
  343. _kill_tree(proc)
  344. try:
  345. exc.stdout, exc.stderr = proc.communicate(timeout=_DRAIN_SECONDS)
  346. except subprocess.TimeoutExpired:
  347. # A helper git started still holds the pipes: leave them to the reader threads.
  348. proc.stdin = proc.stdout = proc.stderr = None
  349. raise
  350. except BaseException:
  351. _kill_tree(proc)
  352. raise
  353. finally:
  354. if dismiss is not None:
  355. dismiss()
  356. code = proc.poll()
  357. if check and code:
  358. raise subprocess.CalledProcessError(code, proc.args, output=stdout, stderr=stderr)
  359. return subprocess.CompletedProcess(proc.args, code, stdout, stderr)
  360. def _partial_clone(git_cmd: Sequence[str], kwargs: dict) -> bool:
  361. """The repository has a promisor remote (``extensions.partialClone``), read from disk."""
  362. from pathlib import Path
  363. from hermes_cli.update_lock import _git_common_dir
  364. cwd = kwargs.get("cwd")
  365. for flag, value in itertools.pairwise(git_cmd):
  366. if flag == "-C":
  367. cwd = Path(cwd or ".") / value
  368. common = _git_common_dir(Path(cwd or "."))
  369. try:
  370. return common is not None and "partialclone" in \
  371. (common / "config").read_text(encoding="utf-8-sig", errors="replace").lower()
  372. except OSError:
  373. return False
  374. def _prefetch_for_move(git_cmd: list[str], args: list[str], kwargs: dict) -> None:
  375. """Partial clones: fetch what a move to another commit will read — the target's changed trees
  376. and blobs — WITHOUT the lock fd and with the user's credential helpers, so the mutator (which
  377. has neither helper nor reason to fetch) never starts a lazy fetch or a credential daemon under
  378. the fd (m3). ``git diff HEAD <target>`` reads exactly the trees and blobs that differ from the
  379. checked-out tree; the rest is local. Best effort: a miss only leaves a helper-less lazy fetch."""
  380. sub = git_subcommand(args)
  381. if sub not in _MOVES or not _partial_clone(git_cmd, kwargs):
  382. return
  383. common = {key: kwargs[key] for key in ("cwd", "env") if key in kwargs}
  384. def resolve(*query: str, stdin: str | None = None) -> str | None:
  385. out = run(git_argv(git_cmd, list(query)), capture_output=True, text=True, encoding="utf-8",
  386. errors="replace", input=stdin, **({} if stdin is not None else {"stdin": subprocess.DEVNULL}),
  387. **common)
  388. return out.stdout.strip() if out.returncode == 0 and out.stdout.strip() else None
  389. try:
  390. targets = []
  391. for arg in args[args.index(sub) + 1:]:
  392. if arg == "--":
  393. break
  394. if not arg.startswith("-"):
  395. targets.append(resolve("rev-parse", "--verify", "-q", f"{arg}^{{commit}}"))
  396. targets = [target for target in dict.fromkeys(targets) if target]
  397. if not targets:
  398. return
  399. # An unborn HEAD (a fresh clone) has nothing checked out: diff from the empty tree.
  400. base = resolve("rev-parse", "--verify", "-q", "HEAD^{commit}") \
  401. or resolve("hash-object", "-t", "tree", "--stdin", stdin="")
  402. for target in targets:
  403. if base and target != base:
  404. run(git_argv(git_cmd, ["diff", "--binary", "--no-ext-diff", "--no-textconv", base, target]),
  405. stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, **common)
  406. except (OSError, ValueError, subprocess.SubprocessError) as exc:
  407. logger.debug("Could not prefetch the objects for git %s: %s", sub, exc)
  408. # A stdlib launcher for children whose Popen the updater never sees (``run_contained``): it
  409. # joins the job named by an inherited handle, starts the real command suspended with the same
  410. # stdio, runs it only once it is in the job, and drops that handle (only the owner's handle may
  411. # keep the job open). A refused join, or a command outside the job, never runs (D2): the launcher says so on stderr, writes the notice to the report file
  412. # (argv[2]) — run_contained captures the child's stderr, so stderr alone is not seen (m1) — and
  413. # exits; the updater logs the notice, notes it in the receipt and raises CustodyRefused.
  414. # The command also runs in a kill-on-close job of the launcher's own, nested in the update job
  415. # (review C3): once the command exits, the launcher terminates that job — whatever npm / the
  416. # builder started and left running — and waits until no process of it is left before it
  417. # returns, so the caller never releases the checkout while a build descendant still writes.
  418. # Only the command's tree: the update job also holds the update's other children. The Windows
  419. # twin of the POSIX launcher below (_REAP_TREE); the nested job's handle closing (a killed
  420. # launcher) kills the tree too.
  421. _CUSTODY_UNAVAILABLE = "hermes: update custody unavailable"
  422. _REFUSED_EXIT = 87
  423. _JOIN_JOB = (
  424. "import ctypes, subprocess, sys, time\n"
  425. "k = ctypes.WinDLL('kernel32', use_last_error=True)\n"
  426. "k.AssignProcessToJobObject.argtypes = [ctypes.c_void_p, ctypes.c_void_p]\n"
  427. "k.GetCurrentProcess.restype = ctypes.c_void_p\n"
  428. "k.CloseHandle.argtypes = [ctypes.c_void_p]\n"
  429. "k.CreateJobObjectW.restype = ctypes.c_void_p\n"
  430. "k.SetInformationJobObject.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_ulong]\n"
  431. "k.TerminateJobObject.argtypes = [ctypes.c_void_p, ctypes.c_uint]\n"
  432. "k.QueryInformationJobObject.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_ulong,\n"
  433. " ctypes.c_void_p]\n"
  434. "h = ctypes.c_void_p(int(sys.argv[1]))\n"
  435. "def refuse(why):\n"
  436. f" note = '{_CUSTODY_UNAVAILABLE} (%s); the command was not run' % why\n"
  437. " sys.stderr.write(note + '\\n')\n"
  438. " sys.stderr.flush()\n"
  439. " try:\n"
  440. " with open(sys.argv[2], 'w', encoding='utf-8') as report:\n"
  441. " report.write(note)\n"
  442. " except OSError:\n"
  443. " pass\n"
  444. f" sys.exit({_REFUSED_EXIT})\n"
  445. "if not k.AssignProcessToJobObject(h, k.GetCurrentProcess()):\n"
  446. " refuse('could not join the update job: %d' % ctypes.get_last_error())\n"
  447. # JOBOBJECT_EXTENDED_LIMIT_INFORMATION (KILL_ON_JOB_CLOSE) and the BASIC_ACCOUNTING record
  448. # whose ActiveProcesses says when the command's tree is gone.
  449. "class Basic(ctypes.Structure):\n"
  450. " _fields_ = [('times', ctypes.c_int64 * 2), ('flags', ctypes.c_uint32), ('ws', ctypes.c_size_t * 2),\n"
  451. " ('procs', ctypes.c_uint32), ('affinity', ctypes.c_size_t), ('classes', ctypes.c_uint32 * 2)]\n"
  452. "class Limits(ctypes.Structure):\n"
  453. " _fields_ = [('basic', Basic), ('io', ctypes.c_uint64 * 6), ('memory', ctypes.c_size_t * 4)]\n"
  454. "class Usage(ctypes.Structure):\n"
  455. " _fields_ = [('times', ctypes.c_int64 * 4), ('faults', ctypes.c_uint32), ('total', ctypes.c_uint32),\n"
  456. " ('active', ctypes.c_uint32), ('terminated', ctypes.c_uint32)]\n"
  457. "tree = k.CreateJobObjectW(None, None)\n"
  458. "limits = Limits()\n"
  459. "limits.basic.flags = 0x2000\n"
  460. "if not tree or not k.SetInformationJobObject(tree, 9, ctypes.byref(limits), ctypes.sizeof(limits)):\n"
  461. " refuse('could not create the job for the command tree: %d' % ctypes.get_last_error())\n"
  462. # Verify, never assume the topology: a packaged interpreter (Store Python) joins, but starts a
  463. # program outside its package with desktop-app breakaway, so the command leaves every job
  464. # that permits breakaway, as the update job does (restarted gateways break away). Start the
  465. # command suspended; if Windows says it is outside the job, put it in explicitly (F80), and
  466. # run it only once it is in (F54): a command the job will not take is refused, never run.
  467. "k.IsProcessInJob.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.POINTER(ctypes.c_int)]\n"
  468. "p = subprocess.Popen(sys.argv[3:], stdin=subprocess.DEVNULL, creationflags=0x4)\n"
  469. "def in_job():\n"
  470. " inside = ctypes.c_int(0)\n"
  471. " return k.IsProcessInJob(ctypes.c_void_p(int(p._handle)), h, ctypes.byref(inside)) and inside.value\n"
  472. "if not in_job() and not (k.AssignProcessToJobObject(h, ctypes.c_void_p(int(p._handle))) and in_job()):\n"
  473. " p.kill()\n"
  474. " p.wait()\n"
  475. " refuse('the command would start outside the update job')\n"
  476. "if not k.AssignProcessToJobObject(tree, ctypes.c_void_p(int(p._handle))):\n"
  477. " error = ctypes.get_last_error()\n"
  478. " p.kill()\n"
  479. " p.wait()\n"
  480. " refuse('could not hold the command tree in a job: %d' % error)\n"
  481. "k.CloseHandle(h)\n"
  482. "nt = ctypes.WinDLL('ntdll')\n"
  483. "nt.NtResumeProcess.argtypes = [ctypes.c_void_p]\n"
  484. "if nt.NtResumeProcess(ctypes.c_void_p(int(p._handle))) != 0:\n"
  485. " p.kill()\n"
  486. "code = p.wait()\n"
  487. # C3: nothing the command started outlives it; return only once its whole tree is gone.
  488. "k.TerminateJobObject(tree, 1)\n"
  489. "usage, end = Usage(), time.monotonic() + 10\n"
  490. "while time.monotonic() < end:\n"
  491. " if not k.QueryInformationJobObject(tree, 1, ctypes.byref(usage), ctypes.sizeof(usage), None):\n"
  492. " break\n"
  493. " if not usage.active:\n"
  494. " break\n"
  495. " time.sleep(0.02)\n"
  496. "sys.exit(code)\n"
  497. )
  498. def _report_refused_join(report: str, argv: Sequence[str]) -> str | None:
  499. """Make a refused job join visible: a warning (errors.log) and a receipt step. The notice,
  500. or None when the child joined (or never ran the launcher)."""
  501. import os
  502. try:
  503. with open(report, encoding="utf-8-sig") as fh:
  504. note = fh.read().strip()
  505. except OSError:
  506. note = ""
  507. finally:
  508. with contextlib.suppress(OSError):
  509. os.unlink(report)
  510. if not note:
  511. return None
  512. logger.warning("%s: %s", note, " ".join(str(arg) for arg in argv[:2]))
  513. with contextlib.suppress(Exception):
  514. from hermes_cli.update_receipt import record_step
  515. record_step("update_custody", False, f"{note}: {' '.join(str(arg) for arg in argv[:2])}")
  516. return note
  517. def _join_launcher_python() -> str:
  518. """The interpreter for the job-joining launcher: the real one, never a venv redirector.
  519. A venv's ``Scripts\\python.exe`` is a redirector: it starts the base interpreter as its own
  520. child, inside a job of its own. That child cannot join the update's job once the job holds a
  521. process from another job hierarchy (the updater's git children): ``AssignProcessToJobObject``
  522. fails with ERROR_ACCESS_DENIED (5). And the redirector, never joined, keeps its inherited copy
  523. of the job handle open, so the job would not close when the owner dies. The launcher is
  524. stdlib-only (``-I -S``), so the base interpreter runs it as is.
  525. """
  526. import os
  527. base = getattr(sys, "_base_executable", None)
  528. if base and os.path.isfile(base):
  529. return base
  530. return sys.executable
  531. # POSIX twin of the launcher (N13, review L1/L6). Node marks the donated lock fd close-on-exec,
  532. # so nothing node starts (npm, esbuild, sh) keeps the checkout locked: custody of them rests on
  533. # this stdlib parent, which holds the fd until no descendant of the command is left.
  534. # * The command stays in the CALLER's process group: every group kill that stops a build (a
  535. # Ctrl-C'd completion child's ``killpg``, Desktop's ``kill(-pid)``) reaches node and everything
  536. # under it that did not start a session of its own. The custodian leaves that group (E): a
  537. # SIGKILL of the group would otherwise kill it too, and a descendant in a session of its own
  538. # would go on writing with the checkout lock free. It outlives the kill, settles the tree as
  539. # below, and only then exits (releasing the fd).
  540. # * Linux: the custodian is a child subreaper, so a descendant orphaned by node's exit (or by any
  541. # intermediate's) is re-parented to it, never to init. Once node exits it SIGKILLs and reaps
  542. # its children until none is left; it only signals its own unreaped children, so no pid it
  543. # kills can have been reused (never a ``killpg`` of an already reaped leader's group).
  544. # * Elsewhere (macOS has no subreaper) it records node's descendant tree from ``ps`` while node
  545. # runs and, once node exits, kills every recorded process whose start time still matches,
  546. # plus what they started. Best effort: a process born and orphaned between two samples escapes.
  547. # * SIGINT/SIGTERM/SIGHUP are forwarded to node (a node still running 10 s later is killed); the
  548. # descendants are then killed as above.
  549. # * The custodian is NOT the process the runner sees (R4): ``subprocess.run`` SIGKILLs its own
  550. # child when Ctrl-C reaches the caller, and no group separation survives a direct kill. The
  551. # runner's child only forks the custodian (own group) and relays signals to it and its exit
  552. # status back; it leaves the caller's group too, or a terminal's Ctrl-C would reach node twice. Killed, it leaves the custodian to settle: node gets the same 10 s it gets after
  553. # a forwarded signal (it is usually still cleaning up from the same Ctrl-C), then the tree is
  554. # killed as above and only then is the fd dropped.
  555. _REAP_TREE = r"""
  556. import os, signal, subprocess, sys, time
  557. KILL = signal.SIGKILL # windows-footgun: ok - POSIX-only launcher
  558. fds = tuple(int(fd) for fd in sys.argv[1].split(','))
  559. caller = os.getpgrp()
  560. if caller != os.getpid():
  561. os.setpgid(0, 0)
  562. stand_in = os.getpid()
  563. custodian = os.fork()
  564. if custodian:
  565. def relay(signum, frame):
  566. try:
  567. os.kill(custodian, signum)
  568. except OSError:
  569. pass
  570. for name in ('SIGINT', 'SIGTERM', 'SIGHUP'):
  571. signal.signal(getattr(signal, name), relay)
  572. code = os.waitstatus_to_exitcode(os.waitpid(custodian, 0)[1])
  573. sys.exit(code if code >= 0 else 128 - code)
  574. os.setpgid(0, 0)
  575. me = os.getpid()
  576. reaper = False
  577. if sys.platform.startswith('linux'):
  578. try:
  579. import ctypes
  580. reaper = ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) == 0 # PR_SET_CHILD_SUBREAPER
  581. except (OSError, AttributeError):
  582. reaper = False
  583. def children():
  584. found = []
  585. for entry in os.listdir('/proc'):
  586. if not entry.isdigit():
  587. continue
  588. try:
  589. with open('/proc/%s/stat' % entry, 'rb') as fh:
  590. stat = fh.read()
  591. if int(stat[stat.rindex(b')') + 2:].split()[1]) == me:
  592. found.append(int(entry))
  593. except (OSError, ValueError, IndexError):
  594. continue
  595. return found
  596. def table():
  597. try:
  598. out = subprocess.run(['ps', '-A', '-o', 'pid=,ppid=,stat=,lstart='], stdin=subprocess.DEVNULL,
  599. capture_output=True, text=True, errors='replace', timeout=10).stdout
  600. except (OSError, subprocess.SubprocessError):
  601. return {}
  602. rows = {}
  603. for line in out.splitlines():
  604. parts = line.split(None, 3)
  605. if len(parts) == 4 and parts[0].isdigit() and parts[1].isdigit() and not parts[2].startswith('Z'):
  606. rows[int(parts[0])] = (int(parts[1]), parts[3].strip())
  607. return rows
  608. seen = {}
  609. def sample(root):
  610. rows = table()
  611. tree = {pid for pid, start in seen.items() if rows.get(pid, (0, None))[1] == start}
  612. if root is not None:
  613. tree.add(root)
  614. grew = True
  615. while grew:
  616. grew = False
  617. for pid, (ppid, start) in rows.items():
  618. if ppid in tree and pid not in tree:
  619. tree.add(pid)
  620. seen[pid] = start
  621. grew = True
  622. return [pid for pid, start in seen.items() if rows.get(pid, (0, None))[1] == start]
  623. def kill(pids):
  624. for pid in pids:
  625. try:
  626. os.kill(pid, KILL)
  627. except OSError:
  628. pass
  629. p = subprocess.Popen(sys.argv[2:], pass_fds=fds, process_group=caller)
  630. got = []
  631. def forward(signum, frame):
  632. got.append(signum)
  633. try:
  634. p.send_signal(signum)
  635. except OSError:
  636. pass
  637. for name in ('SIGINT', 'SIGTERM', 'SIGHUP'):
  638. signal.signal(getattr(signal, name), forward)
  639. deadline = None
  640. while True:
  641. try:
  642. code = p.wait(timeout=0.5)
  643. break
  644. except subprocess.TimeoutExpired:
  645. pass
  646. if not reaper:
  647. sample(p.pid)
  648. if got or os.getppid() != stand_in:
  649. deadline = deadline or time.monotonic() + 10
  650. if time.monotonic() > deadline:
  651. p.kill()
  652. end = time.monotonic() + 10
  653. while time.monotonic() < end:
  654. live = children() if reaper else sample(None)
  655. if not live:
  656. break
  657. kill(live)
  658. while reaper:
  659. try:
  660. if not os.waitpid(-1, os.WNOHANG)[0]:
  661. break
  662. except ChildProcessError:
  663. break
  664. time.sleep(0.02)
  665. sys.exit(code if code >= 0 else 128 - code)
  666. """
  667. @contextlib.contextmanager
  668. def contained_command(argv: Sequence[str], *, inherit_lock: bool = True, root=None):
  669. """``(argv, kwargs)`` for a checkout writer started by a runner that hides its Popen (the Node
  670. build in ``pm.progress.run_contained``). POSIX: the lock fd (this process's, or one it
  671. inherited for checkout ``root``), under a launcher that keeps the command in the caller's
  672. process group and kills every descendant left when it exits (:data:`_REAP_TREE`).
  673. Windows inside an update: the command runs under a launcher that joins the update's
  674. kill-on-close job first and returns only once no process the command started is left (C3);
  675. when the job cannot be handed over or the join is refused, the command never runs and
  676. :class:`CustodyRefused` is raised (D2)."""
  677. argv = list(argv)
  678. if not (sys.platform == "win32" and _held() is not None):
  679. from hermes_cli.update_lock import checkout_lock_fds
  680. fds = tuple(checkout_lock_fds(root)) if inherit_lock and root is not None else _lock_fds(inherit_lock)
  681. if fds:
  682. argv = [sys.executable, "-I", "-S", "-c", _REAP_TREE, ",".join(map(str, fds)), *argv]
  683. yield argv, ({"pass_fds": fds} if fds else {})
  684. return
  685. try:
  686. handle = _inheritable_job_handle()
  687. except OSError as exc:
  688. raise _refuse(argv, exc) from exc
  689. import os
  690. import tempfile
  691. fd, report = tempfile.mkstemp(prefix="hermes-custody-", suffix=".txt")
  692. os.close(fd)
  693. try:
  694. info = subprocess.STARTUPINFO()
  695. info.lpAttributeList = {"handle_list": [handle]}
  696. yield ([_join_launcher_python(), "-I", "-S", "-c", _JOIN_JOB, str(handle), report, *argv],
  697. {"startupinfo": info})
  698. finally:
  699. import ctypes
  700. ctypes.WinDLL("kernel32").CloseHandle(ctypes.c_void_p(handle))
  701. note = _report_refused_join(report, argv)
  702. if note:
  703. cause = note.removeprefix(f"{_CUSTODY_UNAVAILABLE} (").partition(")")[0]
  704. raise _refuse(argv, cause, receipted=True)
  705. _RUN["ran"] = True
  706. def _inheritable_job_handle() -> int:
  707. """An inheritable duplicate of the update's kill-on-close job handle (Windows). Raises
  708. ``OSError`` when the job cannot be created or duplicated."""
  709. import ctypes
  710. from hermes_cli.update_lock import update_tree_job
  711. try:
  712. job = update_tree_job()
  713. kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
  714. kernel32.GetCurrentProcess.restype = ctypes.c_void_p
  715. kernel32.DuplicateHandle.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p,
  716. ctypes.POINTER(ctypes.c_void_p), ctypes.c_ulong, ctypes.c_int,
  717. ctypes.c_ulong]
  718. me, dup = kernel32.GetCurrentProcess(), ctypes.c_void_p()
  719. # DUPLICATE_SAME_ACCESS, inheritable
  720. if not kernel32.DuplicateHandle(me, job, me, ctypes.byref(dup), 0, True, 0x2):
  721. raise ctypes.WinError(ctypes.get_last_error())
  722. return int(dup.value)
  723. except OSError as exc:
  724. logger.warning("Could not hand the update's job to a build child: %s", exc)
  725. raise