update_completion.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586
  1. """Fresh-checkout source update completion and its stdlib-only parent transport.
  2. Imported before a swap; executed by path from the selected tree afterward. The
  3. parent never imports application helpers from the replacement checkout.
  4. """
  5. from __future__ import annotations
  6. import codecs
  7. import json
  8. import os
  9. import signal
  10. from pathlib import Path
  11. import subprocess
  12. import sys
  13. import tempfile
  14. def _write_json(path: Path, data: dict) -> None:
  15. temporary = path.with_suffix(".tmp")
  16. temporary.write_text(json.dumps(data), encoding="utf-8")
  17. temporary.replace(path)
  18. def _exit_status(code: int) -> int:
  19. return code if code >= 0 else 128 - code
  20. def _write_bootstrap_result(request: dict, result_path: Path, code: int, receipt: dict | None) -> int:
  21. """The bootstrap child's answer; windows_resume stays the parent's token (and its atexit net)."""
  22. _write_json(result_path, {
  23. "schema": 1, "update_id": request["receipt"]["update_id"], "exit_code": code,
  24. "receipt": receipt, "windows_resume": None, "pm_receipt": request.get("pm_receipt"),
  25. })
  26. return code
  27. def _bind_and_resume(proc: subprocess.Popen, request: dict, request_path: Path) -> None:
  28. """Windows: bind the suspended completion child to the update's job, then resume it.
  29. Post-commit, refusing the child would fail a committed update, so a child the job refuses
  30. still runs, fenced by its own checkout lease instead (R5b: it joins our lock holding a lease
  31. byte, which keeps the checkout busy after a killed owner until the child exits). Never
  32. silently: the refusal is printed and recorded as a failed ``update_custody`` step in this
  33. run's receipt and, before the child runs (it has not read its request yet), in the receipt it
  34. resumes, so the terminal receipt carries it."""
  35. from hermes_cli import update_receipt
  36. from hermes_cli.update_lock import bind_child_to_update_tree, resume_suspended_child
  37. refusal = bind_child_to_update_tree(proc)
  38. if refusal is not None:
  39. detail = (f"the update's job would not take the completion child ({refusal}), so it runs "
  40. "outside the job, holding its own checkout lease")
  41. print(f" ⚠ Update completion: {detail}")
  42. update_receipt.record_step("update_custody", False, detail)
  43. current = update_receipt._current.get()
  44. if current is not None and current.data.get("update_id") == request["receipt"]["update_id"]:
  45. request["receipt"] = json.loads(json.dumps(current.data))
  46. _write_json(request_path, request)
  47. resume_suspended_child(proc)
  48. def run_completion(request: dict) -> dict:
  49. """Wait for new code; zero exit without a correlated terminal result fails closed."""
  50. root = Path(request["source"])
  51. # The child below runs in a new session without a controlling terminal, so
  52. # this is the last point where sudo can ask for a password. Run the new
  53. # tree's pre-install as its own process (this parent stays stdlib-only); a
  54. # tree without it, or any failure, just leaves the in-lock repair to report.
  55. if sys.platform.startswith("linux"):
  56. subprocess.run([sys.executable, "-I", "-S", "-B", "-c",
  57. "import sys; sys.path.insert(0, sys.argv[1]); "
  58. "from pm.libatomic import install_before_lock; install_before_lock()", str(root)],
  59. cwd=root, stdout=None, stderr=subprocess.DEVNULL, check=False)
  60. env = dict(os.environ, HERMES_HOME=request["home"], PYTHONUNBUFFERED="1")
  61. for key in ("PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV"):
  62. env.pop(key, None)
  63. with tempfile.TemporaryDirectory(prefix="hermes-completion-") as directory:
  64. request_path = Path(directory) / "request.json"
  65. result_path = Path(directory) / "result.json"
  66. request = {**request, "stdout_isatty": sys.stdout.isatty()}
  67. request["bytecode_cache"] = str(Path(directory) / "bytecode")
  68. _write_json(request_path, request)
  69. command = [sys.executable, "-I", "-S", "-u", "-X", "utf8", "-X", f"pycache_prefix={request['bytecode_cache']}",
  70. str(root / "hermes_cli/update_completion.py"),
  71. str(request_path), str(result_path)]
  72. from hermes_cli.update_lock import CREATE_SUSPENDED, checkout_lock_fds
  73. # The child joins the update tree's checkout lock: it inherits the locked fd (POSIX)
  74. # or dies with us (Windows job: created suspended, bound, then resumed, so nothing it
  75. # starts runs outside the job), so the lock is never free while it runs.
  76. proc = subprocess.Popen(
  77. command, cwd=root, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
  78. # detached-spawn: ok — awaited; holds the checkout lock fd so an update finishes atomically even if this parent dies
  79. **({"start_new_session": True, "pass_fds": checkout_lock_fds(root)} if os.name == "posix" else
  80. {"creationflags": subprocess.CREATE_NO_WINDOW | CREATE_SUSPENDED}))
  81. decoder = codecs.getincrementaldecoder("utf-8")("replace")
  82. try:
  83. # A failure here unwinds through the cleanup below, never orphans the child.
  84. if os.name != "posix":
  85. _bind_and_resume(proc, request, request_path)
  86. while True:
  87. chunk = proc.stdout.read1(8192)
  88. sys.stdout.write(decoder.decode(chunk, final=not chunk))
  89. sys.stdout.flush()
  90. if not chunk:
  91. break
  92. code = proc.wait()
  93. except BaseException as exc:
  94. # This group/retained process handle belongs exclusively to us.
  95. # Try to stop descendants before releasing the command's update lock.
  96. try:
  97. try:
  98. if os.name == "posix":
  99. try:
  100. os.killpg(proc.pid, signal.SIGKILL) # windows-footgun: ok — os.name == "posix"; own isolated group
  101. except ProcessLookupError:
  102. pass
  103. else:
  104. subprocess.run(["taskkill", "/T", "/F", "/PID", str(proc.pid)],
  105. stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
  106. stderr=subprocess.DEVNULL, timeout=10, check=True,
  107. creationflags=subprocess.CREATE_NO_WINDOW)
  108. finally:
  109. # A failed tree kill must not bypass retained-handle cleanup.
  110. try:
  111. proc.kill()
  112. finally:
  113. proc.wait(timeout=10)
  114. except BaseException as cleanup_error:
  115. exc.add_note("Completion cleanup failed; child processes may still be running.")
  116. raise exc from cleanup_error
  117. raise
  118. finally:
  119. proc.stdout.close()
  120. code = _exit_status(code)
  121. try:
  122. result = json.loads(result_path.read_text(encoding="utf-8-sig"))
  123. if result["schema"] != 1 or result["update_id"] != request["receipt"]["update_id"]:
  124. raise ValueError("completion response identity mismatch")
  125. if result["exit_code"] != code:
  126. raise ValueError("completion response disagrees with process exit")
  127. receipt = result.get("receipt")
  128. if receipt is not None and (
  129. receipt.get("update_id") != request["receipt"]["update_id"]
  130. or not receipt.get("finished_at")
  131. or (code == 0) != (receipt.get("outcome") == "success")
  132. ):
  133. raise ValueError("completion receipt does not attest this outcome")
  134. if code == 0 and receipt is None:
  135. raise ValueError("completion did not publish a terminal receipt")
  136. except (OSError, ValueError, KeyError, TypeError) as exc:
  137. # No correlated terminal result: the caller settles it (settle_lost_completion).
  138. return {"exit_code": code or 1, "receipt": None, "windows_resume": None,
  139. "error": f"{exc} (the completion process exited {code})"}
  140. return result
  141. def _resume_receipt(data: dict) -> None:
  142. from hermes_cli import update_receipt
  143. # Hydrate the existing run, not a new receipt with a new identity/pre-update probe.
  144. receipt = object.__new__(update_receipt.UpdateReceipt)
  145. receipt.data = data
  146. receipt.correlation_id = data["update_id"]
  147. receipt.current_token = update_receipt._current.set(receipt)
  148. def _read_terminal_receipt(request: dict) -> dict | None:
  149. from hermes_cli import update_receipt
  150. update_id = request["receipt"]["update_id"]
  151. found = update_receipt.read_run_record(update_id)
  152. if found is not None and found[1].get("finished_at"):
  153. return found[1]
  154. # The store refused the terminal write after the commit point (loudly): this process still
  155. # holds the correlated terminal record it finalized, so the exit status never turns into 1.
  156. return update_receipt.finalized_receipt(update_id)
  157. def _running_record(request: dict) -> dict | None:
  158. from hermes_cli.update_receipt import read_run_record
  159. found = read_run_record(request["receipt"]["update_id"])
  160. if found is None or found[1].get("outcome") != "running":
  161. return None
  162. found[1].pop("writer_pid", None)
  163. return found[1]
  164. def _owed_user_action(request: dict) -> str | None:
  165. """The unsettled-autostash notice (#122557) when this run parked the user's local changes.
  166. ``_complete_source_update`` hands it over as the completion message, and a completion message
  167. that is not a ``✓`` line is never a success (``_print_verified_update_completion``).
  168. """
  169. message = request.get("completion_message") or ""
  170. return message if message and not message.startswith("✓") else None
  171. def _record_owed_user_action(request: dict) -> bool:
  172. """Land the parked local changes on the receipt; True when nothing is owed to the user."""
  173. from hermes_cli.update_receipt import record_user_action
  174. notice = _owed_user_action(request)
  175. if notice:
  176. record_user_action("local_changes", notice)
  177. return notice is None
  178. def _publish_gateway_success(request: dict) -> None:
  179. """The gateway /update watcher's status: the code is committed (same as _complete_selected).
  180. Guarded like ``update_cmd_fleet._write_gateway_update_exit_code`` (which this stdlib-only
  181. process cannot import): an unwritable home must not turn the committed run into an error
  182. before its result is written (review P1). The watcher then falls back to its own timeout.
  183. """
  184. if not request.get("gateway_mode"):
  185. return
  186. try:
  187. # Temp + rename (stdlib, like _write_json): the gateway watcher polls this file and reads
  188. # an empty one as exit 1.
  189. marker = Path(request["home"]) / ".update_exit_code"
  190. temporary = marker.with_name(marker.name + ".tmp")
  191. temporary.write_text("0", encoding="utf-8")
  192. temporary.replace(marker)
  193. except OSError as exc:
  194. print(f" ⚠ Could not publish the gateway's update status: {exc}")
  195. #: The one whole line the Desktop hand-off scripts match (scripts/desktop-update/posix.sh and
  196. #: windows.ps1); ``source_build.build_update_products`` prints the same prefix when the build fails.
  197. DESKTOP_BUILD_OWED = "Desktop app build owed:"
  198. def _report_desktop_build_owed(request: dict, record: dict | None, reason: str) -> None:
  199. """Name the owed Desktop build when this run never got the build to its own verdict (F6, review P4).
  200. The builder prints the line only when the build itself fails; a run that never reached it
  201. (dependencies owed, the tail refused by the lock, a completion process lost) would otherwise end
  202. as a plain "Update complete." in the hand-off. A ``build`` stage on the run's record means the
  203. build ran and printed this line itself when the Desktop app failed.
  204. """
  205. if not request.get("desktop"):
  206. return
  207. if any(isinstance(mark, dict) and mark.get("name") == "build" for mark in (record or {}).get("stages") or ()):
  208. return
  209. print(f" {DESKTOP_BUILD_OWED} {reason}", flush=True)
  210. def settle_lost_completion(request: dict, reason: str) -> dict:
  211. """The parent's side of C3: the code is committed, but the completion never answered (review P2).
  212. Reached when spawning or reading the completion failed (an OSError from the temporary
  213. directory, the request file or Popen) or the child died without a correlated terminal result
  214. (OOM, SIGKILL). The tail is owed, never a failed update: it is armed for the next launch, the
  215. run's receipt is a success that names the ``completion`` follow-up, and the exit is 0 -- unless
  216. the user's local changes are still parked (#122557: ``partial``, exit 1). Runs in the parent
  217. (pre-swap) interpreter: everything it uses is already imported there.
  218. """
  219. from hermes_cli import update_receipt
  220. print(f" ⚠ Source update completion did not finish: {reason}", flush=True)
  221. receipt = _read_terminal_receipt(request) # the child finalized the run, then died
  222. if receipt is None:
  223. try:
  224. from hermes_cli.venv_sync import arm_completion
  225. arm_completion(Path(request["source"]))
  226. except Exception as exc: # health: allow BLE001 -- post-commit boundary: stale dependencies still make the next launch sync
  227. print(f" ⚠ Could not record the owed source-update tail: {exc}")
  228. _report_desktop_build_owed(request, _running_record(request), "the update completion did not finish")
  229. update_receipt.resume_run_record() # the child's persisted stages, never this stale snapshot
  230. update_receipt.record_followup("completion", reason, retry="the next launch or `hermes update` finishes it")
  231. parked = not _record_owed_user_action(request)
  232. if parked:
  233. print(_owed_user_action(request)) # the completion child that prints it may never have run
  234. update_receipt.finalize_pending_update_receipt(1 if parked else 0, "completion owed after the code was updated")
  235. receipt = _read_terminal_receipt(request)
  236. outcome = (receipt or {}).get("outcome")
  237. code = 130 if outcome == "interrupted" else 1 if outcome == "partial" else 0
  238. if code == 0:
  239. _publish_gateway_success(request)
  240. return {"exit_code": code, "receipt": receipt, "windows_resume": None, "pm_receipt": None}
  241. def _settle_after_commit(request: dict, result_path: Path, step: str, reason: str) -> int:
  242. """The tree already moved, so a failure here is owed work, never a failed update (C3, A6).
  243. Runs in the bootstrap interpreter (stdlib + the new tree) when dependency preparation failed or
  244. the prepared child died without a result. The tail obligation stays armed, so the next launch
  245. syncs the dependencies and finishes the tail; the run's receipt is a success that names the
  246. follow-up, so nothing reports "still on the previous version" while the tree is new.
  247. """
  248. from hermes_cli import update_receipt
  249. from hermes_cli.venv_sync import arm_completion
  250. try:
  251. arm_completion(Path(request["source"]))
  252. except OSError as exc: # the next launch still sees stale dependencies and syncs them
  253. print(f" ⚠ Could not record the owed source-update tail: {exc}")
  254. receipt = _read_terminal_receipt(request) # a prepared child that finalized, then died
  255. if receipt is None:
  256. # A prepared child that died mid-tail persisted stages the parent's snapshot lacks.
  257. record = _running_record(request) or request["receipt"]
  258. _report_desktop_build_owed(request, record, f"{step} did not finish")
  259. _resume_receipt(record)
  260. update_receipt.record_stage("deps" if step == "dependencies" else "build", "failed")
  261. update_receipt.record_followup(step, reason, retry="dependencies not installed yet — the next launch retries"
  262. if step == "dependencies" else "the next launch or `hermes update` retries it")
  263. if not _record_owed_user_action(request):
  264. print(_owed_user_action(request)) # the completion child that would print it never ran
  265. update_receipt.finalize_pending_update_receipt(0, f"{step} owed after the code was updated")
  266. receipt = _read_terminal_receipt(request)
  267. code = 0 if receipt is not None and receipt.get("outcome") == "success" else 1
  268. if code == 0:
  269. _publish_gateway_success(request)
  270. return _write_bootstrap_result(request, result_path, code, receipt)
  271. def _prepare(request: dict, request_path: Path, result_path: Path) -> int:
  272. import pm
  273. from pm import receipt
  274. from pm.client import ensure_tools_for_sync
  275. from pm.environments import activation_environment, project_python
  276. root = Path(request["source"])
  277. update_id = request["receipt"]["update_id"]
  278. if sys.platform == "win32":
  279. # Windows (review L3): a bootstrap child the update's job refused runs outside the job
  280. # and outlives a killed owner, so it joins the checkout lock before anything below
  281. # writes the checkout (uv's sync children, the generation collector): the join takes a
  282. # lease byte of its own (R5b, as the --prepared child does in complete_source_checkout).
  283. # Held until this bootstrap exits, after the --prepared child; the kernel drops it.
  284. from hermes_cli.update_lock import _acquire_checkout
  285. refused = _acquire_checkout(root)
  286. if refused is not None:
  287. raise RuntimeError("could not join the update's checkout lock "
  288. f"({refused.reason or f'held by process {refused.pid}'})")
  289. from hermes_cli.venv_sync import arm_completion, collect_superseded_generations
  290. # The foreign-owned-venv refusal runs in the parent BEFORE the swap (update_cmd_commit
  291. # .preflight_refusal); the tail was armed there too, so this re-arm is an idempotent backstop.
  292. arm_completion(root)
  293. from hermes_cli.gitlock import convert_treeless_checkout_first
  294. convert_treeless_checkout_first(root)
  295. with receipt.worker_context(update_id):
  296. try:
  297. # This file runs from the new tree, so its lockfile carries the new
  298. # pins; tools (incl. bumped uv/python) land before the sync uses them.
  299. ensure_tools_for_sync()
  300. # An update never fails because of a plugin: misfits are disabled and reported.
  301. pm.sync_venv(explicit=True, project_root=root, evict_incompatible_plugins=True)
  302. collect_superseded_generations(root)
  303. finally:
  304. request["pm_receipt"] = receipt.last_for_update(update_id)
  305. _write_json(request_path, request)
  306. # -X utf8 like the bootstrap child: -I ignores PYTHONIOENCODING, and this child prints ✓/⚠
  307. # (the follow-up protocol) into the parent's pipe, whatever the console code page.
  308. command = [str(project_python(root)),
  309. "-I", "-S", "-u", "-X", "utf8", "-X", f"pycache_prefix={request['bytecode_cache']}",
  310. str(root / "hermes_cli/update_completion.py"),
  311. str(request_path), str(result_path), "--prepared"]
  312. # A second interpreter is mandatory: PM may have selected a different Python
  313. # and dependency graph. No application maintenance runs in this bootstrap.
  314. from hermes_cli.update_lock import checkout_lock_fds
  315. # health: allow HX006 -- the prepared completion child is the update's build; it runs to the end
  316. code = _exit_status(subprocess.call(command, cwd=root, env=activation_environment(root),
  317. pass_fds=checkout_lock_fds(root)))
  318. if not result_path.exists():
  319. return _settle_after_commit(request, result_path, "completion",
  320. f"the completion process exited {code} without a result")
  321. return code
  322. def _complete_selected(request: dict) -> bool:
  323. """Everything after the commit point. No step failure fails the update (contract C3).
  324. Each step is independent; a failed one prints ``⚠``, lands on the receipt as a follow-up
  325. and keeps its own obligation armed (``source-completion-pending`` for the tail, the fleet
  326. restart obligation for gateways), so the next launch or ``hermes update`` retries it.
  327. Returns False only when the user's local changes were left parked in the stash: nothing
  328. retries that, so the run is ``partial`` and exits 1 (#122557), never "Update complete".
  329. """
  330. from hermes_cli import main, update_cmd, update_cmd_config
  331. from hermes_cli.source_completion import complete_source_checkout
  332. from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan
  333. from hermes_cli.update_receipt import (
  334. TAIL_FOLLOWUPS, record_build_stage, record_followup, record_skip, record_stage)
  335. root = Path(request["source"])
  336. main.PROJECT_ROOT = root
  337. complete = _record_owed_user_action(request)
  338. update_cmd_config._LAST_SIBLING_SNAPSHOTS = request["sibling_snapshots"]
  339. plan_data = request["plan"]
  340. plan = None if plan_data is None else UpdatePlan(**{
  341. **plan_data, "runtimes": [RuntimeRecord(**row) for row in plan_data.get("runtimes", [])]})
  342. update_cmd._sweep_bytecode_after_update(request["branch"])
  343. # Launchers, products and post-build maintenance live in one place so an
  344. # install and an update cannot end in different states.
  345. followups: list[tuple[str, str]] = []
  346. # Exit status and runtime safety are separate facts (R8): an unsafe SQLite runtime keeps the
  347. # committed update at exit 0 (reported as the ``sqlite_runtime`` follow-up), but fleet
  348. # verification still needs the real verdict so it never auto-migrates the gateway topology
  349. # on a runtime that can corrupt sessions. Unknown (the tail raised) is not proven safe.
  350. runtime_safe = False
  351. try:
  352. runtime_safe = complete_source_checkout(
  353. root, desktop=request["desktop"], assume_yes=request["assume_yes"],
  354. gateway_mode=request["gateway_mode"], pre_update_snapshot_id=request["snapshot_id"],
  355. pre_update_version=request["pre_update_version"],
  356. completion_message=request.get("completion_message"),
  357. announce=None if request.get("completion_message") else "\n✓ Code updated!",
  358. followups=followups, before_build=lambda: _resume_paused_before_build(request))
  359. except (Exception, SystemExit) as exc: # health: allow BLE001 -- e.g. the shared update lock refused the tail
  360. reason = str(exc) or type(exc).__name__
  361. # The tail's steps catch their own failures, so a raise here means the build never ran.
  362. _report_desktop_build_owed(request, None, "the source-update tail did not run")
  363. record_followup("completion", reason)
  364. followups.append(("completion", reason))
  365. tail_owed = any(step in TAIL_FOLLOWUPS for step, _ in followups)
  366. record_build_stage(followups)
  367. if not tail_owed:
  368. from hermes_cli.venv_sync import clear_completion
  369. clear_completion(root)
  370. # systemctl's KillMode=mixed fallback can kill this whole cgroup. Publish the
  371. # gateway watcher's status BEFORE that operation: the code is committed, so it is 0 unless
  372. # the user's own changes are still parked.
  373. if request["gateway_mode"]:
  374. update_cmd._write_gateway_update_exit_code(complete)
  375. if request.get("no_gateway_restart", False):
  376. record_skip("gateway_restart", "--no-gateway-restart: deferred, marker kept")
  377. record_stage("restart", "skipped")
  378. print("→ Gateway restart deferred (--no-gateway-restart); restart gateways separately.")
  379. return complete
  380. skip = update_cmd._fleet_restart_skip_reason(plan)
  381. if skip and update_cmd._pending_fleet_restart_needed():
  382. # A host already stamped "restarted" for this SHA whose fleet is still off the checkout
  383. # (a stale sibling, a failed resume) gets the restart again instead of a dead end.
  384. print(f" → Gateway restart not skipped ({skip}): gateways are still off the checkout code.")
  385. skip = None
  386. if skip:
  387. record_skip("gateway_restart", skip)
  388. record_stage("restart", "skipped")
  389. print(f" ✓ Gateway restart skipped: {skip}.")
  390. return complete
  391. restart = update_cmd._restart_gateway_fleet_after_update(plan, request["gateway_mode"])
  392. record_stage("restart", "failed" if getattr(restart, "incomplete", False) else "success")
  393. update_cmd._resume_windows_gateways_and_merge_outcome(restart, request["windows_resume"], request["gateway_mode"])
  394. update_cmd._verify_fleet_after_update(
  395. restart, _pre_update_plan=plan, _windows_gateway_resume=request["windows_resume"],
  396. update_complete=bool(runtime_safe) and complete)
  397. return complete
  398. def _resume_paused_before_build(request: dict) -> None:
  399. """Restart the gateways the POSIX pause stopped: the dependencies are synced and the launchers
  400. published, the product builds (minutes) have not started. A gateway needs nothing later to
  401. boot. A failure stays owed: the post-build resume retries it and records the outcome.
  402. POSIX only: the fleet restart skips what this restarted (``already_restarted``). A Windows set
  403. keeps resuming after the fleet restart, which would otherwise drain it a second time."""
  404. token = request.get("windows_resume")
  405. if not token or not token.get("resume_needed") or token.get("platform") != "posix":
  406. return
  407. from hermes_cli.update_cmd import _m
  408. try:
  409. _m()._resume_windows_gateways_after_update(token)
  410. except Exception as exc: # health: allow BLE001 -- retried (and recorded) after the build
  411. print(f" ⚠ Paused gateway restart incomplete ({exc}); retrying after the build")
  412. class _ForwardedOutput:
  413. """The parent's pipe preserves its terminal's prompt policy and log mirror."""
  414. def __init__(self, stream, isatty: bool):
  415. self.stream, self.terminal = stream, isatty
  416. def isatty(self):
  417. return self.terminal
  418. def __getattr__(self, name):
  419. return getattr(self.stream, name)
  420. def _report_unbuilt_desktop(request: dict) -> None:
  421. """The tail raised out of ``_complete_selected``: before the build stage, the build never ran."""
  422. from hermes_cli import update_receipt
  423. current = update_receipt._current.get()
  424. if current is not None: # a finalized run got past the build (verification finalizes it)
  425. _report_desktop_build_owed(request, current.data, "the source-update tail did not finish")
  426. def _finish(request: dict, result_path: Path) -> int:
  427. from hermes_cli import update_receipt
  428. from pm.receipt import accept_worker_receipt
  429. _resume_receipt(request["receipt"])
  430. accept_worker_receipt(request.get("pm_receipt"), request["receipt"]["update_id"])
  431. update_receipt.record_stage("deps", "success") # only a completed PM preparation reaches --prepared
  432. code, reason = 0, "source update completion"
  433. try:
  434. if not _complete_selected(request):
  435. code, reason = 1, "local changes left in the stash; re-apply them by hand"
  436. except KeyboardInterrupt:
  437. # An operator interrupt is not a step failure, and the code already moved: the run is
  438. # ``interrupted`` (never "failed"), and the armed obligations finish the tail.
  439. code, reason = 130, "KeyboardInterrupt: interrupted after the code was updated"
  440. update_receipt.finalize_interrupted_update_receipt(reason, exit_code=code)
  441. except SystemExit as exc:
  442. if exc.code not in (0, None):
  443. _report_unbuilt_desktop(request)
  444. update_receipt.record_followup("completion", f"completion exited {exc.code}")
  445. except BaseException as exc:
  446. _report_unbuilt_desktop(request)
  447. update_receipt.record_followup("completion", f"{type(exc).__name__}: {exc}")
  448. finally:
  449. custody = sys.modules.get("hermes_cli.update_custody")
  450. refusal = custody.refusal_notice() if code and custody is not None else None
  451. if refusal: # m2: what stopped it, whatever error the refusal turned into downstream
  452. print(refusal)
  453. # The new interpreter owns recovery too. The original parent's atexit
  454. # token is updated from the response; it acts only if this process dies.
  455. try:
  456. from hermes_cli.update_cmd import _resume_windows_gateways_after_update
  457. _resume_windows_gateways_after_update(request["windows_resume"])
  458. except Exception as exc:
  459. update_receipt.owe_followup(request["receipt"]["update_id"], "windows_resume",
  460. f"Windows gateway recovery failed: {exc}")
  461. update_receipt.finalize_pending_update_receipt(code, reason)
  462. terminal_receipt = _read_terminal_receipt(request)
  463. if terminal_receipt and terminal_receipt.get("outcome") == "success":
  464. code = 0 # an interrupt that landed after verification closed the run (review regression 3)
  465. if code and request["gateway_mode"]:
  466. from hermes_cli.update_cmd import _write_gateway_update_exit_code
  467. _write_gateway_update_exit_code(False)
  468. if not terminal_receipt:
  469. code = code or 1 # the parent settles a lost result (settle_lost_completion)
  470. _write_json(result_path, {
  471. "schema": 1, "update_id": request["receipt"]["update_id"], "exit_code": code,
  472. "receipt": terminal_receipt, "windows_resume": request["windows_resume"],
  473. })
  474. return code
  475. def main() -> int:
  476. request_path, result_path = map(Path, sys.argv[1:3])
  477. request = json.loads(request_path.read_text(encoding="utf-8-sig"))
  478. if request["schema"] != 1:
  479. raise ValueError("unsupported source completion request")
  480. root = Path(__file__).resolve().parents[1]
  481. if root != Path(request["source"]).resolve():
  482. raise ValueError("completion checkout does not match request")
  483. # -I deliberately ignores inherited PYTHONPATH during PM preparation.
  484. sys.path.insert(0, str(root))
  485. sys.stdout = _ForwardedOutput(sys.stdout, request.get("stdout_isatty", False))
  486. if "--prepared" in sys.argv[3:]:
  487. # Claim the selected generation's lease and process its .pth files only
  488. # after PM selection, before importing any application dependencies.
  489. from pm.environments import activate_dependencies
  490. activate_dependencies(root)
  491. return _finish(request, result_path)
  492. return _bootstrap(request, request_path, result_path)
  493. def _bootstrap(request: dict, request_path: Path, result_path: Path) -> int:
  494. """Dependency preparation in the parent's interpreter; its failure is a follow-up (A6)."""
  495. try:
  496. return _prepare(request, request_path, result_path)
  497. except KeyboardInterrupt:
  498. _resume_receipt(request["receipt"])
  499. from hermes_cli import update_receipt
  500. update_receipt.finalize_interrupted_update_receipt(
  501. "KeyboardInterrupt: interrupted while installing dependencies", exit_code=130)
  502. return _write_bootstrap_result(request, result_path, 130, _read_terminal_receipt(request))
  503. except (Exception, SystemExit) as exc: # health: allow BLE001 -- after the commit point nothing fails the update
  504. # The code is committed but its dependencies are not (A6): an owed follow-up, exit 0.
  505. # Paused Windows gateways stay the parent's to resume (it has the dependencies).
  506. return _settle_after_commit(request, result_path, "dependencies", str(exc) or type(exc).__name__)
  507. if __name__ == "__main__":
  508. raise SystemExit(main())