| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053 |
- """Durable record of the gateways and services ``hermes update`` paused on Windows.
- The pause token used to live only in the updater's memory (plus an ``atexit`` resume), so a
- killed updater (console closed, ``taskkill``, power loss) left every paused gateway and SCM
- service stopped with nothing on disk saying so. The record is written under the ROOT Hermes
- home BEFORE anything is stopped, names its owner (pid + creation time, the update-marker
- identity format), is rewritten as the token changes, and is removed only after a verified
- resume. A later launch whose owner check finds the updater dead (and no update live) resumes it.
- One record per checkout (the file name carries a key of the install root): a checkout sharing
- the home never imports, relabels or certifies another checkout's paused set — its tree gate
- says nothing about the other tree.
- Resuming is gated on a whole tree: no interrupted-pull marker; at the pre-update HEAD, no tracked
- change beyond the ones present at pause time (git died before moving HEAD); at a moved HEAD,
- dependencies current for it where a launch syncs them (a build step may rewrite tracked files there). Every pause folded into
- a record keeps its own baseline (``baselines``): each one taken at the current HEAD must hold.
- Otherwise the record stays for the next launch, which runs after the interrupted-pull restore and
- the dependency sync.
- Custody: every mutation (write, claim, retire, discharge) happens while holding a kernel lock on
- ``<record dir>/.hermes-update-paused-gateways.lock`` (flock / msvcrt, released by the kernel when
- the holder dies; the file is never deleted), and the read → judge → mutate decision is made
- inside that hold. A recovering launch claims a record by publishing ``<record>.<pid>.<nonce>.claim``
- with its own identity already inside (atomic replace), then retires the source; a crash between the
- two leaves two files with ONE obligation id (``pause_id``), and an update that folds orphaned sets
- into its own record lists their ids under ``absorbed`` before retiring them. Readers treat a file
- whose id another file carries as a copy, never as a second obligation, and the next mutator
- retires it. Completing an obligation first adds its ids to ``<record stem>.retired`` (atomic), then
- unlinks its files: an unlink Windows refuses (a reader holding the file without
- FILE_SHARE_DELETE) leaves a copy that is redundant by id, never an obligation that executes again.
- The list is read only under the mutex, and only its absence means "nothing retired": a list that
- cannot be read means unknown, so nothing is claimed and the list is never rewritten from that read.
- A list that reads but does not parse is unknown too while any record or claim exists; with none
- left it guards nothing and is set aside as ``<record stem>.corrupt``.
- """
- from __future__ import annotations
- import glob
- import hashlib
- import json
- import os
- import shutil
- import subprocess
- import sys
- import threading
- import time
- import uuid
- from contextlib import contextmanager, redirect_stdout, suppress
- from pathlib import Path
- RECORD_STEM = ".hermes-update-paused-gateways"
- MUTEX_NAME = RECORD_STEM + ".lock"
- _MUTEX_WAIT_S = 10.0
- # Re-entry depth per THREAD: another thread of this process (a gateway's consumer, a watcher)
- # must wait for the kernel lock like any other holder, never ride on this thread's hold.
- _mutex_held = threading.local()
- def install_root() -> Path:
- return Path(__file__).resolve().parents[1]
- def _install_key(root: Path | None = None) -> str:
- return hashlib.sha256(os.path.normcase(str(root or install_root())).encode("utf-8")).hexdigest()[:12]
- def record_path() -> Path:
- from hermes_constants import get_default_hermes_root
- return get_default_hermes_root() / f"{RECORD_STEM}.{_install_key()}.json"
- def identity(pid: int | None = None) -> dict:
- """``{"pid", "ct"}`` with ``ct`` spelled like the update marker's line 3 (``ct:<s.3f>``).
- The creation time comes from the probe :func:`identity_is_live` judges with
- (``update_lock.process_create_time``: psutil, else stdlib), so a host without psutil still
- writes a provable identity instead of ``ct=None``."""
- from hermes_cli import update_lock
- pid = os.getpid() if pid is None else int(pid)
- created = update_lock.process_create_time(pid)
- return {"pid": pid, "ct": None if created is None else f"ct:{created:.3f}"}
- def identity_is_live(ident: dict | None) -> bool:
- """The update marker's incarnation rule (``update_lock.incarnation_live``) for a recorded identity.
- Unprovable (alive, no readable creation time) counts as live: a record is never taken from, nor
- a paused process restarted over, a process that may still be the one recorded.
- """
- from hermes_cli import update_lock
- ident = ident or {}
- return update_lock.incarnation_live(ident.get("pid") or 0, ident.get("ct") or None) is not False
- class RecordConflict(OSError):
- """The record names another pause whose owner is still live."""
- class RecordBusy(OSError):
- """Another process held the record mutex past the bounded wait."""
- class RetiredUnknown(OSError):
- """The retired-id list exists but cannot be read: which obligations are complete is unknown."""
- class RecordUnknown(OSError):
- """A saved pause or claim exists but cannot be read or parsed: what it still owes is unknown."""
- @contextmanager
- def _mutex(wait_s: float = _MUTEX_WAIT_S):
- """Exclusive kernel lock on the record directory's sidecar (A7). Re-entrant per thread; each
- thread's first entry opens its own descriptor, so threads exclude each other like processes."""
- depth = getattr(_mutex_held, "depth", 0)
- if depth:
- _mutex_held.depth = depth + 1
- try:
- yield
- finally:
- _mutex_held.depth -= 1
- return
- from hermes_cli import update_lock
- path = record_path().with_name(MUTEX_NAME)
- path.parent.mkdir(parents=True, exist_ok=True)
- fd = os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_BINARY", 0), 0o644)
- try:
- deadline = time.monotonic() + wait_s
- while not update_lock._try_lock(fd):
- if time.monotonic() > deadline:
- raise RecordBusy(f"{path} is held by another process or thread")
- time.sleep(0.05)
- _mutex_held.depth = 1
- try:
- yield
- finally:
- _mutex_held.depth = 0
- update_lock._unlock(fd)
- finally:
- os.close(fd)
- def _git_executable() -> str:
- """The git ``hermes update`` runs, found without installing anything: PATH's, else the copy
- install.ps1 staged in PM's store (``_subprocess_compat.expose_pm_git`` puts it on PATH only
- inside the updater). A bare ``git`` on such an install dies with WinError 2, so every gate
- read would fail and the paused gateways would never be restarted."""
- found = shutil.which("git")
- if found:
- return found
- with suppress(Exception): # health: allow BLE001 -- no PM, no store, no Windows git package: PATH's answer ("git") stands
- import pm
- from pm import paths
- from pm.lock import Lockfile
- recorded = pm.installed_package("git", allow_outdated=True)
- if recorded is not None and recorded.binary is not None and recorded.binary.is_file():
- return str(recorded.binary)
- package, target = pm.get_package("git"), pm.current_target()
- version = Lockfile(paths.lockfile_path()).version("git")
- staged = package.binary(paths.store_root() / package.store_entry(version, target), target)
- if staged is not None and staged.is_file():
- return str(staged)
- return "git"
- def _git(root: Path, *args: str) -> subprocess.CompletedProcess | None:
- """A read-only git query in the updater's custody (``update_custody.run_git``: job-bound on
- Windows inside an update, custody config in argv); ``None`` when git cannot run."""
- from hermes_cli.update_custody import run_git
- try:
- return run_git([_git_executable(), "-C", str(root)], args, capture_output=True, text=True,
- encoding="utf-8", errors="replace", stdin=subprocess.DEVNULL, timeout=30, check=False)
- except (OSError, ValueError, subprocess.SubprocessError):
- return None
- def head_sha(root: Path) -> str | None:
- result = _git(root, "rev-parse", "--verify", "HEAD")
- if result is None or result.returncode != 0:
- return None
- return result.stdout.strip() or None
- def tracked_changes(root: Path) -> list[str] | None:
- """Tracked paths that differ from HEAD (index or worktree); ``None`` when git cannot say."""
- result = _git(root, "status", "--porcelain=v1", "-z", "--untracked-files=no")
- if result is None or result.returncode != 0:
- return None
- paths = set()
- entries = result.stdout.split("\0")
- index = 0
- while index < len(entries):
- entry = entries[index]
- index += 1
- if len(entry) < 4:
- continue
- paths.add(entry[3:])
- if entry[0] in "RC": # rename/copy: the source path follows as its own entry
- index += 1
- return sorted(paths)
- def stamp_tree(token: dict, root: Path | None = None) -> dict:
- """Record what "the tree before this update" was, for the whole-tree gate: one baseline per
- pause (idempotent for this token's ``pause_id``), next to the baselines of the pauses it absorbed."""
- root = root or install_root()
- pause_id = token.setdefault("pause_id", uuid.uuid4().hex)
- baselines = token.setdefault("baselines", [])
- if not any(b.get("pause_id") == pause_id for b in baselines) and (state := tree_state(root)):
- baselines.append({"pause_id": pause_id, **state})
- return token
- def tree_state(root: Path) -> dict | None:
- """HEAD, the tracked changes and their bytes now; ``None`` when *root* is not a git checkout."""
- if not (root / ".git").exists():
- return None
- dirty = tracked_changes(root)
- # The bytes, not just the names: an autostashed edit and git's half-written bytes share a path.
- return {"pre_sha": head_sha(root), "dirty_at_pause": dirty,
- "dirty_digests": {path: _digest(root / path) for path in dirty or []}}
- def _digest(path: Path) -> str | None:
- try:
- return hashlib.sha256(path.read_bytes()).hexdigest()
- except OSError: # deleted (or unreadable) at pause: it must read the same way to count as unchanged
- return None
- def mark_move(token: dict | None, target: str) -> None:
- """Persist, BEFORE git writes the tree, the commit this run's checkout move goes to, on this
- pause's own baseline: an unmoved HEAD is later judged against what THAT move could write, never
- against refs a later fetch replaced. Raises when it cannot be recorded: the move must not run."""
- if not token or not token.get("pause_id") or not target:
- return
- for baseline in token.get("baselines") or []:
- if baseline.get("pause_id") == token["pause_id"]:
- baseline["move_targets"] = sorted({*baseline.get("move_targets", []), target})
- write({**token, "resume_needed": True})
- _MAX_MOVE_TARGETS = 16
- def _paths_between(root: Path, targets) -> set[str] | None:
- """Tracked paths that differ between HEAD and any of *targets* — all a fast-forward, reset or
- merge toward them can write; ``None`` when git cannot say."""
- paths: set[str] = set()
- for target in sorted(targets):
- diff = _git(root, "diff", "--name-only", "-z", "--no-renames", "HEAD", target, "--")
- if diff is None or diff.returncode != 0:
- return None
- paths.update(filter(None, diff.stdout.split("\0")))
- return paths
- def _paths_a_move_could_write(root: Path) -> set[str] | None:
- """For a pause with no recorded move (:func:`mark_move`: none attempted, or a record from before
- it was kept): the paths between HEAD and a commit the checkout fetched (``FETCH_HEAD``) or tracks
- (``refs/remotes``). ``None`` when git cannot say or nothing was fetched (target unknown)."""
- where = _git(root, "rev-parse", "--git-path", "FETCH_HEAD")
- refs = _git(root, "for-each-ref", "--format=%(objectname)", "refs/remotes")
- if where is None or where.returncode != 0 or refs is None or refs.returncode != 0:
- return None
- try:
- fetched = (root / where.stdout.strip()).read_text(encoding="utf-8-sig", errors="replace").split("\n")
- except FileNotFoundError:
- fetched = []
- except OSError:
- return None
- targets = {line.split()[0] for line in fetched if line.split()} | set(refs.stdout.split())
- if not targets or len(targets) > _MAX_MOVE_TARGETS:
- return None
- return _paths_between(root, targets)
- def _seen_at_pause(root: Path, baseline: dict, path: str) -> bool:
- """*path* is as the pause saw it: dirty then, with the same bytes now (a baseline from before
- digests were kept has only the name)."""
- if path not in (baseline.get("dirty_at_pause") or []):
- return False
- digests = baseline.get("dirty_digests")
- return digests is None or digests.get(path) == _digest(root / path)
- def _half_written(root: Path, changes: list[str], at_head: list[dict]) -> list[str]:
- """Tracked changes at an unmoved HEAD that git may have left half-written, for the first
- baseline that has any. A change the pause did not see counts only on a path the update's move
- could write: a build/sync step (or the user) rewriting any other tracked file is not git's
- doing, and holding the set for it would keep gateways stopped until someone cleans the file.
- The move is the one recorded before it ran, so a later fetch never narrows it; with none
- recorded the fetched refs stand in, and when even those are unknown every unseen change counts."""
- for baseline in at_head:
- unexpected = sorted(path for path in changes if not _seen_at_pause(root, baseline, path))
- if not unexpected:
- continue
- # move_paths: files a step rewrote toward this HEAD (stash push/apply, reset, checkout --).
- targets, paths = baseline.get("move_targets"), baseline.get("move_paths")
- writable = _paths_between(root, targets) if targets else set() if paths else _paths_a_move_could_write(root)
- torn = unexpected if writable is None else [path for path in unexpected if path in {*writable, *(paths or ())}]
- if torn:
- return torn
- return []
- def torn_by_move(root: Path, found: dict, changes: list[str] | None) -> list[str] | None:
- """Paths a move from *found*'s HEAD to the current one wrote that do not hold the current
- commit's bytes: tracked changes on them that were not already there, byte for byte, when the
- move started. ``None`` when git cannot say."""
- writable = _paths_between(root, [found["pre_sha"]]) if found.get("pre_sha") else None
- if writable is None or changes is None:
- return None
- return sorted(path for path in changes if path in writable and not _seen_at_pause(root, found, path))
- def _landed_torn(root: Path, head: str, changes: list[str], landed: list[dict]) -> list[str]:
- """Tracked changes at a HEAD a recorded move reached that are not that commit's bytes. git can
- move HEAD past a file it failed to write (``checkout`` exits 0 after "unable to unlink old"),
- so a moved HEAD vouches for nothing by itself. The verdict the updater took the moment git
- returned stands while those paths stay changed: a later dependency sync or stash restore
- rewriting a file is not the move's doing. A move the updater died before judging is judged now."""
- for baseline in landed:
- verdict = (baseline.get("landed") or {}).get(head)
- if verdict is None:
- verdict = torn_by_move(root, baseline, changes)
- torn = list(changes) if verdict is None else [path for path in verdict if path in changes]
- if torn:
- return torn
- return []
- def _deps_hold_resume(root: Path) -> bool:
- """At a moved HEAD: are stale dependencies a reason to keep the paused set stopped?
- Only where waiting converges. A self-managed install (``updateMechanism: self``) syncs its
- dependencies at the start of every launch (``venv_sync.prepare_launch``), before startup
- recovery runs, so a deferred set comes back on the next launch. Anywhere else nothing a launch
- does makes them current (a developer checkout owns its own venv): the set resumes now, as it
- did before the pause was durable, and a currency the probe cannot establish never holds it
- either — the restarted gateway's own launch syncs or reports the remedy.
- """
- if (os.environ.get("HERMES_DISABLE_LAZY_INSTALLS", "").lower() in ("1", "true", "yes")
- or not (root / ".git").exists() or not (root / "pyproject.toml").is_file()):
- return False # prepare_launch skips the sync: waiting would never end
- try:
- from hermes_cli.steward import read_install_stamp
- if read_install_stamp(root).get("updateMechanism") != "self":
- return False
- import pm
- return not pm.venv_is_current(project_root=root)
- except Exception: # health: allow BLE001 -- unknown currency must not strand the set: resume, the gateway's own launch syncs or names the remedy
- return False
- def _paused_on_unreadable_head(token: dict) -> bool:
- """Every pause in the set already found HEAD unreadable and none bound a tree move since.
- The gateways were serving exactly this tree when they stopped, and no step of the update wrote
- a file (``_checkout_move`` records its targets/paths before git writes), so restarting them
- leaves the tree as they knew it. Without this a ``.git`` git rejects (an empty ``HEAD``) kept
- the set stopped on every later launch, since no update can fetch until the checkout is repaired.
- """
- baselines = token.get("baselines") or []
- return bool(baselines) and all(
- b.get("pre_sha") is None and not (b.get("move_targets") or b.get("move_paths") or b.get("landed"))
- for b in baselines)
- def tree_is_whole(token: dict, root: Path | None = None) -> tuple[bool, str]:
- """May paused gateways start on this checkout now? ``(verdict, reason when not)``."""
- from hermes_cli._early_recovery import interrupted_pull_marker
- root = root or install_root()
- if (root / ".git").exists():
- if interrupted_pull_marker(root).exists():
- return False, "the checkout is mid-pull (interrupted-pull marker present)"
- head = head_sha(root)
- if head is None:
- if _paused_on_unreadable_head(token):
- return True, ""
- return False, "the checkout HEAD is unreadable"
- baselines = token.get("baselines") or []
- at_head = [b for b in baselines if b.get("pre_sha") == head]
- # HEAD reached by a move these pauses recorded: judged against that commit's tree.
- landed = [b for b in baselines if b.get("pre_sha") != head
- and (head in (b.get("landed") or {}) or head in (b.get("move_targets") or []))]
- if at_head or landed:
- # HEAD never moved for the at_head pauses: a tracked change one of them did not see is
- # git's half-written checkout. Each is judged on its own set — a killed run that moved
- # HEAD elsewhere vouches for nothing here, and this run's set never certifies an older one's.
- changes = tracked_changes(root)
- if changes is None:
- return False, "git cannot read the checkout state"
- torn = _half_written(root, changes, at_head) or _landed_torn(root, head, changes, landed)
- if torn:
- return False, f"the checkout has {len(torn)} file(s) git left half-written (e.g. {torn[0]})"
- if at_head:
- return True, ""
- if _deps_hold_resume(root):
- return False, "dependencies are not current for the updated code yet"
- return True, ""
- def _atomic_write(path: Path, body: dict) -> None:
- path.parent.mkdir(parents=True, exist_ok=True)
- tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
- try:
- with open(tmp, "w", encoding="utf-8") as fh:
- json.dump(body, fh, indent=1, sort_keys=True)
- fh.flush()
- os.fsync(fh.fileno())
- os.replace(tmp, path)
- except BaseException:
- # A refused replace (a Windows reader without FILE_SHARE_DELETE) or a failed write must
- # not leave the half-published temp beside the record; the error itself still propagates.
- with suppress(OSError):
- tmp.unlink()
- raise
- def read(path: Path | None = None) -> dict | None:
- """The saved body; ``None`` only when the file is absent. One that exists but cannot be read
- (a Windows sharing violation, an AV scanner) or does not parse raises :class:`RecordUnknown`:
- it may be the newest copy of an obligation (its partial progress) or the only one, so it never
- lets an older copy execute nor a new pause replace it. Callers fail closed and retry later."""
- path = path or record_path()
- try:
- text = path.read_text(encoding="utf-8-sig")
- except FileNotFoundError:
- return None
- except OSError as exc:
- raise RecordUnknown(f"cannot read the paused-gateway record {path}: {exc}") from exc
- try:
- body = json.loads(text)
- except ValueError:
- body = None
- if not isinstance(body, dict) or not isinstance(body.get("token"), dict):
- raise RecordUnknown(f"the paused-gateway record {path} is malformed; fix or remove it to resume")
- return body
- UNOWNED = {"pid": 0, "ct": None}
- def write(token: dict, *, owner: dict | None = None, path: Path | None = None) -> None:
- """Persist *token*. The owner of the same pause is kept; another pause's orphaned set is merged
- into *token* (in place, so this run's resume brings it back); a live one refuses."""
- path = path or record_path()
- with _mutex():
- existing = read(path)
- same = existing is not None and existing["token"].get("pause_id") == token.get("pause_id")
- if existing is not None and not same:
- if existing.get("install_root") != str(install_root()):
- raise RecordConflict(f"{path} holds gateways paused for another checkout ({existing.get('install_root')})")
- if identity_is_live(existing.get("owner")):
- raise RecordConflict(f"{path} holds gateways paused by live process {existing['owner'].get('pid')}")
- merge_into(token, drop_never_stopped(dict(existing["token"])))
- if owner is None:
- owner = existing["owner"] if same and existing is not None else identity()
- _atomic_write(path, {"schema": 1, "owner": owner, "install_root": str(install_root()), "token": token})
- def mark_stop_requested(token: dict, pids, markers: dict | None = None) -> None:
- """Persist, BEFORE the first stop request, the stops this update intends and where each mapped
- gateway's request (its planned-stop marker) will be written. Intent is not a request: a live
- process counts as asked only once the producer or consumer checkpoints it, or while the
- valid request this update wrote is still on disk (:func:`_asked`)."""
- token["stop_requested"] = sorted({*map(str, token.get("stop_requested") or []), *(str(int(p)) for p in pids)})
- token["stop_sent"] = sorted(map(str, token.get("stop_sent") or []))
- token["stopper_pid"] = os.getpid()
- token["stop_markers"] = {**(token.get("stop_markers") or {}), **{str(int(p)): str(m) for p, m in (markers or {}).items()}}
- write(token, owner=identity())
- def mark_stop_sent(token: dict, pid) -> None:
- """Persist that *pid*'s stop request was issued (its planned-stop marker is on disk, or the
- POSIX pause records it just before its socket request): from here a live *pid* is draining and
- keeps its restart debt until it exits. Best effort: the marker itself is evidence until this
- lands; its consumer checkpoints before unlinking it."""
- token["stop_sent"] = sorted({*map(str, token.get("stop_sent") or []), str(int(pid))})
- try:
- write(token, owner=identity())
- except OSError as exc:
- print(f" ⚠ Could not record the stop request for gateway {pid}: {exc}")
- def mark_stop_consumed(path: Path, marker: dict) -> None:
- """Checkpoint a validated gateway consume BEFORE its marker disappears.
- Called under the pause mutex by the consumer. Preserve the holder's identity;
- this is evidence of a stop, not a transfer of the updater's custody.
- """
- from gateway.status import _same_hermes_home
- pid = str(marker["target_pid"])
- for src, body in _files(record_path()):
- token = body["token"]
- recorded_path = (token.get("stop_markers") or {}).get(pid)
- if (str(token.get("stopper_pid")) != str(marker.get("stopper_pid"))
- or pid not in (token.get("stop_requested") or [])
- or not recorded_path or not _same_hermes_home(recorded_path, path)):
- continue
- token["stop_sent"] = sorted({*map(str, token.get("stop_sent") or []), pid})
- _atomic_write(src, body) # Failure leaves the request intact for recovery.
- def _accepted_path(marker_path: Path) -> Path:
- return marker_path.with_name(marker_path.name + ".accepted")
- def mark_stop_accepted(path: Path, marker: dict) -> None:
- """The consumer's receipt when it cannot checkpoint into the record (busy mutex, refused
- replace): this incarnation accepted the request and drains until it exits, however long past
- the request's TTL. Written beside the marker, in the gateway's own home, never into the record.
- A home that refuses the new file (an ACL, a read-only directory) still lets the kept request be
- rewritten in place: stamped ``accepted`` it is the receipt. Torn mid-write it reads malformed,
- which recovery holds as unknown, never as "not asked". The consumer accepts the stop either way."""
- try:
- _atomic_write(_accepted_path(path), dict(marker))
- return
- except OSError as exc:
- refused = exc
- try:
- with open(path, "r+", encoding="utf-8") as fh: # r+: never creates a request that was not there
- json.dump({**marker, "accepted": True}, fh)
- fh.truncate()
- fh.flush()
- os.fsync(fh.fileno())
- except OSError as exc:
- print(f" ⚠ Could not record the accepted stop request {path}: {refused}; {exc}", file=sys.stderr)
- def discharge(token: dict, path: Path | None = None) -> None:
- """Remove the record only while it still names this pause (decided under the mutex)."""
- path = path or record_path()
- with _mutex():
- existing = read(path)
- if existing is not None and existing["token"].get("pause_id") == token.get("pause_id"):
- _retire(path, [(path, existing)])
- def sync(token: dict) -> None:
- """Mirror the token after a resume attempt: done → delete; anything still owed → rewrite."""
- if not token.get("pause_id") or token.get("recovery"):
- return # a recovering launch keeps what it owes in its own claim (_resume_claimed)
- try:
- if token.get("resume_needed") or token.get("resume_deferred"):
- owed = {**token, "resume_needed": True}
- owed.pop("resume_deferred", None)
- write(owed)
- else:
- discharge(token)
- except OSError as exc:
- print(f" ⚠ Could not update the paused-gateway record {record_path()}: {exc}")
- def _live_update_elsewhere() -> bool:
- """A live update other than the one this process belongs to. The update tree holding the
- checkout lock (this process plus the partner whose marker it adopted) is not "elsewhere"."""
- from hermes_cli import update_lock
- root = install_root()
- return not update_lock.holds_checkout_lock(root) and update_lock.update_in_progress(root)
- def _claims(path: Path) -> list[Path]:
- return sorted(path.parent.glob(f"{glob.escape(path.name)}.*.claim"))
- def _retired_path(path: Path) -> Path:
- return path.with_suffix(".retired")
- def _retired(path: Path) -> set[str]:
- """The completed obligation ids (call under the mutex). Only an absent list is empty: one that
- exists but cannot be read raises :class:`RetiredUnknown`, so a completed obligation is never
- claimed again and the list is never rewritten from a read that failed."""
- target = _retired_path(path)
- try:
- text = target.read_text(encoding="utf-8-sig", errors="replace")
- except FileNotFoundError:
- return set()
- except OSError as exc: # a refused read (sharing violation, AV scanner): unknown
- raise RetiredUnknown(f"cannot read the retired paused-gateway list {target}: {exc}") from exc
- try:
- ids = json.loads(text).get("ids")
- except (ValueError, AttributeError): # not JSON / not an object: malformed, like a non-list
- ids = None
- if not isinstance(ids, list):
- return _quarantine_retired(path, target)
- return {str(i) for i in ids}
- def _quarantine_retired(path: Path, target: Path) -> set[str]:
- """A retired list that reads but does not parse. While a record or claim of this checkout
- exists the list may be the only proof it is complete: unknown, fail closed. With none left it
- guards nothing, and keeping it would abort every Windows update: set it aside and go on."""
- if path.exists() or _claims(path):
- raise RetiredUnknown(f"the retired paused-gateway list {target} is malformed")
- corrupt = target.with_suffix(".corrupt")
- try:
- os.replace(target, corrupt)
- except OSError as exc:
- raise RetiredUnknown(f"cannot set aside the malformed retired paused-gateway list {target}: {exc}") from exc
- print(f" ⚠ Set aside a malformed paused-gateway history (no paused set refers to it): {corrupt}",
- file=sys.stderr)
- return set()
- def _retire(path: Path, carriers: list[tuple[Path, dict]]) -> None:
- """Complete the obligations *carriers* hold (call under the mutex): their ids (and every id
- they absorbed) go on the durable retired list FIRST; a failed read or write raises and deletes
- nothing. Then the files go; one that cannot be deleted is a redundant copy from here on."""
- ids = {str(i) for _src, body in carriers
- for i in [body["token"].get("pause_id"), *(body["token"].get("absorbed") or [])] if i}
- retired = _retired(path)
- if not ids <= retired:
- _atomic_write(_retired_path(path), {"schema": 1, "ids": sorted(retired | ids)})
- for src, _body in carriers:
- with suppress(OSError):
- src.unlink()
- def _holder(src: Path, body: dict) -> dict:
- """Who holds *src*: a record's owner, a claim's claimer (published with the claim itself)."""
- held_by = body.get("owner") if src.suffix == ".json" else body.get("claimer")
- return held_by if isinstance(held_by, dict) else UNOWNED
- def _files(path: Path) -> list[tuple[Path, dict]]:
- """This checkout's record and claims; one that cannot be read raises :class:`RecordUnknown`."""
- found = []
- for src in (path, *_claims(path)):
- body = read(src)
- if body is not None and body.get("install_root") == str(install_root()):
- found.append((src, body))
- return found
- def _redundant(found: list[tuple[Path, dict]], held: set[Path], retired: set[str] = frozenset()) -> set[Path]:
- """Files that only copy an obligation another file carries or that was completed: a source an
- update absorbed but died before retiring, one of two same-id files a claim transfer left (the
- furthest-progressed copy is kept), a retired id's leftover. Never a held file."""
- carriers: dict[str, Path] = {}
- for src, body in found:
- for oid in body["token"].get("absorbed") or []:
- carriers.setdefault(str(oid), src)
- redundant: set[Path] = set()
- by_id: dict[str, list[Path]] = {}
- for src, body in found:
- oid = str(body["token"].get("pause_id") or "")
- if not oid:
- continue
- if oid in retired or (oid in carriers and carriers[oid] != src):
- redundant.add(src)
- else:
- by_id.setdefault(oid, []).append(src)
- rev = {src: int(body.get("rev") or 0) for src, body in found}
- for copies in by_id.values():
- keep = next((s for s in copies if s in held), max(copies, key=lambda s: rev[s]))
- redundant.update(s for s in copies if s != keep)
- return redundant - held
- def _survey(path: Path) -> tuple[list[tuple[Path, dict]], set[Path], set[Path]]:
- found = _files(path)
- held = {src for src, body in found if identity_is_live(_holder(src, body))}
- return found, held, _redundant(found, held, _retired(path))
- def orphans(path: Path | None = None) -> list[tuple[Path, dict]]:
- """``(file, body)`` for every obligation of this checkout whose holder is dead — one file per
- obligation id; empty while another update is live (it adopts them itself). Surveyed under the
- mutex (a mutator replaces the retired list meanwhile); raises when the list cannot be read."""
- path = path or record_path()
- if not path.exists() and not _claims(path):
- return []
- with _mutex():
- found, held, redundant = _survey(path)
- found = [(src, body) for src, body in found if src not in held and src not in redundant]
- if not found or _live_update_elsewhere():
- return []
- return found
- def retire_redundant(path: Path | None = None) -> None:
- """Delete copies of obligations another file already carries (crash leftovers of a transfer)."""
- path = path or record_path()
- if not path.exists() and not _claims(path) and not _retired_path(path).exists():
- return
- with suppress(RecordBusy), _mutex():
- _found, _held, redundant = _survey(path)
- for src in redundant:
- with suppress(OSError):
- src.unlink()
- _prune_retired(path)
- def _prune_retired(path: Path) -> None:
- """Forget retired ids no file carries any more (a carrier that cannot be read raises first)."""
- retired = _retired(path)
- bodies = [body for body in map(read, (path, *_claims(path))) if body is not None]
- if not retired:
- return
- carried = {str(b["token"].get("pause_id")) for b in bodies}
- keep = retired & carried
- with suppress(OSError):
- if keep:
- _atomic_write(_retired_path(path), {"schema": 1, "ids": sorted(keep)})
- else:
- _retired_path(path).unlink()
- def claim(src: Path) -> tuple[Path, dict] | None:
- """Take *src* (an orphaned record or a dead claimer's claim) for this process; ``None`` when a
- concurrent launch won it or holds the mutex. The claim is published with our identity inside,
- then the source is retired; a crash between leaves a same-id copy :func:`_redundant` drops."""
- path = record_path()
- mine = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex[:8]}.claim")
- try:
- with _mutex():
- body = read(src)
- if body is None or body.get("install_root") != str(install_root()):
- return None
- _found, held, redundant = _survey(path)
- if src in held or src in redundant or src not in {s for s, _ in _found}:
- return None
- body["claimer"] = identity()
- body["rev"] = int(body.get("rev") or 0) + 1 # a copy left behind is never the newer one
- _atomic_write(mine, body)
- with suppress(OSError):
- src.unlink()
- except OSError:
- return None
- return mine, body
- def adopt_orphans() -> tuple[dict | None, list[Path]]:
- """For ``hermes update``: claim every orphaned pause and merge their sets. The caller records the
- merged set durably (its ``absorbed`` ids name the claims), then :func:`release_claims`."""
- retire_redundant()
- adopted, claims = None, []
- for src, _body in orphans():
- won = claim(src)
- if won is None:
- continue
- claims.append(won[0])
- with _mutex():
- body = read(won[0]) or won[1]
- adopted = merge_into(adopted, drop_never_stopped(dict(body["token"])))
- return adopted, claims
- def release_claims(claims: list[Path]) -> None:
- with suppress(RecordBusy), _mutex():
- for path in claims:
- with suppress(OSError):
- path.unlink()
- def record_pause(token: dict, adopted: dict | None, claims: list[Path]) -> dict:
- """Merge the adopted set, stamp the tree and persist under this process BEFORE the first stop.
- Publish then retire: a crash between leaves claims whose ids the record lists as absorbed."""
- if adopted is not None:
- merge_into(token, adopted)
- stamp_tree(token)
- token.setdefault("stop_requested", [])
- write(token, owner=identity())
- release_claims(claims)
- return token
- _CARRIED = ("pause_id", "baselines", "identities", "stop_requested", "stop_sent", "stopper_pid",
- "stop_markers", "absorbed")
- def finish_pause(token: dict, intended: dict, adopted: dict | None) -> dict:
- """Carry the recorded pause identity onto the final token and mirror it to disk."""
- if adopted is not None:
- merge_into(token, adopted)
- token.update({key: intended[key] for key in _CARRIED if key in intended})
- sync(token)
- return token
- def abandon_pause(intended: dict, adopted: dict | None) -> None:
- """This run's own stops were rolled back in-line: only an adopted orphan set is still owed,
- and it goes back on disk unowned for the next launch."""
- if adopted is None:
- discharge(intended)
- return
- carried = {key: intended[key] for key in ("pause_id", "baselines", "absorbed") if key in intended}
- write({**adopted, **carried}, owner=UNOWNED)
- def _live_pids(token: dict) -> set[str]:
- return {str(pid) for pid, ct in (token.get("identities") or {}).items()
- if identity_is_live({"pid": pid, "ct": ct})}
- def _without(token: dict, pids: set[str]) -> dict:
- token["profiles"] = {p: pid for p, pid in (token.get("profiles") or {}).items() if str(pid) not in pids}
- token["unmapped"] = [u for u in (token.get("unmapped") or []) if str(u.get("pid")) not in pids]
- if "posix_units" in token: # supervised POSIX entries (update_cmd_posix_pause) are judged like the rest
- token["posix_units"] = [u for u in token["posix_units"] or [] if str(u.get("pid")) not in pids]
- return token
- def _request_on_disk(token: dict, pid: str) -> bool | None:
- """The planned-stop marker this update's stopper wrote for *pid* is still on disk (the gateway's
- watcher has not consumed it yet): the request was issued even if the updater died before
- :func:`mark_stop_sent` — or the gateway's receipt of accepting it (:func:`mark_stop_accepted`).
- A marker naming another stopper (a user's ``hermes gateway stop``) is not. ``None``: one of them
- exists but cannot be read or parsed — the stop may have been accepted, so it is unknown."""
- path = (token.get("stop_markers") or {}).get(str(pid))
- if not path:
- return False
- from gateway.status import _PLANNED_STOP_MARKER_TTL_S
- # An unconsumed request expires; the receipt of an accepted one is evidence for the whole drain
- # (only the accepting incarnation is ever judged: a later one is not this entry's live process).
- verdicts = (_names_request(token, pid, Path(path), _PLANNED_STOP_MARKER_TTL_S),
- _names_request(token, pid, _accepted_path(Path(path)), None))
- return True if True in verdicts else None if None in verdicts else False
- def _names_request(token: dict, pid: str, path: Path, ttl_s: int | None) -> bool | None:
- """Does *path* hold this stopper's request for *pid*'s incarnation? Only an absent file or a
- valid one naming something else is ``False``; refused (a sharing violation, an AV scanner) or
- malformed is ``None``, unknown."""
- from gateway.status import _marker_is_stale, get_process_start_time
- try:
- marker = json.loads(path.read_text(encoding="utf-8-sig"))
- target, stopper = int(marker["target_pid"]), int(marker["stopper_pid"])
- except FileNotFoundError:
- return False
- except (OSError, ValueError, TypeError, KeyError):
- return None
- if (target != int(pid) or stopper != int(token.get("stopper_pid") or 0)
- or (ttl_s is not None and not marker.get("accepted")
- and _marker_is_stale(marker.get("written_at") or "", ttl_s))):
- return False
- expected, actual = marker.get("target_start_time"), get_process_start_time(int(pid))
- # Match the consumer's optional birth fingerprint, including unavailable clocks.
- return None in (expected, actual) or expected == actual
- def _asked(token: dict) -> tuple[set[str], set[str]]:
- """``(asked, unknown)``: pids this update actually asked to stop (recorded as sent, or whose
- request is still on disk), and pids whose request evidence exists but cannot be read."""
- sent = {str(p) for p in token.get("stop_sent") or []}
- verdicts = {str(p): _request_on_disk(token, str(p)) for p in token.get("stop_requested") or [] if str(p) not in sent}
- return sent | {p for p, v in verdicts.items() if v}, {p for p, v in verdicts.items() if v is None}
- def drop_never_stopped(token: dict) -> dict:
- """Drop entries whose recorded process is the same live incarnation AND was never actually asked
- to stop (the update died before issuing its request, even with the intent on disk): it is still
- serving, and a later exit — a user's ``hermes gateway stop`` included — is not this update's to
- undo. One it asked may be draining: it keeps its restart debt (:func:`split_draining`). The
- evidence is resolved here, once, into ``stop_sent`` (a merge carries it under another stopper).
- Evidence that cannot be read keeps the entry too (it may be draining) but resolves nothing: it is
- judged again next time. No stop record at all (a set from before stop tracking): nothing can be
- told apart."""
- if token.get("stop_requested") is None:
- return token
- asked, unknown = _asked(token)
- token["stop_sent"] = sorted(asked)
- return _without(token, _live_pids(token) - asked - unknown)
- def split_draining(token: dict) -> dict:
- """Remove and return ``{"profiles", "unmapped", "posix_units"}`` whose recorded process is still
- running: it was asked to stop and has not exited yet, so it can only be restarted once it has."""
- live = _live_pids(token)
- draining = {"profiles": {p: pid for p, pid in (token.get("profiles") or {}).items() if str(pid) in live},
- "unmapped": [u for u in (token.get("unmapped") or []) if str(u.get("pid")) in live],
- "posix_units": [u for u in (token.get("posix_units") or []) if str(u.get("pid")) in live]}
- _without(token, live)
- return draining
- def merge_into(token: dict | None, adopted: dict) -> dict:
- """Fold an orphaned pause into this update's token so its resume brings both sets back; its
- obligation id (and every id it absorbed) is recorded so a leftover copy is never resumed twice."""
- token = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []}
- token["resume_needed"] = True
- # Every pause keeps its own tree evidence: the adopted baseline stops this run's stamp from
- # certifying what a killed update left half-written as "dirty before the pause", and this
- # run's own baseline still guards the HEAD a killed update moved to.
- baselines = token.setdefault("baselines", [])
- baselines.extend(b for b in adopted.get("baselines") or [] if b not in baselines)
- absorbed = token.setdefault("absorbed", [])
- absorbed.extend(i for i in [adopted.get("pause_id"), *(adopted.get("absorbed") or [])] if i and i not in absorbed)
- profiles = token.setdefault("profiles", {})
- for name, pid in (adopted.get("profiles") or {}).items():
- profiles.setdefault(name, pid)
- unmapped = token.setdefault("unmapped", [])
- unmapped.extend(u for u in adopted.get("unmapped") or [] if u not in unmapped)
- identities = token.setdefault("identities", {})
- for pid, ct in (adopted.get("identities") or {}).items():
- identities.setdefault(pid, ct)
- if "stop_requested" in adopted or "stop_requested" in token:
- requested, sent = adopted.get("stop_requested"), adopted.get("stop_sent")
- if requested is None: # a set with no stop record keeps every live entry: all count as asked
- requested = sent = list(adopted.get("identities") or {})
- token["stop_requested"] = sorted({*map(str, token.get("stop_requested") or []), *map(str, requested)})
- token["stop_sent"] = sorted({*map(str, token.get("stop_sent") or []), *map(str, sent or [])})
- cold = token.setdefault("cold_start_profiles", {})
- for name, generation in (adopted.get("cold_start_profiles") or {}).items():
- cold.setdefault(name, generation)
- if adopted.get("cold_start_if_installed"):
- token["cold_start_if_installed"] = True
- if "attested_generation" in adopted:
- token.setdefault("attested_generation", adopted["attested_generation"])
- if adopted.get("platform"): # a POSIX set (update_cmd_posix_pause): its supervised units ride along
- token["platform"] = adopted["platform"]
- units = token.setdefault("posix_units", [])
- units.extend(u for u in adopted.get("posix_units") or [] if u not in units)
- services = token.setdefault("services", [])
- services.extend(s for s in adopted.get("services") or [] if s not in services)
- if services:
- token.setdefault("expected_services", []).extend(s for s in services if s not in token["expected_services"])
- token.setdefault("restarted_services", [])
- token.setdefault("service_profiles", {}).update(adopted.get("service_profiles") or {})
- return token
- def _has_work(token: dict) -> bool:
- return bool(token.get("profiles") or any(u.get("argv") for u in token.get("unmapped") or [])
- or token.get("services") or token.get("posix_units") or token.get("cold_start_if_installed") or token.get("cold_start_profiles"))
- def _resume_claimed(claim_path: Path, body: dict) -> None:
- """Resume under the checkout lock, so no update mutates the tree between the gate and the
- verified start; an update that took the checkout after our claim gets the set back untouched."""
- from hermes_cli.update_lock import UpdateLock
- fence = UpdateLock()
- if not fence.acquire_checkout(install_root()):
- with suppress(OSError), _mutex():
- _atomic_write(claim_path, {**(read(claim_path) or body), "claimer": UNOWNED})
- return
- try:
- _resume_fenced(claim_path, body)
- finally:
- fence.release()
- def _resume_fenced(claim_path: Path, body: dict) -> None:
- with _mutex():
- # A live consumer may have checkpointed after claim() returned its snapshot.
- body = read(claim_path) or body
- token = drop_never_stopped(dict(body["token"]))
- draining = split_draining(token)
- _hold_backing_off(token, draining)
- attempted = _relaunch_keys(token)
- token.update(resume_needed=True, recovery=True)
- try:
- if _has_work(token):
- print("→ Restarting gateway(s) paused by an interrupted `hermes update`...", file=sys.stderr)
- from hermes_cli.update_cmd_windows import _resume_windows_gateways_after_update
- # Whatever command this launch runs owns stdout (``--json`` output, a Desktop pipe).
- with redirect_stdout(sys.stderr):
- _resume_windows_gateways_after_update(token)
- else:
- token["resume_needed"] = False
- except Exception as exc: # health: allow BLE001 -- recovery boundary: any resume error is reported and the claim handed back (finally), never raised into the launch
- print(f" ⚠ Could not restart every paused gateway: {exc}. Run `hermes update` or "
- "`hermes gateway start`.", file=sys.stderr)
- finally:
- _back_off_unready(token, attempted, draining)
- _hand_back(claim_path, body, token, draining)
- # A recovering launch runs before every command (chat, doctor, a Desktop-spawned serve) and waits
- # for each relaunched gateway to become ready. One that does not is re-launched only after this
- # backoff (``relaunch_retry``), so an interactive command pays that wait at most once per window;
- # the debt itself stays owed, and ``hermes update`` (which drops the backoff) retries at once.
- _RELAUNCH_RETRY_BASE_S = 60.0
- _RELAUNCH_RETRY_MAX_S = 3600.0
- def _relaunch_keys(token: dict) -> set[str]:
- return ({f"profile:{name}" for name in token.get("profiles") or {}}
- | {f"unmapped:{u.get('pid')}" for u in token.get("unmapped") or [] if u.get("argv")}
- | {f"unit:{u.get('unit') or u.get('label')}" for u in token.get("posix_units") or []})
- def _hold_backing_off(token: dict, held: dict) -> None:
- """Move the profile/unmapped/supervised entries still inside their relaunch backoff from *token*
- into *held* (beside the draining ones): owed, not relaunched by this launch."""
- now = time.time()
- waiting = {key for key, state in (token.get("relaunch_retry") or {}).items() if now < float(state.get("next_at") or 0)}
- held["profiles"].update({n: p for n, p in token["profiles"].items() if f"profile:{n}" in waiting})
- held["unmapped"].extend(u for u in token["unmapped"] if u.get("argv") and f"unmapped:{u.get('pid')}" in waiting)
- token["profiles"] = {n: p for n, p in token["profiles"].items() if f"profile:{n}" not in waiting}
- token["unmapped"] = [u for u in token["unmapped"] if u not in held["unmapped"]]
- if units := token.get("posix_units"):
- held["posix_units"].extend(u for u in units if f"unit:{u.get('unit') or u.get('label')}" in waiting)
- token["posix_units"] = [u for u in units if u not in held["posix_units"]]
- def _back_off_unready(token: dict, attempted: set[str], held: dict) -> None:
- """Start or extend the backoff of each attempted entry the resume left owed (not ready, not
- launched); forget it for one that came back. A tree-gate deferral attempted nothing."""
- retry = dict(token.get("relaunch_retry") or {})
- if not token.get("resume_deferred"):
- unready = _relaunch_keys(token)
- for key in attempted:
- if key not in unready:
- retry.pop(key, None)
- continue
- attempts = int((retry.get(key) or {}).get("attempts") or 0) + 1
- retry[key] = {"attempts": attempts, "next_at": time.time() + min(
- _RELAUNCH_RETRY_BASE_S * 2 ** min(attempts - 1, 6), _RELAUNCH_RETRY_MAX_S)}
- owed = _relaunch_keys(token) | _relaunch_keys(held)
- token["relaunch_retry"] = {key: state for key, state in retry.items() if key in owed}
- def _hand_back(claim_path: Path, body: dict, token: dict, draining: dict) -> None:
- """Retire the claim when nothing is owed; else hand it back unowned for the next launch — this
- launch may live for hours (a chat) — which retries at once, except a relaunch that did not
- become ready (its backoff, :func:`_back_off_unready`). A draining process stays owed until it
- has exited and been restarted. A failed rewrite leaves our claimer line, dead once we exit."""
- token["profiles"] = {**(token.get("profiles") or {}), **draining["profiles"]}
- token["unmapped"] = [*(token.get("unmapped") or []), *draining["unmapped"]]
- if draining.get("posix_units"):
- token["posix_units"] = [*(token.get("posix_units") or []), *draining["posix_units"]]
- owed = bool(token.get("resume_needed") or token.get("resume_deferred") or draining["profiles"]
- or draining["unmapped"] or draining.get("posix_units"))
- with suppress(OSError), _mutex():
- if owed:
- kept = {key: value for key, value in token.items() if key not in ("recovery", "resume_deferred")}
- _atomic_write(claim_path, {**body, "token": {**kept, "resume_needed": True}, "claimer": UNOWNED})
- else:
- _retire(record_path(), [(claim_path, read(claim_path) or body)])
- def recover(argv: list[str] | None = None) -> None:
- """Start-of-run recovery: resume every orphaned pause. Never raises."""
- from hermes_cli._parser import command_argv
- command = command_argv(list(sys.argv[1:] if argv is None else argv))
- if command[:1] == ["update"] or command[:2] == ["gateway", "run"]:
- return # the update adopts it itself; a booting gateway must not block on its siblings
- try:
- retire_redundant()
- for src, _body in orphans():
- won = claim(src)
- if won is not None:
- _resume_claimed(*won)
- except Exception as exc: # health: allow BLE001 -- never brick a launch on recovery; the record stays for the next one
- print(f" ⚠ Paused-gateway recovery skipped: {exc}", file=sys.stderr)
|