update_fleet_scope.py 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185
  1. """Home scoping for ``hermes update``'s fleet restart (#93349).
  2. The restart phase enumerates ``hermes-gateway*``/``hermes-serve*`` units, ``ai.hermes.gateway*``
  3. LaunchAgents and every ``gateway run`` process on the host. Those are HOST-wide namespaces: a
  4. second Hermes install (another ``HERMES_HOME`` root under the same account, its own checkout and
  5. venv) shares them, and the update used to restart that install's gateway too — including the
  6. account's real ``hermes-gateway.service`` when a scratch home ran ``hermes update``.
  7. The fleet an update owns is the set of homes its plan inventories: the updating root plus every
  8. ``<root>/profiles/<name>``. Ownership is judged from what a runtime actually runs on — the live
  9. process environment (``HERMES_HOME``/``HOME``, via ``_hermes_home_for_pid``), the unit's declared
  10. ``Environment=`` or the plist's pinned ``HERMES_HOME`` — never from a unit label or an argv
  11. substring. Unknown ownership is left alone: a restart we cannot prove is ours is somebody else's
  12. outage.
  13. """
  14. from __future__ import annotations
  15. import shlex
  16. from contextlib import suppress
  17. from pathlib import Path
  18. def _resolved(path) -> Path | None:
  19. try:
  20. return Path(str(path)).expanduser().resolve()
  21. except (OSError, RuntimeError, ValueError):
  22. return None
  23. def update_scope_homes() -> set[Path]:
  24. """Resolved homes the running update owns: the invoking home, the install root and its profiles."""
  25. homes: set[Path] = set()
  26. with suppress(Exception):
  27. from hermes_constants import get_hermes_home
  28. if (home := _resolved(get_hermes_home())) is not None:
  29. homes.add(home)
  30. with suppress(Exception):
  31. from hermes_cli.update_receipt import _profile_homes
  32. for _profile, home in _profile_homes():
  33. if (resolved := _resolved(home)) is not None:
  34. homes.add(resolved)
  35. return homes
  36. def home_in_update_scope(home, scope: set[Path] | None = None) -> bool:
  37. """True when *home* (a path or string) is one of the homes this update owns."""
  38. if not home:
  39. return False
  40. resolved = _resolved(home)
  41. if resolved is None:
  42. return False
  43. return resolved in (update_scope_homes() if scope is None else scope)
  44. def _ledger_gateway_home(pid: int) -> str | None:
  45. """Home this install's spawn ledger records for gateway *pid* (``register_self`` at gateway boot),
  46. only for an exact live ``(pid, create_time)`` match. Proof for a gateway whose environment is
  47. unreadable (elevated, or another account's service) while its create time still is."""
  48. with suppress(Exception):
  49. from hermes_cli.process_identity import ledger_entries
  50. for entry in ledger_entries(verified_only=True):
  51. if entry.get("pid") == pid and entry.get("purpose") == "gateway":
  52. return entry.get("hermes_home") or None
  53. return None
  54. def gateway_pid_home(pid: int) -> str | None:
  55. """The home gateway *pid* runs on: its live environment, else the exact-birth ledger witness;
  56. ``None`` when neither proves one. Discovery, the Windows pause record and its readiness check
  57. share it, so the home discovery proved is the home the restart debt keeps."""
  58. from hermes_cli.dashboard_procs import _hermes_home_for_pid
  59. try:
  60. home = _hermes_home_for_pid(pid)
  61. except Exception: # health: allow BLE001 -- tri-state resolver: any environment-read failure is "unknown" and the ledger witness decides
  62. home = None
  63. return home or _ledger_gateway_home(pid)
  64. def gateway_pid_in_update_scope(pid: int, scope: set[Path] | None = None) -> bool | None:
  65. """Does gateway *pid* run on a home this update owns? ``None`` when its home cannot be read."""
  66. home = gateway_pid_home(pid)
  67. if home is None:
  68. return None
  69. return home_in_update_scope(home, scope)
  70. def partition_gateway_pids_by_scope(pids, scope: set[Path] | None = None) -> tuple[list[int], list[tuple[int, str | None]]]:
  71. """``(owned, foreign)`` split of *pids*; ``foreign`` pairs each PID with its home (None = unreadable)."""
  72. scope = update_scope_homes() if scope is None else scope
  73. owned: list[int] = []
  74. foreign: list[tuple[int, str | None]] = []
  75. for pid in pids:
  76. verdict = gateway_pid_in_update_scope(pid, scope)
  77. if verdict:
  78. owned.append(pid)
  79. else:
  80. home = None
  81. if verdict is False:
  82. with suppress(Exception):
  83. from hermes_cli.dashboard_procs import _hermes_home_for_pid
  84. home = _hermes_home_for_pid(pid)
  85. foreign.append((pid, home))
  86. return owned, foreign
  87. def systemd_unit_hermes_home(scope_cmd: list, svc_name: str) -> str | None:
  88. """Home the systemd unit *svc_name* runs on: its live MainPID's environment first, then the
  89. unit's declared ``Environment=HERMES_HOME``; for a user-scope unit that declares none, the
  90. user's own default home. ``None`` when nothing readable names a home."""
  91. from hermes_cli.update_cmd_fleet import _systemctl, _unit_main_pid
  92. pid = _unit_main_pid(scope_cmd, svc_name)
  93. if pid > 0:
  94. with suppress(Exception):
  95. from hermes_cli.dashboard_procs import _hermes_home_for_pid
  96. if (home := _hermes_home_for_pid(pid)) is not None:
  97. return home
  98. try:
  99. shown = _systemctl(list(scope_cmd) + ["show", svc_name, "--property=Environment", "--value"], timeout=10)
  100. except Exception:
  101. return None
  102. if getattr(shown, "returncode", 1) != 0:
  103. return None
  104. env_line = (getattr(shown, "stdout", "") or "").strip()
  105. try:
  106. tokens = shlex.split(env_line)
  107. except ValueError:
  108. tokens = env_line.split()
  109. for token in tokens:
  110. key, sep, value = token.partition("=")
  111. if sep and key == "HERMES_HOME" and value.strip():
  112. return value.strip()
  113. if "--user" in scope_cmd:
  114. return str(Path.home() / ".hermes")
  115. return None
  116. def systemd_unit_in_update_scope(scope_cmd: list, svc_name: str, scope: set[Path] | None = None) -> bool | None:
  117. """Does unit *svc_name* belong to this update? ``None`` = ownership unreadable (leave it alone)."""
  118. home = systemd_unit_hermes_home(scope_cmd, svc_name)
  119. if home is None:
  120. return None
  121. return home_in_update_scope(home, scope)
  122. def launchd_label_foreign_home(label: str, scope: set[Path] | None = None) -> str | None:
  123. """The HERMES_HOME a derived launchd label's installed plist pins when that home is NOT one of
  124. this update's — labels are account-global, so root B's default profile derives the same bare
  125. ``ai.hermes.gateway`` root A installed. ``None`` = ours, or no/unreadable plist (the locate step
  126. decides whether a job exists; only a proven foreign home is refused)."""
  127. import plistlib
  128. with suppress(Exception):
  129. from hermes_cli.gateway import get_launchd_plist_path
  130. plist_path = get_launchd_plist_path().with_name(f"{label}.plist")
  131. if not plist_path.exists():
  132. return None
  133. data = plistlib.loads(plist_path.read_bytes())
  134. pinned = str(data["EnvironmentVariables"]["HERMES_HOME"])
  135. return None if home_in_update_scope(pinned, scope) else pinned
  136. return None
  137. def describe_skipped_runtime(kind: str, name: str, home: str | None) -> str:
  138. """One notice line for a runtime the update leaves alone (foreign home or unreadable ownership)."""
  139. if home is None:
  140. return f" ↷ {name}: {kind} whose Hermes home could not be read — left alone (not restarted)"
  141. return f" ↷ {name}: {kind} of another Hermes home ({home}) — left alone (not restarted)"
  142. def announce_skipped_runtime(kind: str, name: str, home: str | None) -> None:
  143. """Print :func:`describe_skipped_runtime` once per update run, and name the runtime on its receipt.
  144. The pre-swap pause and the post-swap restart phase each scope the same host-wide gateway list,
  145. and the restart phase runs in a fresh completion process, so only the run's receipt (handed to
  146. that process) knows what the pause already announced. No open receipt: every call prints.
  147. """
  148. from hermes_cli import update_receipt
  149. entry = [kind, name, home]
  150. current = update_receipt._current.get()
  151. announced = list((current.data if current is not None else {}).get("runtimes_left_alone") or [])
  152. if entry in announced:
  153. return
  154. print(describe_skipped_runtime(kind, name, home))
  155. update_receipt.record_fact("runtimes_left_alone", [*announced, entry])