gateway_supervised_restart.py 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149
  1. """``hermes gateway restart`` for a gateway whose supervisor Hermes did not install.
  2. A custom launchd agent / systemd unit / any KeepAlive-style manager running ``gateway run
  3. --external-supervisor`` owns the respawn. The manual fallback in ``_cmd_restart`` (SIGTERM, then a
  4. foreground ``run_gateway`` inside the restart CLI) stamps the CLI's own PID as the gateway, so every
  5. supervisor respawn refuses with "Gateway already running (PID <restart>)" and the gateway stays
  6. down until the restart process is killed (#110637). The gateway must instead exit back to its
  7. supervisor (SIGUSR1 drain), and success is a fresh supervised PID — never the bare exit.
  8. """
  9. from __future__ import annotations
  10. import sys
  11. from pathlib import Path
  12. from agent.deadline import poll_until
  13. # A custom KeepAlive supervisor keeps its own respawn interval (launchd's is ~once per 10s,
  14. # per LAUNCHD_SUPERVISION_VERIFY_TIMEOUT); 15s matches _wait_for_launchd_service_pid's budget.
  15. SUPERVISED_REPLACEMENT_VERIFY_TIMEOUT = 15.0
  16. def gateway_declares_external_supervisor(pid: int, home: Path | None = None) -> bool:
  17. """True when the running gateway ``pid`` was launched for an external supervisor.
  18. The supervisor is SELF-declared by the gateway from its launch context: the control socket
  19. ``identify`` answer (any launch supervisor other than ``"manual"`` — a custom systemd unit or
  20. launchd agent sets INVOCATION_ID / the XPC name, so the gateway answers ``systemd``/``launchd``
  21. even though ``_installed_service_kind_for`` saw no canonical unit, and the same supervisor owns
  22. the respawn), OR the ``--external-supervisor`` argv marker read live (same marker
  23. ``_prepare_profile_gateway_update_restart`` trusts), else the argv the gateway stamped into
  24. ``gateway_state.json`` when psutil cannot read the live command line.
  25. """
  26. if not pid or pid <= 1:
  27. return False
  28. from gateway.control_socket import identify_gateway, launch_supervisor
  29. from gateway.status import _get_process_hermes_home, read_runtime_status
  30. from hermes_cli.gateway import _capture_gateway_argv
  31. home = home or _get_process_hermes_home()
  32. identity = identify_gateway(home) or {}
  33. # Not ``identity["supervisor"]``: the runtime protocol reports a manual gateway as "none".
  34. if identity.get("pid") == pid and launch_supervisor(identity) not in (None, "manual"):
  35. return True
  36. argv = _capture_gateway_argv(pid)
  37. if argv is None:
  38. record = read_runtime_status(home / "gateway_state.json") or {}
  39. argv = record.get("argv") if record.get("pid") == pid else None
  40. return bool(argv) and "--external-supervisor" in argv
  41. def _wait_for_supervised_gateway_replacement(
  42. old_pid: int, timeout: float | None = None, *, poll_interval: float = 0.5
  43. ) -> int | None:
  44. """Poll the pidfile until the supervisor's replacement gateway registers a fresh PID.
  45. A graceful SIGUSR1 exit only proves the old process left — an unloaded, broken, or
  46. stopped-retrying supervisor leaves the gateway down. Custom-supervisor counterpart of
  47. ``_wait_for_launchd_service_pid``: the label is invisible to launchctl queries, so identity
  48. comes from ``get_running_pid``'s lock+PID liveness verification and freshness from ``!= old_pid``.
  49. Returns the fresh PID, or None once ``timeout`` passes.
  50. """
  51. from gateway.status import get_running_pid
  52. if timeout is None:
  53. timeout = SUPERVISED_REPLACEMENT_VERIFY_TIMEOUT
  54. def _fresh_pid() -> int | None:
  55. pid = get_running_pid()
  56. return pid if pid is not None and pid > 0 and pid != old_pid else None
  57. return poll_until(_fresh_pid, max(timeout, 0.5), poll_interval)
  58. def restart_externally_supervised_gateway(supervised_pid: int) -> None:
  59. """Hand ``supervised_pid`` back to its supervisor (SIGUSR1 drain) and report the fresh PID.
  60. Never falls through to SIGTERM + foreground run on either failure branch: that would
  61. SIGTERM a KeepAlive-armed process and stamp this CLI's PID, recreating the competing-owner
  62. wedge (#110637). A broken/unloaded supervisor surfaces as exit 1, not a success printed over
  63. a dead gateway (the contract ``launchd_restart`` enforces via ``_wait_for_launchd_service_pid``).
  64. """
  65. from hermes_cli.gateway import _get_restart_exit_wait_budget, _graceful_restart_via_sigusr1, _print_lines
  66. wait_budget = _get_restart_exit_wait_budget()
  67. print(f"→ Restarting externally-supervised gateway (PID {supervised_pid}) — "
  68. f"draining in-flight runs (up to {wait_budget:.0f}s)...")
  69. if _graceful_restart_via_sigusr1(supervised_pid, wait_budget):
  70. replacement_pid = _wait_for_supervised_gateway_replacement(supervised_pid)
  71. if replacement_pid is not None:
  72. print()
  73. print(f"✓ Gateway relaunched by its supervisor (PID {replacement_pid})")
  74. return
  75. print("⚠ Supervisor did not relaunch the gateway after its graceful exit")
  76. else:
  77. print(f"⚠ Gateway did not exit within {wait_budget:.0f}s of SIGUSR1 (or the signal could not be sent)")
  78. _print_lines(
  79. "",
  80. "✗ Not stopping or foreground-running a supervisor-owned gateway.",
  81. " Check the supervisor (it may be unloaded, wedged, or stopped retrying),",
  82. " then rerun once it is healthy: hermes gateway restart",
  83. )
  84. sys.exit(1)
  85. def refuse_restart_of_service_managed_gateway(pid: int | None) -> None:
  86. """Refuse the manual restart fallback when systemd owns the gateway under ANY unit name.
  87. Canonical-unit checks miss pre-convention installs (``hermes.service``), so the fallback's
  88. stop + in-process ``run_gateway`` SIGKILLed a live service-managed gateway and spawned an
  89. unsupervised orphan in the caller's cgroup while the unit flapped against the stolen lock
  90. (#126474). Reuses the dashboard's MainPID-verified lookup: a bare ``.service`` cgroup alone
  91. (the caller itself started under some unit) never proves ownership.
  92. """
  93. if not pid or pid <= 1:
  94. return
  95. from hermes_cli import main_dashboard as _dash
  96. owner = _dash._systemd_owner_for_pid(pid)
  97. if owner is None:
  98. return
  99. unit, scope = owner
  100. cmds = _dash._systemd_restart_commands(unit, scope)
  101. from hermes_cli.gateway import _print_lines
  102. _print_lines(
  103. "",
  104. f"✗ Gateway (PID {pid}) is managed by systemd unit {unit}.",
  105. " `hermes gateway restart` cannot restart it from here: the fallback would",
  106. " kill a live gateway and spawn an unsupervised replacement in this shell's",
  107. " cgroup while the unit keeps failing against the stolen lock.",
  108. " Restart it through its unit instead:",
  109. *(f" {c}" for c in cmds),
  110. " (Pre-convention `hermes.service` installs: `hermes gateway migrate-legacy`",
  111. " removes the legacy unit so the canonical one can own the gateway.)",
  112. )
  113. sys.exit(1)
  114. def start_restart_replacement(args, *, force: bool) -> None:
  115. """Start the manual fallback's replacement once the old owner is gone.
  116. ``--detach`` (Desktop's bounded, piped client; the updater's recovery call) starts it the way
  117. ``gateway ensure`` starts an absent owner. In the foreground that caller became the long-lived
  118. gateway it only meant to restart: a child its own timeout killed, holding the app's quit open.
  119. """
  120. from hermes_cli import gateway as gw
  121. if getattr(args, "detach", False):
  122. from hermes_cli.gateway_runtime_start import start_detached_replacement
  123. sys.exit(start_detached_replacement(gw.get_hermes_home(), force=force))
  124. gw.run_gateway(verbose=0, force=force)