gateway_lifecycle_guards.py 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129
  1. """Refusals that keep ``hermes gateway`` lifecycle verbs off gateways they must not take down.
  2. Two rules, one per failure that took a production gateway down:
  3. * **Foreign service definition.** A systemd unit / launchd plist pins the ``HERMES_HOME`` its gateway
  4. runs on. A lifecycle verb (start/stop/restart/uninstall) on a definition pinning ANOTHER home acts on
  5. another install's gateway: a ``HOME=<scratch>`` probe resolved the bare ``hermes-gateway`` name and ran
  6. ``systemctl --user stop`` on the account's real unit. Install/refresh already refused to WRITE such a
  7. definition (``gateway_service_owner``); the same module now refuses to OPERATE one:
  8. ``refuse_foreign_systemd_unit`` (salvaged from #141908, checked before any home sync) and
  9. ``refuse_foreign_launchd_plist``.
  10. * **Live sessions.** ``gateway stop``/``restart`` name the turns, cron jobs or background delegations the
  11. gateway is running and ask y/N before interrupting them; ``-y``/``--yes`` (alias ``--interrupt``)
  12. skips the question, and with no terminal to ask on they refuse. The in-band restart
  13. drains turns, but its stop still interrupts every async delegation (``interrupt_all``), and a stop
  14. ends the sessions outright.
  15. """
  16. from __future__ import annotations
  17. import sys
  18. from pathlib import Path
  19. from typing import Any, Optional
  20. INTERRUPT_FLAG = "--yes"
  21. def _live_work_from_state(home: Path) -> Optional[dict[str, Any]]:
  22. """``active_agents`` from a fresh ``gateway_state.json`` whose writer is alive (older gateways)."""
  23. from gateway.status import parse_active_agents, read_runtime_status, runtime_status_is_stale, runtime_status_pid_is_live
  24. record = read_runtime_status(home / "gateway_state.json")
  25. if not record or runtime_status_is_stale(record) or not runtime_status_pid_is_live(record):
  26. return None
  27. work = record.get("active_work")
  28. return {"active_agents": parse_active_agents(record.get("active_agents")),
  29. "active_work": work if isinstance(work, list) else []}
  30. def live_work(home: Path, profile: Optional[str] = None) -> Optional[dict[str, Any]]:
  31. """What the gateway serving ``home`` is running now: ``{"active_agents", "active_work"}``, or None
  32. when no gateway answers. The control socket answers live; the state file covers older gateways.
  33. ``profile`` narrows the answer to one served profile's work (the state file cannot, so it is skipped)."""
  34. from gateway.control_socket import query_gateway_control
  35. answer = query_gateway_control(home, "active-work", params={"profile": profile} if profile else None)
  36. if isinstance(answer, dict) and isinstance(answer.get("active_work"), list):
  37. return answer
  38. return None if profile else _live_work_from_state(home)
  39. def _candidate_homes(system: bool = False) -> list[Path]:
  40. """Homes whose gateway a stop/restart from here takes down, in order: the system unit's pinned home
  41. (``sudo`` strips HERMES_HOME, so this process sees root's), this home's own gateway (a ``--force``
  42. standalone one included), then the live host owner that serves it. Only READS the unit: a foreign
  43. unit is skipped here and refused by the systemd verb before it adopts any home."""
  44. from hermes_cli import gateway as gw, gateway_service_owner
  45. homes = [gw.get_hermes_home()]
  46. if (gw.supports_systemd_services() and gw._select_systemd_scope(system)
  47. and gateway_service_owner.foreign_systemd_unit_home(True) is None
  48. and (unit_home := gateway_service_owner.service_home_for_unit(gw.get_systemd_unit_path(system=True), True))
  49. not in homes):
  50. homes.insert(0, unit_home)
  51. owner = gw._host_multiplexer_for_all_verb()
  52. if owner is not None and Path(owner.home) not in homes:
  53. homes.append(Path(owner.home))
  54. return homes
  55. def refusal_lines(verb: str, homes: list[Path], *, flag_hint: str = f"hermes gateway {{verb}} {INTERRUPT_FLAG}",
  56. profile: Optional[str] = None) -> list[str]:
  57. """Refusal text for ``verb`` while the first answering gateway among ``homes`` has live work, else []."""
  58. for home in homes:
  59. work = live_work(home, profile)
  60. if work is not None:
  61. return _work_lines(verb, home, work, flag_hint)
  62. return []
  63. def _work_lines(verb: str, home: Path, work: dict[str, Any], flag_hint: str) -> list[str]:
  64. from hermes_cli.update_cmd_drain_report import describe_active_work_unit
  65. units = [u for u in work.get("active_work") or [] if isinstance(u, dict)]
  66. count = max(len(units), int(work.get("active_agents") or 0))
  67. if not count:
  68. return []
  69. lines = [f"Refusing to {verb} the gateway: {count} live session(s) would be interrupted."]
  70. lines += [f" • {describe_active_work_unit(u, home)}" for u in units]
  71. if not units:
  72. lines.append(f" • {count} active agent run(s) (this gateway does not name them)")
  73. lines += [" Let them finish, or interrupt them deliberately:",
  74. f" {flag_hint.format(verb=verb)} # interrupts {count} live session(s)"]
  75. return lines
  76. def refuse_while_sessions_live(verb: str, args, *, host_home: Optional[Path] = None, profile: Optional[str] = None) -> None:
  77. """Exit 1 naming the live work unless ``--interrupt`` was passed; with it, say what is interrupted.
  78. ``host_home``/``profile``: a served profile's stop/restart unserves only that profile in the host,
  79. so only its work counts."""
  80. if profile:
  81. hint = f"hermes -p {profile} gateway {{verb}} {INTERRUPT_FLAG}"
  82. # No answer (control socket down) proceeds: the park marker path exists for exactly that host.
  83. lines = refusal_lines(verb, [Path(host_home)], profile=profile, flag_hint=hint)
  84. else:
  85. lines = refusal_lines(verb, _candidate_homes(getattr(args, "system", False)))
  86. if not lines:
  87. return
  88. from hermes_cli.gateway import print_error, print_warning
  89. if getattr(args, "interrupt", False):
  90. print_warning(lines[0].replace(f"Refusing to {verb} the gateway", f"--yes: {verb} proceeds"))
  91. for line in lines[1:-2]:
  92. print(line)
  93. return
  94. if sys.stdin.isatty() and sys.stdout.isatty():
  95. from hermes_cli.cli_output import prompt_yes_no
  96. print_warning(lines[0].replace(f"Refusing to {verb} the gateway", f"{verb.capitalize()} the gateway"))
  97. for line in lines[1:-2]:
  98. print(line)
  99. if prompt_yes_no(f"Interrupt them and {verb} the gateway?", default=False):
  100. return
  101. print(f" Cancelled: the gateway was not {'stopped' if verb == 'stop' else 'restarted'}.")
  102. sys.exit(1)
  103. print_error(lines[0])
  104. for line in lines[1:]:
  105. print(line)
  106. sys.exit(1)