runtime_downgrade_exit.py 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104
  1. """Stop a gateway whose checkout was moved back to a release that predates the gateway runtime.
  2. Downgrading is unsupported, but a manual ``git checkout <older release>`` under a running gateway
  3. leaves this process serving every local session from code the tree no longer has. Older clients
  4. cannot attach to it, and older ``hermes gateway run`` refuses because this process still holds the
  5. profile. Nothing on the older side knows how to stop it. So the gateway watches its own tree, using the boot
  6. revision ``gateway.code_skew`` records. Once the checkout has provably moved (two consecutive
  7. reads, never while an update holds the tree) to code whose state-store modules lack the runtime
  8. ledger, it logs why and stops with the normal drain (a hard exit if that stop fails on the older
  9. tree). The stop is a plain stop, never a restart of this process, but it exits with the
  10. service-restart code (75): systemd relaunches on any exit, while the launchd plist's
  11. ``KeepAlive.SuccessfulExit=false`` parks exit 0 and would leave the gateway down for good. Either
  12. supervisor then starts the older release's own gateway from the tree on disk. Every uncertain read
  13. keeps it up.
  14. """
  15. from __future__ import annotations
  16. import asyncio
  17. import logging
  18. import os
  19. from pathlib import Path
  20. from typing import Callable, Optional
  21. from gateway.restart import GATEWAY_SERVICE_RESTART_EXIT_CODE
  22. logger = logging.getLogger(__name__)
  23. POLL_S = 30.0
  24. _CONFIRMATIONS = 2
  25. _RUNTIME_SCHEMA_MARKER = "CREATE TABLE IF NOT EXISTS session_admissions"
  26. _PROJECT_ROOT = Path(__file__).resolve().parent.parent
  27. def tree_predates_runtime(root: Path = _PROJECT_ROOT) -> bool:
  28. """True only when the checkout's state-store modules are readable and none defines the ledger."""
  29. try:
  30. sources = [path.read_text(encoding="utf-8-sig", errors="replace") for path in root.glob("hermes_state*.py")]
  31. except OSError:
  32. return False # mid-checkout or unreadable: proves nothing
  33. return bool(sources) and not any(_RUNTIME_SCHEMA_MARKER in text for text in sources)
  34. def downgrade_reason(skew, *, update_in_progress: bool, predates: Callable[[], bool]) -> Optional[str]:
  35. """The log line when this gateway must stop for a downgraded tree, else None."""
  36. if not skew or update_in_progress or not predates():
  37. return None
  38. return ("This gateway loaded {}, but its checkout is now at {}, a release that predates the gateway "
  39. "runtime. Downgrading is unsupported; stopping so the service manager starts the older "
  40. "release's own gateway.".format(*tuple(skew)))
  41. async def _stop_or_exit(runner, *, hard_exit: Callable[[int], None] = os._exit) -> None:
  42. """The normal drain, and a hard exit if it raises. The stop path lazily imports modules that
  43. the older tree on disk no longer has; a stop that raised never sets the shutdown event and has
  44. already disarmed its own watchdog, so without this the process idles on forever."""
  45. try:
  46. await runner.stop()
  47. except Exception: # health: allow BLE001 -- any failure of the stop on a downgraded tree ends the same way
  48. logger.exception("Gateway stop failed on the downgraded checkout; exiting now")
  49. hard_exit(1)
  50. async def runtime_downgrade_watcher(runner, *, poll_s: float = POLL_S, skew_fn=None, update_probe=None,
  51. predates=tree_predates_runtime, max_polls: Optional[int] = None) -> None:
  52. """Supervised gateway watcher: ``runner.stop()`` after ``_CONFIRMATIONS`` consecutive downgrade reads.
  53. The probes are bound here, at gateway start: a lazy import after the tree moved would load the
  54. older release's module into this process."""
  55. if skew_fn is None:
  56. from gateway.code_skew import detect_code_skew as skew_fn
  57. if update_probe is None:
  58. from hermes_cli.update_lock import update_in_progress as update_probe
  59. observe_skew: Callable = skew_fn
  60. observe_update: Callable[[], bool] = update_probe
  61. def _read() -> Optional[str]:
  62. try:
  63. busy = observe_update()
  64. except Exception: # health: allow BLE001 -- an unreadable update lock cannot prove the swap is over
  65. busy = True
  66. return downgrade_reason(observe_skew(), update_in_progress=busy, predates=predates)
  67. seen = polls = 0
  68. while getattr(runner, "_running", False) and (max_polls is None or polls < max_polls):
  69. polls += 1
  70. reason = await asyncio.to_thread(_read)
  71. seen = seen + 1 if reason else 0
  72. if seen >= _CONFIRMATIONS:
  73. logger.error("%s", reason)
  74. runner._exit_reason = reason
  75. # A confirmed downgrade past a free update lock means the tree on disk is
  76. # the operator's settled choice. Exit with the service-restart code so the
  77. # supervisor relaunches and the settled tree serves — systemd's
  78. # Restart=always already does this on any exit; the launchd plist
  79. # (KeepAlive SuccessfulExit=false) parks exit 0, which would leave the
  80. # gateway down until a human notices. The stop itself stays a plain stop:
  81. # the relaunch runs whatever release is on disk, not this process.
  82. runner._exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE
  83. # Detached, strongly held: stop() cancels every background task, this watcher included,
  84. # and a stop awaited from here would be cancelled with it (the #12875 class).
  85. runner._runtime_downgrade_stop = asyncio.get_running_loop().create_task(_stop_or_exit(runner))
  86. return
  87. await asyncio.sleep(poll_s)