update_cmd_fleet_unit_budget.py 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149
  1. """systemd unit timing budgets for the update's gateway restarts (``update_cmd_fleet`` sibling).
  2. How long a restart client must wait (the unit's stop + start budgets) and how long an ``is-active``
  3. poller must outwait the unit's ``RestartSec`` cooldown. Both read unit properties through
  4. ``update_cmd_fleet._systemctl`` (late-imported: that is the seam tests patch).
  5. """
  6. from __future__ import annotations
  7. import subprocess
  8. import time
  9. from contextlib import suppress
  10. # poll() takes signed 32-bit milliseconds; keep headroom for rounding in communicate().
  11. _SYSTEMCTL_RESTART_TIMEOUT_MAX = (2**31 - 1) // 1000 - 1
  12. def _unit_stop_start_budgets(scope_cmd: list, svc_name: str) -> tuple[float, float]:
  13. """``(TimeoutStopUSec, TimeoutStartUSec)`` in seconds. Unknown or infinite limits use
  14. systemd's usual 90s per phase so automation stays bounded."""
  15. from gateway.shutdown_forensics import parse_systemd_duration_to_us
  16. from hermes_cli.update_cmd_fleet import _systemctl
  17. budgets = {"TimeoutStopUSec": 90.0, "TimeoutStartUSec": 90.0}
  18. try:
  19. show = _systemctl(
  20. scope_cmd + ["show", svc_name, "--property=TimeoutStopUSec,TimeoutStartUSec"],
  21. timeout=5,
  22. )
  23. except (FileNotFoundError, subprocess.TimeoutExpired):
  24. return budgets["TimeoutStopUSec"], budgets["TimeoutStartUSec"]
  25. if show.returncode == 0:
  26. for line in (show.stdout or "").splitlines():
  27. key, _, raw = line.partition("=")
  28. if key in budgets:
  29. # The shared parser returns None for infinity/unrecognized units.
  30. try:
  31. raw = raw.strip()
  32. duration = int(raw) if raw.isascii() and raw.isdigit() else parse_systemd_duration_to_us(raw)
  33. if duration is not None and duration > 0:
  34. budgets[key] = duration / 1_000_000
  35. except (ValueError, OverflowError):
  36. pass
  37. return budgets["TimeoutStopUSec"], budgets["TimeoutStartUSec"]
  38. def systemd_restart_timeout(scope_cmd: list, svc_name: str, *, start_only: bool = False) -> float:
  39. """Outwait the unit's stop + start budgets, not just the systemctl client.
  40. A client timeout does not cancel the manager's queued restart: ``run_unit_job`` then
  41. waits for that job instead of calling the restart failed.
  42. """
  43. stop, start = _unit_stop_start_budgets(scope_cmd, svc_name)
  44. return min((0.0 if start_only else stop) + start + 15.0, _SYSTEMCTL_RESTART_TIMEOUT_MAX)
  45. #: A wait on a unit's own job never runs past this, whatever its timeouts say.
  46. _UNIT_JOB_HARD_CAP_S = 900.0
  47. #: Exit status ``run_unit_job`` reports when systemd's job for the unit is still running.
  48. UNIT_JOB_STILL_RUNNING = 124
  49. _TRANSITIONAL_STATES = ("activating", "deactivating", "reloading", "refreshing")
  50. def unit_job_budget(scope_cmd: list, svc_name: str, verb: str) -> float:
  51. """How long systemd may legitimately take to finish *verb* on the unit: TimeoutStopUSec
  52. twice (the SIGTERM wait, then the final-SIGKILL wait systemd bounds by the same value; a
  53. cgroup child stuck in D state takes both), plus TimeoutStartUSec for a start, plus slack."""
  54. stop, start = _unit_stop_start_budgets(scope_cmd, svc_name)
  55. budget = 2 * stop + (start if verb in ("start", "restart") else 0.0) + 30.0
  56. return min(budget, _UNIT_JOB_HARD_CAP_S)
  57. def unit_job_state(scope_cmd: list, svc_name: str) -> dict[str, str]:
  58. """``ActiveState``/``SubState``/``MainPID``/``Job`` of the unit; ``{}`` when unreadable."""
  59. from hermes_cli.update_cmd_fleet import _systemctl
  60. try:
  61. show = _systemctl(scope_cmd + ["show", svc_name, "--property=ActiveState,SubState,MainPID,Job"], timeout=10)
  62. except (FileNotFoundError, subprocess.TimeoutExpired):
  63. return {}
  64. if show.returncode != 0:
  65. return {}
  66. return dict(line.partition("=")[::2] for line in (show.stdout or "").splitlines() if "=" in line)
  67. def _settled(state: dict[str, str]) -> bool:
  68. return state.get("ActiveState") not in (None, "", *_TRANSITIONAL_STATES) and not state.get("Job")
  69. def run_unit_job(manage_cmd: list, scope_cmd: list, verb: str, svc_name: str, *, timeout: float):
  70. """``systemctl <verb> <unit>``, where a client timeout is UNKNOWN, not failed.
  71. The timeout cancels nothing: systemd keeps running the job (a ``start`` queued behind a
  72. slow stop included), so judging it failed reports a restart that then happens, and a
  73. rollback ``start`` queues behind the stop and boots the unit later, unasked. Past the
  74. timeout this waits for the unit's job to settle, bounded by ``unit_job_budget`` from the
  75. call, and reports the settled state: exit 0 when the unit reached what *verb* asked for
  76. (``active`` for start/restart, stopped for stop), else ``UNIT_JOB_STILL_RUNNING`` while
  77. the job still runs, 1 when it settled elsewhere. A call that returns in time is untouched.
  78. """
  79. from hermes_cli.update_cmd_fleet import _systemctl
  80. started = time.monotonic()
  81. try:
  82. return _systemctl(manage_cmd + [verb, svc_name], timeout=timeout)
  83. except subprocess.TimeoutExpired as exc:
  84. argv = exc.cmd
  85. deadline = started + max(unit_job_budget(scope_cmd, svc_name, verb), timeout)
  86. state = unit_job_state(scope_cmd, svc_name)
  87. while not _settled(state) and time.monotonic() < deadline:
  88. time.sleep(1.0)
  89. state = unit_job_state(scope_cmd, svc_name)
  90. waited, active = time.monotonic() - started, state.get("ActiveState", "unknown")
  91. if not _settled(state):
  92. print(f" ⏳ systemctl {verb} {svc_name} timed out after {timeout:.0f}s; systemd's job is still "
  93. f"running after {waited:.0f}s ({active}/{state.get('SubState', '?')})")
  94. return subprocess.CompletedProcess(argv, UNIT_JOB_STILL_RUNNING, "", f"systemd's {verb} job still running")
  95. reached = active == "active" if verb in ("start", "restart") else active in ("inactive", "failed")
  96. print(f" ⏳ systemctl {verb} {svc_name} timed out after {timeout:.0f}s; systemd finished the job "
  97. f"after {waited:.0f}s ({active})")
  98. return subprocess.CompletedProcess(argv, 0 if reached else 1, "", "" if reached else f"unit settled {active}")
  99. _RESTART_SEC_UNITS = (("ms", 0.001), ("us", 0.000001), ("min", 60.0), ("s", 1.0))
  100. def service_restart_sec(scope_cmd_: list, svc_name_: str, default: float = 0.0) -> float:
  101. """Read the unit's ``RestartUSec`` in seconds. ``is-active`` pollers must wait
  102. >= RestartSec + slack or they give up *during* the cooldown and misreport."""
  103. from hermes_cli.update_cmd_fleet import _systemctl
  104. try:
  105. _show = _systemctl(scope_cmd_ + ["show", svc_name_, "--property=RestartUSec", "--value"], timeout=5)
  106. except (FileNotFoundError, subprocess.TimeoutExpired):
  107. return default
  108. raw = (_show.stdout or "").strip()
  109. # Values like "30s", "100ms", "1min 30s", "infinity"; on any miss return default.
  110. if not raw or raw == "infinity":
  111. return default
  112. total = 0.0
  113. matched = False
  114. for part in raw.split():
  115. for _suf, _mult in _RESTART_SEC_UNITS:
  116. if part.endswith(_suf):
  117. with suppress(ValueError):
  118. total += float(part[: -len(_suf)]) * _mult
  119. matched = True
  120. break
  121. return total if matched else default