update_cmd_drain_report.py 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129
  1. """Name what a draining gateway is waiting on while ``hermes update`` blocks on it.
  2. The gateway's in-band restart (SIGUSR1 → ``request_restart``) defers ``stop()`` until in-flight
  3. work finishes, capped by ``agent.restart_after_turn_timeout`` (30 min by default). From the
  4. updater's side that was a bare "draining (up to 1875s)..." followed by silence, which reads as a
  5. hung update. The gateway publishes each unit it is holding for in ``gateway_state.json``
  6. (``active_work``, written by ``GatewayShutdownMixin._describe_active_work``); this module turns
  7. that into progress lines.
  8. """
  9. from __future__ import annotations
  10. import time
  11. from pathlib import Path
  12. from typing import Callable, Optional
  13. # Progress cadence: the gateway refreshes ``active_work`` every 30s; printing faster only repeats it.
  14. DRAIN_REPORT_INTERVAL_S = 30.0
  15. def _fmt_elapsed(seconds: object) -> str:
  16. try:
  17. total = int(float(seconds)) # type: ignore[arg-type]
  18. except (TypeError, ValueError):
  19. return "?"
  20. return f"{total // 60}m{total % 60:02d}s" if total >= 60 else f"{total}s"
  21. def _cron_job_name(job_id: str, home: Optional[Path]) -> Optional[str]:
  22. """``name`` from the profile's ``jobs.json`` (None when unreadable — the id alone still identifies it)."""
  23. try:
  24. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  25. from cron.jobs import load_jobs
  26. token = set_hermes_home_override(home) if home else None
  27. try:
  28. for job in load_jobs():
  29. if str(job.get("id")) == job_id:
  30. return str(job.get("name") or "") or None
  31. finally:
  32. if token is not None:
  33. reset_hermes_home_override(token)
  34. except Exception:
  35. return None
  36. return None
  37. def describe_active_work_unit(unit: dict, home: Optional[Path] = None) -> str:
  38. """One human line for one ``active_work`` entry; unknown shapes degrade to their ``kind``."""
  39. kind = str(unit.get("kind") or "work")
  40. pid = unit.get("pid")
  41. pid_part = f" pid {pid}" if pid else ""
  42. elapsed = unit.get("elapsed_s")
  43. elapsed_part = f", running {_fmt_elapsed(elapsed)}" if elapsed is not None else ""
  44. if kind == "cron":
  45. job_id = str(unit.get("job_id") or "?")
  46. name = _cron_job_name(job_id, home)
  47. label = f"cron job {job_id}" + (f" ({name})" if name else "")
  48. where = f" in external worker{pid_part}" if unit.get("external") else f" in-process{pid_part}"
  49. return f"{label}{where}{elapsed_part}"
  50. if kind == "chat":
  51. session = str(unit.get("session") or "?")
  52. model = unit.get("model")
  53. tool = unit.get("current_tool")
  54. detail = ", ".join(p for p in (f"model {model}" if model else "", f"tool {tool}" if tool else "") if p)
  55. return f"chat turn {session}{pid_part}{elapsed_part}" + (f" [{detail}]" if detail else "")
  56. if kind == "delegation":
  57. goal = str(unit.get("goal") or "").strip()
  58. return (f"background delegation {unit.get('delegation_id') or '?'}{pid_part}{elapsed_part}"
  59. + (f" [{goal}]" if goal else ""))
  60. return f"{kind} run{pid_part}{elapsed_part}"
  61. def read_active_work(home: Optional[Path] = None) -> Optional[list]:
  62. """``active_work`` as the gateway last published it, or None (old gateway / not draining / unreadable)."""
  63. try:
  64. from gateway.status import read_runtime_status
  65. record = read_runtime_status(home / "gateway_state.json" if home else None) or {}
  66. work = record.get("active_work")
  67. return list(work) if isinstance(work, list) else None
  68. except Exception:
  69. return None
  70. def _tui_command(home: Optional[Path]) -> str:
  71. """The TUI command that opens *home*'s profile: a bare ``hermes`` opens the default one."""
  72. from hermes_constants import profile_name_for_home
  73. name = profile_name_for_home(home)
  74. return f"hermes -p {name} --tui" if name and name != "default" else "hermes --tui"
  75. def format_drain_report(work: Optional[list], *, remaining_s: float, home: Optional[Path] = None) -> str:
  76. """Multi-line progress block: what the gateway is waiting on plus how to stop waiting."""
  77. lines = [f" ⏳ still draining — {max(remaining_s, 0):.0f}s left before the forced restart"]
  78. if work is None:
  79. lines.append(" (gateway did not report what it is waiting on — pre-update gateway or unreadable state file)")
  80. elif not work:
  81. lines.append(" (no active work reported; the gateway should exit momentarily)")
  82. else:
  83. lines.append(f" waiting on {len(work)} active work unit(s):")
  84. lines.extend(f" • {describe_active_work_unit(u, home)}" for u in work)
  85. if any(isinstance(u, dict) and u.get("kind") == "chat" for u in work):
  86. # The draining gateway stays reachable for this (#139041): a chat turn parked on an
  87. # approval/clarify is answered from a client, not waited out.
  88. lines.append(" a chat turn waiting on an approval or clarify question can be answered now: "
  89. f"open it with `{_tui_command(home)}` (resume the session)")
  90. lines.append(" finish or kill the work above to release the drain now; "
  91. "agent.restart_after_turn_timeout in config.yaml caps this wait")
  92. return "\n".join(lines)
  93. def drain_progress_reporter(home: Optional[Path] = None, *, budget_s: float,
  94. interval_s: float = DRAIN_REPORT_INTERVAL_S,
  95. emit: Callable[[str], None] = print) -> Callable[[], None]:
  96. """Return a zero-arg callback for ``_wait_for_pid_exit(on_progress=...)`` that prints the drain
  97. report every ``interval_s`` while the wait is in progress."""
  98. started = time.monotonic()
  99. state = {"last": started}
  100. def _tick() -> None:
  101. now = time.monotonic()
  102. if now - state["last"] < interval_s:
  103. return
  104. state["last"] = now
  105. emit(format_drain_report(read_active_work(home), remaining_s=budget_s - (now - started), home=home))
  106. return _tick