| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129 |
- """Name what a draining gateway is waiting on while ``hermes update`` blocks on it.
- The gateway's in-band restart (SIGUSR1 → ``request_restart``) defers ``stop()`` until in-flight
- work finishes, capped by ``agent.restart_after_turn_timeout`` (30 min by default). From the
- updater's side that was a bare "draining (up to 1875s)..." followed by silence, which reads as a
- hung update. The gateway publishes each unit it is holding for in ``gateway_state.json``
- (``active_work``, written by ``GatewayShutdownMixin._describe_active_work``); this module turns
- that into progress lines.
- """
- from __future__ import annotations
- import time
- from pathlib import Path
- from typing import Callable, Optional
- # Progress cadence: the gateway refreshes ``active_work`` every 30s; printing faster only repeats it.
- DRAIN_REPORT_INTERVAL_S = 30.0
- def _fmt_elapsed(seconds: object) -> str:
- try:
- total = int(float(seconds)) # type: ignore[arg-type]
- except (TypeError, ValueError):
- return "?"
- return f"{total // 60}m{total % 60:02d}s" if total >= 60 else f"{total}s"
- def _cron_job_name(job_id: str, home: Optional[Path]) -> Optional[str]:
- """``name`` from the profile's ``jobs.json`` (None when unreadable — the id alone still identifies it)."""
- try:
- from hermes_constants import reset_hermes_home_override, set_hermes_home_override
- from cron.jobs import load_jobs
- token = set_hermes_home_override(home) if home else None
- try:
- for job in load_jobs():
- if str(job.get("id")) == job_id:
- return str(job.get("name") or "") or None
- finally:
- if token is not None:
- reset_hermes_home_override(token)
- except Exception:
- return None
- return None
- def describe_active_work_unit(unit: dict, home: Optional[Path] = None) -> str:
- """One human line for one ``active_work`` entry; unknown shapes degrade to their ``kind``."""
- kind = str(unit.get("kind") or "work")
- pid = unit.get("pid")
- pid_part = f" pid {pid}" if pid else ""
- elapsed = unit.get("elapsed_s")
- elapsed_part = f", running {_fmt_elapsed(elapsed)}" if elapsed is not None else ""
- if kind == "cron":
- job_id = str(unit.get("job_id") or "?")
- name = _cron_job_name(job_id, home)
- label = f"cron job {job_id}" + (f" ({name})" if name else "")
- where = f" in external worker{pid_part}" if unit.get("external") else f" in-process{pid_part}"
- return f"{label}{where}{elapsed_part}"
- if kind == "chat":
- session = str(unit.get("session") or "?")
- model = unit.get("model")
- tool = unit.get("current_tool")
- detail = ", ".join(p for p in (f"model {model}" if model else "", f"tool {tool}" if tool else "") if p)
- return f"chat turn {session}{pid_part}{elapsed_part}" + (f" [{detail}]" if detail else "")
- if kind == "delegation":
- goal = str(unit.get("goal") or "").strip()
- return (f"background delegation {unit.get('delegation_id') or '?'}{pid_part}{elapsed_part}"
- + (f" [{goal}]" if goal else ""))
- return f"{kind} run{pid_part}{elapsed_part}"
- def read_active_work(home: Optional[Path] = None) -> Optional[list]:
- """``active_work`` as the gateway last published it, or None (old gateway / not draining / unreadable)."""
- try:
- from gateway.status import read_runtime_status
- record = read_runtime_status(home / "gateway_state.json" if home else None) or {}
- work = record.get("active_work")
- return list(work) if isinstance(work, list) else None
- except Exception:
- return None
- def _tui_command(home: Optional[Path]) -> str:
- """The TUI command that opens *home*'s profile: a bare ``hermes`` opens the default one."""
- from hermes_constants import profile_name_for_home
- name = profile_name_for_home(home)
- return f"hermes -p {name} --tui" if name and name != "default" else "hermes --tui"
- def format_drain_report(work: Optional[list], *, remaining_s: float, home: Optional[Path] = None) -> str:
- """Multi-line progress block: what the gateway is waiting on plus how to stop waiting."""
- lines = [f" ⏳ still draining — {max(remaining_s, 0):.0f}s left before the forced restart"]
- if work is None:
- lines.append(" (gateway did not report what it is waiting on — pre-update gateway or unreadable state file)")
- elif not work:
- lines.append(" (no active work reported; the gateway should exit momentarily)")
- else:
- lines.append(f" waiting on {len(work)} active work unit(s):")
- lines.extend(f" • {describe_active_work_unit(u, home)}" for u in work)
- if any(isinstance(u, dict) and u.get("kind") == "chat" for u in work):
- # The draining gateway stays reachable for this (#139041): a chat turn parked on an
- # approval/clarify is answered from a client, not waited out.
- lines.append(" a chat turn waiting on an approval or clarify question can be answered now: "
- f"open it with `{_tui_command(home)}` (resume the session)")
- lines.append(" finish or kill the work above to release the drain now; "
- "agent.restart_after_turn_timeout in config.yaml caps this wait")
- return "\n".join(lines)
- def drain_progress_reporter(home: Optional[Path] = None, *, budget_s: float,
- interval_s: float = DRAIN_REPORT_INTERVAL_S,
- emit: Callable[[str], None] = print) -> Callable[[], None]:
- """Return a zero-arg callback for ``_wait_for_pid_exit(on_progress=...)`` that prints the drain
- report every ``interval_s`` while the wait is in progress."""
- started = time.monotonic()
- state = {"last": started}
- def _tick() -> None:
- now = time.monotonic()
- if now - state["last"] < interval_s:
- return
- state["last"] = now
- emit(format_drain_report(read_active_work(home), remaining_s=budget_s - (now - started), home=home))
- return _tick
|