run_notifications_update.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374
  1. """``hermes update --gateway`` progress/completion notifications for GatewayRunner.
  2. Split out of ``gateway/run_notifications.py``; ``GatewayNotificationsMixin`` inherits it, so
  3. these stay bound onto ``GatewayRunner`` via the MRO. ``gateway.run`` internals are imported
  4. lazily inside method bodies (import cycle), so ``patch("gateway.run.X")`` keeps intercepting them.
  5. """
  6. from __future__ import annotations
  7. import asyncio
  8. import dataclasses
  9. import json
  10. import logging
  11. from contextlib import suppress
  12. from datetime import datetime
  13. from pathlib import Path
  14. from typing import Any, Optional
  15. from agent.i18n import t
  16. from gateway.config import Platform
  17. from gateway.run_shutdown import _log_suppressed
  18. # Log-record parity with the origin module.
  19. logger = logging.getLogger("gateway.run")
  20. # A failed /update leaves the previous version running; the full pip/git log stays on the host
  21. # (`hermes update` re-runs it in the terminal) and only a short tail is quoted in chat.
  22. def _update_failed_notice() -> str:
  23. return t("gateway.update.failed_notice")
  24. # An update's completion notice waits for its target platform adapter to (re)connect before it
  25. # can be delivered. Nothing bounds that wait, so a marker naming a platform that is not
  26. # configured at all — no adapter will ever appear — would keep itself on disk and re-log a
  27. # deferred line on every poll, in every process, forever. Stop waiting past this age.
  28. _UPDATE_NOTIFY_MAX_ADAPTER_WAIT_SECONDS = 3600.0
  29. def _update_output_tail(output: str, limit: int) -> str:
  30. """Last ``limit`` chars of an update log, prefixed with an ellipsis when cut."""
  31. return output if len(output) <= limit else "…" + output[-limit:]
  32. class GatewayUpdateNotificationsMixin:
  33. """Update watcher: streams ``hermes update --gateway`` output, forwards prompts, sends the final notice."""
  34. @dataclasses.dataclass
  35. class _UpdatePaths:
  36. """Marker files ``hermes update --gateway`` and its watcher exchange under HERMES_HOME."""
  37. pending: Path
  38. claimed: Path
  39. output: Path
  40. exit_code: Path
  41. prompt: Path
  42. response: Path
  43. def any_pending(self) -> bool:
  44. return self.pending.exists() or self.claimed.exists()
  45. def unlink_all(self) -> None:
  46. for p in (self.pending, self.claimed, self.output, self.exit_code, self.prompt, self.response):
  47. p.unlink(missing_ok=True)
  48. @dataclasses.dataclass
  49. class _UpdateTarget:
  50. """Resolved delivery target for update watcher messages."""
  51. adapter: Any
  52. chat_id: Any
  53. session_key: Optional[str]
  54. metadata: Any
  55. platform: Any
  56. def send_metadata(self):
  57. from gateway.run import _non_conversational_metadata
  58. return _non_conversational_metadata(self.metadata, platform=self.platform)
  59. async def send(self, text: str):
  60. return await self.adapter.send(self.chat_id, text, metadata=self.send_metadata())
  61. def _schedule_update_notification_watch(self) -> None:
  62. """Ensure a background task is watching for update completion."""
  63. existing_task = getattr(self, "_update_notification_task", None)
  64. if existing_task and not existing_task.done():
  65. return
  66. try:
  67. self._update_notification_task = asyncio.create_task(self._watch_update_progress())
  68. except RuntimeError:
  69. logger.debug("Skipping update notification watcher: no running event loop")
  70. @classmethod
  71. def _update_paths(cls) -> GatewayUpdateNotificationsMixin._UpdatePaths:
  72. from gateway.run import _hermes_home
  73. return cls._UpdatePaths(
  74. pending=_hermes_home / ".update_pending.json",
  75. claimed=_hermes_home / ".update_pending.claimed.json", output=_hermes_home / ".update_output.txt",
  76. exit_code=_hermes_home / ".update_exit_code",
  77. prompt=_hermes_home / ".update_prompt.json", response=_hermes_home / ".update_response",
  78. )
  79. @staticmethod
  80. def _marker_profile(data: dict) -> Optional[str]:
  81. """Owning profile of a persisted restart/update marker: explicit ``profile``, else the
  82. ``agent:<profile>:`` lane of its ``session_key`` (markers written before ``profile`` was
  83. persisted); ``None`` = default profile."""
  84. profile = str(data.get("profile") or "").strip()
  85. if profile:
  86. return profile
  87. from gateway.session import profile_from_session_key_namespace
  88. parts = str(data.get("session_key") or "").split(":")
  89. if len(parts) >= 5 and parts[0] == "agent" and parts[1] not in ("main", ""):
  90. return profile_from_session_key_namespace(parts[1])
  91. return None
  92. @staticmethod
  93. def _marker_age_seconds(data: dict) -> Optional[float]:
  94. """Age of a persisted update marker, from the ``timestamp`` stamped by its writer.
  95. ``None`` when the marker carries no parseable stamp — the field is absent on markers
  96. written before it existed, and callers keep the old retry behavior rather than guess.
  97. """
  98. raw = str(data.get("timestamp") or "").strip()
  99. if not raw:
  100. return None
  101. try:
  102. stamped = datetime.fromisoformat(raw)
  103. except ValueError:
  104. return None
  105. # The writer stamps a naive local ``datetime.now()``; tolerate a tz-aware one too.
  106. now = datetime.now(stamped.tzinfo) if stamped.tzinfo else datetime.now()
  107. return (now - stamped).total_seconds()
  108. def _resolve_update_target(self, paths: _UpdatePaths) -> Optional[_UpdateTarget]:
  109. """Resolve adapter/chat/session for update watcher messages from the pending marker."""
  110. for path in (paths.claimed, paths.pending):
  111. if not path.exists():
  112. continue
  113. with suppress(Exception):
  114. pending = json.loads(path.read_text(encoding="utf-8-sig"))
  115. platform_str = pending.get("platform")
  116. chat_id = pending.get("chat_id")
  117. session_key = pending.get("session_key")
  118. if not (platform_str and chat_id):
  119. continue # BASE: an incomplete marker falls through to the next path, not "unresolved"
  120. platform = Platform(platform_str)
  121. # The requester's OWN profile bot (marker ``profile``, else the ``agent:<profile>:`` key
  122. # lane); a bare self.adapters lookup is the default bot under multiplex.
  123. adapter = self._authorization_adapter(platform, self._marker_profile(pending))
  124. if not adapter:
  125. return None
  126. metadata = self._pending_marker_metadata(platform, chat_id, pending, adapter)
  127. # Fallback session key if not stored (old pending files)
  128. return self._UpdateTarget(
  129. adapter, chat_id, session_key or f"{platform_str}:{chat_id}", metadata, platform,
  130. )
  131. return None
  132. def _pending_marker_metadata(self, platform, chat_id, data: dict, adapter):
  133. """Thread metadata for a persisted update/restart marker (thread_id/chat_type/message_id keys)."""
  134. return self._thread_metadata_for_target(
  135. platform, chat_id, data.get("thread_id"), chat_type=data.get("chat_type"),
  136. reply_to_message_id=data.get("message_id"), adapter=adapter,
  137. )
  138. async def _watch_update_completion_only(self, paths: _UpdatePaths, deadline: float, poll_interval: float) -> None:
  139. """Fallback when no adapter/chat can be resolved: wait for the exit code, then notify."""
  140. logger.warning("Update watcher: cannot resolve adapter/chat_id, falling back to completion-only")
  141. # Poll until _send_update_notification delivers (it returns False while the platform reconnects).
  142. loop = asyncio.get_running_loop()
  143. while paths.any_pending() and loop.time() < deadline:
  144. if paths.exit_code.exists() and await self._send_update_notification():
  145. return
  146. await asyncio.sleep(poll_interval)
  147. if paths.any_pending() and not paths.exit_code.exists():
  148. paths.exit_code.write_text("124", encoding="utf-8")
  149. await self._send_update_notification()
  150. @staticmethod
  151. def _update_exit_code(paths: _UpdatePaths) -> int:
  152. return int(paths.exit_code.read_text(encoding="utf-8-sig").strip() or "1")
  153. @staticmethod
  154. def _read_update_output_since(path: Path, offset: int) -> tuple[str, int]:
  155. """Read update output defensively; logs may contain invalid UTF-8."""
  156. try:
  157. data = path.read_bytes()
  158. except OSError:
  159. return "", offset
  160. if len(data) <= offset:
  161. return "", len(data)
  162. return data[offset:].decode("utf-8", errors="replace"), len(data)
  163. async def _send_update_output(self, target: _UpdateTarget, text: str) -> None:
  164. """Send buffered update output as fenced chunks that fit message limits (Telegram: 4096)."""
  165. from tools.ansi_strip import strip_ansi
  166. clean = strip_ansi(text).strip()
  167. if not clean:
  168. return
  169. max_chunk = 3500
  170. for i in range(0, len(clean), max_chunk):
  171. with _log_suppressed(logging.DEBUG, "Update stream send failed: %s"):
  172. await target.send(f"```\n{clean[i:i + max_chunk]}\n```")
  173. async def _forward_update_prompt(self, target: _UpdateTarget, prompt_text: str, default: str) -> None:
  174. """Forward an update prompt: platform-native buttons first (Discord, Telegram), else text."""
  175. sent_buttons = False
  176. adapter = target.adapter
  177. if getattr(type(adapter), "send_update_prompt", None) is not None:
  178. with _log_suppressed(logging.DEBUG, "Button-based update prompt failed: %s"):
  179. await adapter.send_update_prompt(
  180. chat_id=target.chat_id, prompt=prompt_text, default=default,
  181. session_key=target.session_key, metadata=target.send_metadata(),
  182. )
  183. sent_buttons = True
  184. if not sent_buttons:
  185. default_hint = t("gateway.update.prompt_default", default=default) if default else ""
  186. _p = getattr(adapter, "typed_command_prefix", "/")
  187. await target.send(t("gateway.update.prompt", prompt=prompt_text, default_hint=default_hint, prefix=_p))
  188. # Keep the prompt marker on disk until answered so a restarted watcher can re-forward it.
  189. self._session_state(target.session_key).persistent.update_prompt_pending = True
  190. logger.info("Forwarded update prompt to %s: %s", target.session_key, prompt_text[:80])
  191. def _clear_update_markers(self, paths: _UpdatePaths, session_key: Optional[str]) -> None:
  192. paths.unlink_all()
  193. state = self._peek_session_state(session_key)
  194. if state is not None:
  195. state.persistent.update_prompt_pending = False
  196. async def _watch_update_progress(
  197. self, poll_interval: float = 2.0, stream_interval: float = 4.0, timeout: float = 1800.0
  198. ) -> None:
  199. """Watch ``hermes update --gateway``, streaming output + forwarding prompts.
  200. Polls ``.update_output.txt`` for new content and sends chunks to the user periodically;
  201. detects ``.update_prompt.json`` (written when the update process needs input) and forwards it.
  202. """
  203. paths = self._update_paths()
  204. loop = asyncio.get_running_loop()
  205. deadline = loop.time() + timeout
  206. target = self._resolve_update_target(paths)
  207. if target is None:
  208. await self._watch_update_completion_only(paths, deadline, poll_interval)
  209. return
  210. session_key = target.session_key
  211. bytes_sent = 0
  212. last_stream_time = loop.time()
  213. buffer = ""
  214. async def _flush_buffer() -> None:
  215. nonlocal buffer, last_stream_time
  216. text, buffer = buffer, ""
  217. if text.strip():
  218. last_stream_time = loop.time()
  219. await self._send_update_output(target, text)
  220. def _read_new_output() -> None:
  221. nonlocal buffer, bytes_sent
  222. if paths.output.exists():
  223. with suppress(OSError):
  224. chunk, bytes_sent = self._read_update_output_since(paths.output, bytes_sent)
  225. buffer += chunk
  226. while loop.time() < deadline:
  227. if paths.exit_code.exists():
  228. _read_new_output()
  229. await _flush_buffer()
  230. with _log_suppressed(logging.WARNING, "Update final notification failed: %s"):
  231. exit_code = self._update_exit_code(paths)
  232. await target.send(
  233. t("gateway.update.finished") if exit_code == 0 else _update_failed_notice()
  234. )
  235. logger.info("Update finished (exit=%s), notified %s", exit_code, session_key)
  236. self._clear_update_markers(paths, session_key)
  237. return
  238. _read_new_output()
  239. if buffer.strip() and (loop.time() - last_stream_time) >= stream_interval:
  240. await _flush_buffer()
  241. # Forward a prompt only when none is pending, else every poll re-forwards the same prompt.
  242. _pending_state = self._peek_session_state(session_key) if session_key else None
  243. if paths.prompt.exists() and session_key and not getattr(
  244. getattr(_pending_state, "persistent", None), "update_prompt_pending", False
  245. ):
  246. try:
  247. prompt_data = json.loads(paths.prompt.read_text(encoding="utf-8-sig"))
  248. prompt_text = prompt_data.get("prompt", "")
  249. if prompt_text:
  250. await _flush_buffer() # user sees context before the prompt
  251. await self._forward_update_prompt(target, prompt_text, prompt_data.get("default", ""))
  252. except (json.JSONDecodeError, OSError) as e:
  253. logger.debug("Failed to read update prompt: %s", e)
  254. await asyncio.sleep(poll_interval)
  255. if not paths.exit_code.exists():
  256. logger.warning("Update watcher timed out after %.0fs", timeout)
  257. paths.exit_code.write_text("124", encoding="utf-8")
  258. await _flush_buffer()
  259. with suppress(Exception):
  260. await target.send(t("gateway.update.timed_out"))
  261. self._clear_update_markers(paths, session_key)
  262. async def _send_update_notification(self) -> bool:
  263. """If an update finished, notify the user.
  264. False while the update is still running (caller may retry); True after a definitive send/skip.
  265. """
  266. from gateway.run import _non_conversational_metadata
  267. paths = self._update_paths()
  268. if not paths.any_pending():
  269. return False
  270. cleanup = True
  271. active_pending_path = paths.claimed
  272. def _defer(reason: str, *args) -> bool:
  273. nonlocal cleanup, active_pending_path
  274. logger.info(reason, *args)
  275. cleanup = False
  276. active_pending_path = paths.pending
  277. paths.claimed.replace(paths.pending)
  278. return False
  279. try:
  280. if paths.pending.exists():
  281. try:
  282. paths.pending.replace(paths.claimed)
  283. except FileNotFoundError:
  284. if not paths.claimed.exists():
  285. return True
  286. elif not paths.claimed.exists():
  287. return True
  288. pending = json.loads(paths.claimed.read_text(encoding="utf-8-sig"))
  289. platform_str = pending.get("platform")
  290. chat_id = pending.get("chat_id")
  291. if not paths.exit_code.exists():
  292. return _defer("Update notification deferred: update still running")
  293. exit_code = self._update_exit_code(paths)
  294. output = paths.output.read_bytes().decode("utf-8", errors="replace") if paths.output.exists() else ""
  295. platform = Platform(platform_str)
  296. adapter = self._authorization_adapter(platform, self._marker_profile(pending))
  297. if chat_id and not adapter:
  298. age = self._marker_age_seconds(pending)
  299. if age is not None and age > _UPDATE_NOTIFY_MAX_ADAPTER_WAIT_SECONDS:
  300. # The platform never came back. Deferring forever leaks the markers and re-logs
  301. # on every poll for the life of the install: the startup path reschedules this
  302. # watcher whenever the markers are still on disk, so an undeliverable marker
  303. # outlives every restart. Give up loudly, clear the markers, and report a
  304. # definitive decision (True) so the caller stops rescheduling.
  305. logger.warning(
  306. "Post-update notification for %s:%s dropped after %.1fh: %s adapter never "
  307. "connected", platform_str, chat_id, age / 3600.0, platform_str)
  308. self._clear_update_markers(paths, pending.get("session_key"))
  309. return True
  310. # Target platform not reconnected yet (common right after the update's restart): keep the
  311. # markers for a later retry instead of silently losing the notification.
  312. return _defer("Update notification deferred: %s adapter not connected yet", platform_str)
  313. if chat_id:
  314. metadata = self._pending_marker_metadata(platform, chat_id, pending, adapter)
  315. from tools.ansi_strip import strip_ansi
  316. output = strip_ansi(output).strip()
  317. if exit_code == 0:
  318. msg = t("gateway.update.finished_success")
  319. if output:
  320. msg = f"{msg}\n\n```\n{_update_output_tail(output, 3500)}\n```"
  321. else:
  322. msg = _update_failed_notice()
  323. if output:
  324. msg = t("gateway.update.last_lines", msg=msg, tail=_update_output_tail(output, 800))
  325. await adapter.send(chat_id, msg, metadata=_non_conversational_metadata(metadata, platform=platform))
  326. logger.info("Sent post-update notification to %s:%s (exit=%s)", platform_str, chat_id, exit_code)
  327. except Exception as e:
  328. logger.warning("Post-update notification failed: %s", e)
  329. finally:
  330. if cleanup:
  331. for p in (active_pending_path, paths.claimed, paths.output, paths.exit_code):
  332. p.unlink(missing_ok=True)
  333. return True