run_heartbeat_restore.py 4.2 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879
  1. """Recover heartbeat watches from the gateway's canonical persisted routing index."""
  2. from __future__ import annotations
  3. import logging
  4. from pathlib import Path
  5. logger = logging.getLogger("gateway.run")
  6. def _watched_homes(runner, default_home) -> list:
  7. """Every home the sweep's ``_profile_scope_for_source`` can resolve an origin to: the gateway home
  8. plus, under multiplex, the whole served set INCLUDING ``default`` — a ``-p work`` multiplexer's own
  9. home is not ``~/.hermes``, so dropping ``default`` here would hide its heartbeats forever."""
  10. from gateway.run import _multiplex_profile_homes
  11. homes = [default_home]
  12. if getattr(getattr(runner, "config", None), "multiplex_profiles", False):
  13. homes += [home for _name, home in _multiplex_profile_homes(runner.config)]
  14. return list(dict.fromkeys(Path(home) for home in homes))
  15. async def restore_heartbeat_watches(runner) -> None:
  16. """Retryable startup/poll scan; failed reads never prune existing watches.
  17. SessionStore owns one routing index across profiles. Its origin and exact key,
  18. rather than a second heartbeat routing snapshot, also cover pre-upgrade state.
  19. Run all storage work off-loop so a cold profile DB cannot block adapters.
  20. """
  21. from gateway.run import _profile_runtime_scope
  22. from gateway.run_idle_gates import profile_has_active_heartbeat
  23. from hermes_cli.heartbeat import HeartbeatManager
  24. from hermes_constants import get_hermes_home
  25. store = runner.session_store
  26. def scan():
  27. restored = []
  28. # The poller may have been spawned by a named profile's /heartbeat command.
  29. # Anchor even default origins to the gateway home, not inherited context.
  30. home = getattr(store, "_routing_home", None) or get_hermes_home()
  31. # Cheap gate: with no heartbeat persisted in any served profile there is nothing to
  32. # restore — skip the per-origin profile-scope re-parse over every routed session.
  33. if not any(profile_has_active_heartbeat(h) for h in _watched_homes(runner, home)):
  34. return restored
  35. with _profile_runtime_scope(home):
  36. # Enter each profile's scope once per scan, not once per routed session: a scope entry
  37. # hydrates the secret scope and terminal policy, so N sessions cost N parses otherwise.
  38. # Sources are read through _restored_source so the persisted receiving bot is re-pinned
  39. # before the scope key is derived; the same source object is what gets registered.
  40. by_scope: dict = {}
  41. for entry in store.list_sessions():
  42. if entry.origin is None or not entry.session_id or entry.suspended:
  43. continue
  44. try:
  45. source = runner._restored_source(entry)
  46. by_scope.setdefault(runner._profile_scope_key_for_source(source), []).append((entry, source))
  47. except Exception:
  48. logger.debug("heartbeat restore for %s failed", entry.session_key, exc_info=True)
  49. for group in by_scope.values():
  50. try:
  51. with runner._profile_scope_for_source(group[0][1]):
  52. for entry, source in group:
  53. try:
  54. if HeartbeatManager(entry.session_id).is_active():
  55. restored.append((entry.session_key, source, entry.session_id))
  56. except Exception:
  57. logger.debug("heartbeat restore for %s failed", entry.session_key, exc_info=True)
  58. except Exception:
  59. logger.debug("heartbeat restore scope for %s failed", group[0][0].session_key, exc_info=True)
  60. return restored
  61. try:
  62. candidates = await runner._run_in_executor_with_context(scan)
  63. for key, source, session_id in candidates:
  64. # A reset/compression may have published a new owner during the executor hop.
  65. if store.peek_session_id(key) == session_id:
  66. runner._register_heartbeat_watch(key, source, session_id)
  67. except Exception:
  68. logger.debug("heartbeat restore scan failed; retrying on next poll", exc_info=True)