nous_auth_keepalive.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260
  1. """Background keepalive for long-lived Nous Portal sessions."""
  2. from __future__ import annotations
  3. import logging
  4. import os
  5. import threading
  6. from typing import Optional
  7. from hermes_cli.auth import (
  8. ACCESS_TOKEN_REFRESH_SKEW_SECONDS,
  9. NOUS_INVOKE_JWT_MIN_TTL_SECONDS,
  10. AuthError,
  11. _agent_key_is_usable,
  12. _is_expiring,
  13. get_provider_auth_state,
  14. resolve_nous_runtime_credentials,
  15. )
  16. logger = logging.getLogger(__name__)
  17. # Two things must line up for the keepalive to keep anything alive:
  18. # 1. The tick must be frequent enough to see the credential before it dies. Lifetimes vary by
  19. # account (~3594s and ~899s observed), so the tick derives from the lifetime the server issued,
  20. # capped by the configured interval and floored so a pathological lifetime can't spin the thread.
  21. # 2. The refresh must fire while the tick can still act on it: refresh triggers only within a skew
  22. # window of expiry, so the keepalive widens that window to "will this credential outlive my
  23. # next tick?" instead of the request path's 120s. Ticking faster alone never closes the gap.
  24. NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS = 15 * 60
  25. NOUS_AUTH_KEEPALIVE_MIN_INTERVAL_SECONDS = 60
  26. # Ticks per credential lifetime: four keeps refresh comfortably ahead of expiry without chatter.
  27. NOUS_AUTH_KEEPALIVE_TICKS_PER_LIFETIME = 4
  28. NOUS_AUTH_KEEPALIVE_INITIAL_DELAY_SECONDS = 60
  29. NOUS_AUTH_KEEPALIVE_INTERVAL_CONFIG_KEY = "keepalive_interval_seconds"
  30. _keepalive_lock = threading.Lock()
  31. _keepalive_stop = threading.Event()
  32. _keepalive_thread: Optional[threading.Thread] = None
  33. def _timeout_seconds(value: Optional[float]) -> float:
  34. if value is not None:
  35. return float(value)
  36. try:
  37. return float(os.getenv("HERMES_NOUS_TIMEOUT_SECONDS", "15"))
  38. except (TypeError, ValueError):
  39. return 15.0
  40. def _nous_config() -> dict:
  41. """The ``nous:`` section of config.yaml, or {} on any failure (config loader imported lazily)."""
  42. try:
  43. from hermes_cli.config import load_config
  44. section = load_config().get("nous")
  45. return section if isinstance(section, dict) else {}
  46. except Exception:
  47. return {}
  48. def _interval_seconds(value: Optional[int]) -> int:
  49. """Tick interval: explicit argument, then ``nous.keepalive_interval_seconds`` in config.yaml,
  50. then the module default. Non-positive disables the keepalive thread (the documented way off).
  51. """
  52. if value is not None:
  53. try:
  54. return int(value)
  55. except (TypeError, ValueError):
  56. return NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS
  57. raw = _nous_config().get(NOUS_AUTH_KEEPALIVE_INTERVAL_CONFIG_KEY)
  58. if raw is None or (isinstance(raw, str) and not raw.strip()):
  59. return NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS
  60. try:
  61. return int(float(raw))
  62. except (TypeError, ValueError):
  63. logger.warning(
  64. "Ignoring invalid nous.%s=%r; using %ds",
  65. NOUS_AUTH_KEEPALIVE_INTERVAL_CONFIG_KEY, raw, NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS,
  66. )
  67. return NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS
  68. def _observed_lifetime_seconds() -> Optional[int]:
  69. """Server-issued lifetime (seconds) of the current Nous credentials; the shorter of the access
  70. token and the invoke agent key governs. None when nothing usable is stored.
  71. """
  72. state = get_provider_auth_state("nous") or {}
  73. lifetimes = []
  74. for key in ("expires_in", "agent_key_expires_in"):
  75. try:
  76. value = int(float(state.get(key)))
  77. except (TypeError, ValueError):
  78. continue
  79. if value > 0:
  80. lifetimes.append(value)
  81. return min(lifetimes, default=None)
  82. def _tick_seconds(configured_interval: int, lifetime: Optional[int]) -> int:
  83. """Tick fast enough to refresh several times per credential lifetime."""
  84. if not lifetime or lifetime <= 0:
  85. return configured_interval
  86. derived = lifetime // NOUS_AUTH_KEEPALIVE_TICKS_PER_LIFETIME
  87. return max(NOUS_AUTH_KEEPALIVE_MIN_INTERVAL_SECONDS, min(configured_interval, derived))
  88. def _refresh_horizon_seconds(tick_seconds: int, floor_seconds: int) -> int:
  89. """Life a credential needs to be left alone this tick: it must survive until the next tick
  90. (nothing looks at it again before then), hence tick + skew rather than the bare skew.
  91. """
  92. return max(floor_seconds, tick_seconds + ACCESS_TOKEN_REFRESH_SKEW_SECONDS)
  93. def _entry_state(entry: object) -> dict:
  94. return {k: getattr(entry, k, None) for k in ("agent_key", "agent_key_expires_at", "scope")}
  95. def _refresh_selected_pool_entry(*, min_key_ttl_seconds: int, min_access_ttl_seconds: Optional[int] = None) -> Optional[bool]:
  96. """Refresh the current pool entry when stale. True = usable/refreshed; False = pool exists but
  97. no usable entry; None = no Nous pool.
  98. """
  99. try:
  100. from agent.credential_pool import load_pool
  101. pool = load_pool("nous")
  102. except Exception as exc:
  103. logger.debug("Nous auth keepalive: credential pool unavailable: %s", exc)
  104. return None
  105. if not pool or not pool.has_credentials():
  106. return None
  107. try:
  108. entry = pool.select()
  109. except Exception as exc:
  110. logger.debug("Nous auth keepalive: credential pool selection failed: %s", exc)
  111. return False
  112. if entry is None:
  113. return False
  114. if min_access_ttl_seconds is None:
  115. min_access_ttl_seconds = ACCESS_TOKEN_REFRESH_SKEW_SECONDS
  116. access_expiring = _is_expiring(getattr(entry, "expires_at", None), min_access_ttl_seconds)
  117. key_usable = _agent_key_is_usable(_entry_state(entry), min_key_ttl_seconds)
  118. if access_expiring or not key_usable:
  119. if pool.try_refresh_current() is None:
  120. return False
  121. logger.debug("Nous auth keepalive: refreshed credential pool entry")
  122. return True
  123. def refresh_nous_auth_keepalive_once(
  124. *, min_key_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS,
  125. min_access_ttl_seconds: Optional[int] = None, timeout_seconds: Optional[float] = None,
  126. ) -> bool:
  127. """Refresh Nous auth once if credentials are configured (pool entry first, then singleton state)."""
  128. # This runs in a bare daemon thread, so it does not inherit a request's ContextVars. Once a
  129. # gateway multiplexes profiles, even the launch profile must bind its own scope before a
  130. # credential read; otherwise the fail-closed routing reader warns on every tick.
  131. from tui_gateway.launch_profile_policy import launch_profile_scope_if_multiplexed
  132. with launch_profile_scope_if_multiplexed():
  133. pool_result = _refresh_selected_pool_entry(
  134. min_key_ttl_seconds=max(60, int(min_key_ttl_seconds)), min_access_ttl_seconds=min_access_ttl_seconds
  135. )
  136. if pool_result is not None:
  137. return pool_result
  138. state = get_provider_auth_state("nous")
  139. if not state:
  140. return False
  141. # Mid-life twin of start_nous_auth_keepalive's guard (a guest minted after the thread
  142. # started): its tokens are minted on demand, and an inference mint here would put a guest
  143. # who only uses connectors through the free-inference challenge every tick.
  144. from hermes_cli.anon_auth import is_guest_state
  145. if is_guest_state(state):
  146. return False
  147. try:
  148. # Nobody is waiting on a keepalive tick: a free-tier browser challenge is announced to the
  149. # desktop (so it can clear before a token is needed) but never waited on here.
  150. from hermes_cli.anon_challenge import background_caller
  151. with background_caller():
  152. resolve_nous_runtime_credentials(timeout_seconds=_timeout_seconds(timeout_seconds))
  153. logger.debug("Nous auth keepalive: refreshed singleton auth state")
  154. return True
  155. except Exception as exc:
  156. if isinstance(exc, AuthError) and exc.relogin_required:
  157. logger.info("Nous auth keepalive requires re-login: %s", exc)
  158. else:
  159. logger.debug("Nous auth keepalive failed: %s", exc)
  160. return False
  161. def _keepalive_loop(
  162. stop_event: threading.Event, *, interval_seconds: int, initial_delay_seconds: int,
  163. min_key_ttl_seconds: int, timeout_seconds: Optional[float],
  164. ) -> None:
  165. if initial_delay_seconds > 0 and stop_event.wait(initial_delay_seconds):
  166. return
  167. while not stop_event.is_set():
  168. # Re-read each pass: the lifetime changes with account/plan/policy; caching it would go
  169. # stale in exactly the case the keepalive exists to cover.
  170. tick = _tick_seconds(interval_seconds, _observed_lifetime_seconds())
  171. horizon = _refresh_horizon_seconds(tick, min_key_ttl_seconds)
  172. refresh_nous_auth_keepalive_once(
  173. min_key_ttl_seconds=horizon, min_access_ttl_seconds=horizon, timeout_seconds=timeout_seconds
  174. )
  175. stop_event.wait(tick)
  176. def start_nous_auth_keepalive(
  177. *, interval_seconds: Optional[int] = None,
  178. initial_delay_seconds: int = NOUS_AUTH_KEEPALIVE_INITIAL_DELAY_SECONDS,
  179. min_key_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, timeout_seconds: Optional[float] = None,
  180. ) -> Optional[threading.Thread]:
  181. """Start the process-wide Nous auth keepalive thread (idempotent; None when disabled)."""
  182. # Free-tier handoff retries run in every long-lived Nous process, guest or not and keepalive or
  183. # not: a CLI sign-in in another process may have left a move this one has to finish.
  184. from hermes_cli.guest_handoff import start_worker
  185. start_worker()
  186. interval_seconds = _interval_seconds(interval_seconds)
  187. if interval_seconds <= 0:
  188. return None
  189. # The free tier has no refresh token to keep alive: its access token is re-minted from the
  190. # anon credential on demand by the request path, so a background refresher has nothing to do.
  191. from hermes_cli.anon_auth import is_guest_state
  192. try:
  193. if is_guest_state(get_provider_auth_state("nous")):
  194. logger.debug("Nous auth keepalive skipped: free tier has no refresh token")
  195. return None
  196. except Exception:
  197. pass
  198. global _keepalive_thread
  199. with _keepalive_lock:
  200. if _keepalive_thread is not None and _keepalive_thread.is_alive():
  201. return _keepalive_thread
  202. _keepalive_stop.clear()
  203. _keepalive_thread = threading.Thread(
  204. target=_keepalive_loop, args=(_keepalive_stop,), daemon=True, name="nous-auth-keepalive",
  205. kwargs={
  206. "interval_seconds": int(interval_seconds),
  207. "initial_delay_seconds": max(0, int(initial_delay_seconds)),
  208. "min_key_ttl_seconds": max(60, int(min_key_ttl_seconds)),
  209. "timeout_seconds": timeout_seconds,
  210. },
  211. )
  212. _keepalive_thread.start()
  213. logger.debug("Nous auth keepalive started")
  214. return _keepalive_thread
  215. def stop_nous_auth_keepalive(timeout: float = 5.0) -> None:
  216. """Stop the keepalive thread. Intended for graceful shutdown/tests."""
  217. from hermes_cli.guest_handoff import stop_worker
  218. stop_worker(timeout)
  219. global _keepalive_thread
  220. with _keepalive_lock:
  221. thread = _keepalive_thread
  222. _keepalive_stop.set()
  223. if thread is not None and thread.is_alive():
  224. thread.join(timeout=timeout)
  225. with _keepalive_lock:
  226. if _keepalive_thread is thread:
  227. _keepalive_thread = None