endpoint.py 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157
  1. """Endpoint resolution for llamacpp-alias requests (provider integration).
  2. ``provider: llamacpp`` with no explicit base_url resolves, in order, to the managed server (state
  3. file), a detected external llama-server, or — during a backend boot race — the managed server once
  4. its state file appears.
  5. """
  6. from __future__ import annotations
  7. from contextlib import suppress
  8. import json
  9. import logging
  10. import threading
  11. import urllib.request
  12. from agent.deadline import poll_until
  13. LLAMACPP_ALIASES = frozenset({"llamacpp", "llama.cpp", "llama-cpp"})
  14. def child_query(model_id: str) -> str:
  15. """Query string for a status read of one router child (``/props``, ``/slots``, ``/metrics``).
  16. The router loads an unloaded model for any request that names it (``--models-autoload``), so a
  17. status read right after an Eject or an idle unload would load the model straight back.
  18. ``autoload=false`` makes the router answer 400 "model is not loaded" instead.
  19. """
  20. from urllib.parse import quote
  21. return f"model={quote(model_id)}&autoload=false"
  22. logger = logging.getLogger(__name__)
  23. def _pid_alive(pid: int) -> bool:
  24. """Liveness for the state file's supervisor-child pid: psutil when available, else True
  25. (optimistic). On Windows ``os.kill(pid, 0)`` TERMINATES the process — never use it as a probe."""
  26. if not pid or pid < 0:
  27. return False
  28. with suppress(Exception):
  29. import psutil # type: ignore
  30. return psutil.pid_exists(pid)
  31. return True
  32. def _state_endpoint() -> dict | None:
  33. from hermes_cli.local_runtime.recovery import (
  34. is_modern,
  35. legacy_recorded_process,
  36. read_state,
  37. recorded_process,
  38. )
  39. state = read_state()
  40. base_url = state.get("base_url", "")
  41. if not isinstance(base_url, str) or not base_url:
  42. return None
  43. if is_modern(state):
  44. if recorded_process(state) is None:
  45. return None
  46. else:
  47. if legacy_recorded_process(state) is None:
  48. return None
  49. return {"base_url": base_url, "api_key": state.get("api_key", "")}
  50. def managed_root() -> tuple[str, str] | None:
  51. """(base_root, api_key) of the managed router, or None. Resolved through the
  52. ownership-guarded reader, not a raw state-file read: on the shared stable port a foreign
  53. install's server answers /health for anyone, and a raw read would attach callers to someone
  54. else's server."""
  55. with suppress(Exception):
  56. state = _state_endpoint()
  57. if state is None:
  58. return None
  59. base = str(state.get("base_url", "")).rsplit("/v1", 1)[0]
  60. return (base, str(state.get("api_key", ""))) if base else None
  61. return None
  62. def managed_get_json(base: str, api_key: str, route: str, timeout_s: float) -> object:
  63. """Authenticated GET against the managed router; raises on any transport/decode failure."""
  64. req = urllib.request.Request(f"{base}{route}",
  65. headers={"Authorization": f"Bearer {api_key}"})
  66. with urllib.request.urlopen(req, timeout=timeout_s) as r:
  67. return json.loads(r.read())
  68. def resolve_llamacpp_endpoint(config: dict | None = None,
  69. wait_for_boot_s: float = 8.0) -> dict | None:
  70. """Managed-first, detection-second endpoint for llamacpp aliases.
  71. Boot-race rung: on a fresh backend start there is NO state file yet — the lifespan boot thread
  72. is still spawning the server (≈1-3 s) while the desktop's readiness probe fires the moment the
  73. WebSocket connects.
  74. """
  75. managed = _state_endpoint()
  76. if managed:
  77. return managed
  78. from hermes_cli.local_runtime.detect import detect_server
  79. ports = ((config or {}).get("local_runtime") or {}).get("detect_ports") or []
  80. hit = detect_server(extra_ports=tuple(int(p) for p in ports))
  81. if hit and not hit.auth_required:
  82. return {"base_url": hit.base_url, "api_key": ""}
  83. if wait_for_boot_s > 0 and _boot_in_flight(config):
  84. _kick_managed_boot(config)
  85. return poll_until(_state_endpoint, wait_for_boot_s, 0.25) or None
  86. return None
  87. _KICK_LOCK = threading.Lock()
  88. def _load_config_if_none(config: dict | None) -> dict | None:
  89. if config is not None:
  90. return config
  91. from hermes_cli.config import load_config
  92. return load_config()
  93. def _kick_managed_boot(config: dict | None) -> None:
  94. """Actively start the managed server when resolution finds it missing — the wait loop assumes
  95. some OTHER thread is bringing it up, which is true only at backend start."""
  96. if not _KICK_LOCK.acquire(blocking=False):
  97. return # a kick is already in flight
  98. def _boot() -> None:
  99. try:
  100. from hermes_cli.local_runtime.bootstrap import ensure_local_runtime
  101. ensure_local_runtime(_load_config_if_none(config))
  102. except Exception:
  103. logger.warning("on-demand managed-server boot failed", exc_info=True)
  104. finally:
  105. _KICK_LOCK.release()
  106. from agent.memory_provider import spawn_context_thread
  107. # The caller's profile scope: ``config=None`` resolves (and the boot prices) under it.
  108. spawn_context_thread(_boot, name="lr-on-demand-boot").start()
  109. def _boot_in_flight(config: dict | None) -> bool:
  110. """True when the managed runtime is enabled and PM holds an installed engine."""
  111. with suppress(Exception):
  112. section = (_load_config_if_none(config) or {}).get("local_runtime") or {}
  113. if not section.get("enabled"):
  114. return False
  115. from hermes_cli.local_runtime.binaries import installed_engine
  116. return installed_engine(section.get("backend") or "auto") is not None
  117. return False