chat_completion_local.py 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102
  1. """Chat completions against llama-server (sibling of ``chat_completion_helpers``).
  2. Which requests go to llama-server, the turn-start health check that lets a recovered server try
  3. again past the stale breaker, and the live load / prompt-reading notice shown before the first
  4. token.
  5. """
  6. from __future__ import annotations
  7. import http.client
  8. import json
  9. import logging
  10. import urllib.request
  11. from typing import Optional
  12. from urllib.parse import urlparse, urlsplit
  13. from utils import env_int
  14. logger = logging.getLogger(__name__)
  15. def _streams_llamacpp(agent) -> bool:
  16. """True when this agent's chat requests go to llama-server: the managed server, or a
  17. ``llamacpp`` provider endpoint. These requests always stream and ask for prompt progress
  18. (unless the user turned streaming off in config). A non-streamed request can't be cancelled
  19. through the router, so it would keep the server busy after Hermes gives up on it."""
  20. if getattr(agent, "api_mode", "") not in ("", "chat_completions"):
  21. return False
  22. from hermes_cli.local_runtime.capabilities import is_managed_provider
  23. return is_managed_provider(str(getattr(agent, "provider", "") or ""), str(getattr(agent, "base_url", "") or ""))
  24. def stale_giveup() -> int:
  25. """Consecutive stale attempts after which calls fail at once; 0 turns the breaker off."""
  26. return env_int("HERMES_STREAM_STALE_GIVEUP", 5)
  27. def _local_server_answers(agent) -> bool:
  28. """True when the llama-server behind *agent* answers ``/health`` within a few seconds."""
  29. parts = urlsplit(str(getattr(agent, "base_url", "") or ""))
  30. if not parts.scheme or not parts.hostname:
  31. return False
  32. # "localhost" can resolve to ::1 first and wait out a timeout on a server bound to IPv4.
  33. netloc = parts.netloc.replace("localhost", "127.0.0.1", 1) if parts.hostname == "localhost" else parts.netloc
  34. opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
  35. try:
  36. with opener.open(f"{parts.scheme}://{netloc}/health", timeout=3) as response:
  37. return response.status == 200
  38. except (OSError, ValueError, http.client.HTTPException):
  39. return False
  40. def retry_recovered_local_server(agent) -> bool:
  41. """Turn start: when the stale breaker is open, the agent talks to llama-server, and the
  42. server answers its health check, let one attempt through. Within a turn the breaker stops
  43. endless retries; across turns, a local server that crashed or fell behind and has since
  44. recovered gets the user's next message instead of an instant error. The router answers
  45. ``/health`` even while its model is stuck, so one more stale attempt shuts the breaker again."""
  46. from agent.chat_completion_helpers import _stale_streak
  47. giveup, streak = stale_giveup(), _stale_streak(agent)
  48. if giveup <= 0 or streak < giveup or not _streams_llamacpp(agent) or not _local_server_answers(agent):
  49. return False
  50. logger.info("Local model server answers its health check after %d stale attempts; trying once more.", streak)
  51. agent._consecutive_stale_streams = giveup - 1
  52. return True
  53. def _managed_local_load_notice(agent, api_kwargs: dict) -> Optional[str]:
  54. """Live phase notice ("⏳ loading <model> into memory — N%" / "⚙ processing
  55. prompt — P%") while the managed local server works before the first token;
  56. None when neither applies. Otherwise a cold load reads as a generic stall."""
  57. try:
  58. base = str(getattr(agent, "base_url", "") or "")
  59. if not base:
  60. return None
  61. from agent.chat_completion_helpers import estimate_request_context_tokens
  62. from hermes_cli.local_runtime.load_progress import get_loading_progress, get_prefill_progress
  63. from hermes_cli.local_runtime.supervisor import state_path
  64. state = json.loads(state_path().read_text(encoding="utf-8-sig"))
  65. managed = urlparse(str(state.get("base_url", ""))).netloc.lower()
  66. if not managed or urlparse(base).netloc.lower() != managed:
  67. return None
  68. model = str(api_kwargs.get("model", ""))
  69. progress = get_loading_progress().get(model)
  70. if progress is not None:
  71. return (f"⏳ loading {model} into memory — {progress['percent']}% "
  72. "(responses start once the model is loaded)")
  73. prefill = get_prefill_progress(model)
  74. if prefill is None:
  75. return None
  76. processed = int(prefill["processed"])
  77. total = estimate_request_context_tokens(api_kwargs)
  78. if total and total >= processed:
  79. return f"⚙ processing prompt — {max(0, min(100, round(processed / total * 100)))}%"
  80. # Counter past the estimate (estimator undercounted): no honest denominator, label-only.
  81. return "⚙ processing prompt"
  82. except Exception:
  83. # A status line must never break the stream it describes.
  84. logger.debug("local load notice unavailable", exc_info=True)
  85. return None