server.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303
  1. """HTTP server that forwards OpenAI-compatible requests to a configured upstream.
  2. A credential-attaching forwarder: request/response bodies are never mediated, logged, or rewritten.
  3. The one shim: after a *clean* upstream EOF, a ``text/event-stream`` response that carried a terminal
  4. ``finish_reason`` or ``lastOne: true`` but omitted ``data: [DONE]`` gets a single ``[DONE]`` frame.
  5. """
  6. from __future__ import annotations
  7. import asyncio
  8. import ipaddress
  9. import logging
  10. import signal
  11. from typing import Optional
  12. from urllib.parse import urlsplit
  13. try:
  14. import aiohttp
  15. from aiohttp import web
  16. AIOHTTP_AVAILABLE = True
  17. except ImportError:
  18. aiohttp = None # type: ignore[assignment]
  19. web = None # type: ignore[assignment]
  20. AIOHTTP_AVAILABLE = False
  21. from hermes_cli.proxy.adapters.base import UpstreamAdapter, UpstreamCredential
  22. from hermes_cli.proxy.sse_done import DONE_SSE_FRAME, SseDoneTracker, content_type_is_sse
  23. logger = logging.getLogger(__name__)
  24. # Stripped when forwarding upstream: ``host``/``content-length`` are recomputed by aiohttp,
  25. # ``authorization`` is replaced with our bearer; everything else passes through.
  26. _HOP_BY_HOP_HEADERS = frozenset({
  27. "host", "content-length", "connection", "keep-alive", "proxy-authenticate",
  28. "proxy-authorization", "te", "trailers", "transfer-encoding", "upgrade", "authorization",
  29. })
  30. # aiohttp recomputes Content-Encoding/Content-Length on stream — let it.
  31. _RESPONSE_DROP_HEADERS = _HOP_BY_HOP_HEADERS | {"content-encoding", "content-length"}
  32. DEFAULT_PORT = 8645
  33. DEFAULT_HOST = "127.0.0.1"
  34. # Mirrors api_server's MAX_REQUEST_BYTES (10 MB); client_max_size bounds every read path,
  35. # including chunked bodies.
  36. MAX_REQUEST_BYTES = 10_000_000
  37. def _require_aiohttp() -> None:
  38. if not AIOHTTP_AVAILABLE:
  39. raise RuntimeError("aiohttp is required for `hermes proxy`. Run `hermes setup` to install it.")
  40. def _json_error(status: int, message: str, code: str = "proxy_error") -> web.Response:
  41. """OpenAI-style error JSON response."""
  42. body = {"error": {"message": message, "type": code, "code": code}}
  43. return web.json_response(body, status=status)
  44. _LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
  45. # Sec-Fetch-Site values a browser sends for a request no foreign page made (typed URL / own page).
  46. _UNFORGED_FETCH_SITES = frozenset({"none", "same-origin"})
  47. # On a wildcard bind a DNS-rebound page is same-origin with its Host, so only "none" is trusted.
  48. _WILDCARD_FETCH_SITES = frozenset({"none"})
  49. def _canonical_host(host: str) -> str:
  50. # ``localhost.`` is the absolute form of ``localhost``, and ``[0:0::1]`` is ``[::1]``;
  51. # a rebound name stays foreign either way.
  52. host = host.lower().rstrip(".")
  53. try:
  54. return ipaddress.ip_address(host).compressed
  55. except ValueError:
  56. return host
  57. def _parse_authority(authority: str) -> Optional[tuple[str, int]]:
  58. """(hostname, port) of a Host header or Origin authority; None when malformed (fails closed)."""
  59. if not authority or any(c in authority for c in "@/\\?# \t"):
  60. return None
  61. try:
  62. parts = urlsplit("//" + authority)
  63. port = parts.port or 80 # an Origin omits the default port that a Host may spell out
  64. except ValueError:
  65. return None
  66. return (_canonical_host(parts.hostname), port) if parts.hostname else None
  67. def _is_wildcard(bound: str) -> bool:
  68. try:
  69. return bound == "" or ipaddress.ip_address(bound).is_unspecified
  70. except ValueError:
  71. return False
  72. def _local_request_error(
  73. host_header: str,
  74. origin: Optional[str],
  75. fetch_site: Optional[str],
  76. *,
  77. wildcard: bool,
  78. allowed_hosts: frozenset,
  79. ) -> Optional[str]:
  80. """Why a request must be refused, or None. The proxy attaches the operator's subscription
  81. credential to whatever reaches it, so a web page in the operator's browser must not be able
  82. to drive it: a loopback/specific-IP bind accepts only its own Host names (a DNS-rebound
  83. hostname fails), and any Origin other than the proxy's own is a cross-site browser request.
  84. Browsers omit Origin on GET, but they always send ``Sec-Fetch-Site``, which a page cannot
  85. forge, so it is judged too. Non-browser clients send neither and are unaffected. A wildcard
  86. bind is an explicit LAN opt-in, so any Host name may reach it; it therefore has no origin of
  87. its own to match, and a DNS-rebound page is same-origin with its Host, so every page-made
  88. browser request is refused there."""
  89. target = _parse_authority(host_header)
  90. if target is None or (not wildcard and target[0] not in allowed_hosts):
  91. return "host_not_allowed"
  92. if origin is not None:
  93. scheme, sep, rest = origin.strip().partition("://")
  94. if wildcard or not sep or scheme.lower() != "http" or _parse_authority(rest) != target:
  95. return "origin_not_allowed"
  96. trusted_sites = _WILDCARD_FETCH_SITES if wildcard else _UNFORGED_FETCH_SITES
  97. if fetch_site is not None and fetch_site.strip().lower() not in trusted_sites:
  98. return "origin_not_allowed"
  99. return None
  100. def _filter_headers(headers, drop: frozenset = _HOP_BY_HOP_HEADERS) -> dict:
  101. """Strip hop-by-hop (+ auth) headers; ``drop`` widens the set for upstream responses."""
  102. return {key: value for key, value in headers.items() if key.lower() not in drop}
  103. async def _open_upstream(request: web.Request, rel_path: str, body: bytes, cred: UpstreamCredential):
  104. """Send the request upstream with ``cred``; returns ``(session, response)`` or
  105. ``(error_response, None)``."""
  106. upstream_url = f"{cred.base_url.rstrip('/')}{rel_path}"
  107. if request.query_string: # preserved verbatim
  108. upstream_url = f"{upstream_url}?{request.query_string}"
  109. fwd_headers = _filter_headers(request.headers)
  110. fwd_headers["Authorization"] = f"{cred.token_type} {cred.bearer}"
  111. logger.debug("proxy: forwarding %s %s -> %s (body=%d bytes)", request.method, rel_path, upstream_url, len(body))
  112. try:
  113. session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None, sock_connect=15, sock_read=300))
  114. except Exception as exc: # pragma: no cover - aiohttp setup issue
  115. return _json_error(500, f"proxy session init failed: {exc}"), None
  116. try:
  117. upstream_resp = await session.request(
  118. request.method, upstream_url, data=body if body else None, headers=fwd_headers, allow_redirects=False
  119. )
  120. except RuntimeError as exc:
  121. await session.close()
  122. return _json_error(500, str(exc)), None
  123. except aiohttp.ClientError as exc:
  124. await session.close()
  125. logger.warning("proxy: upstream connection failed: %s", exc)
  126. return _json_error(502, f"upstream connection failed: {exc}", code="upstream_unreachable"), None
  127. except TimeoutError:
  128. await session.close()
  129. return _json_error(504, "upstream request timed out", code="upstream_timeout"), None
  130. except Exception:
  131. await session.close()
  132. raise
  133. return session, upstream_resp
  134. async def _stream_back(request: web.Request, session, upstream_resp) -> web.StreamResponse:
  135. """Relay status + filtered headers, then the body chunk-by-chunk, appending a missing SSE
  136. ``[DONE]`` only after a clean EOF."""
  137. resp = web.StreamResponse(
  138. status=upstream_resp.status, headers=_filter_headers(upstream_resp.headers, _RESPONSE_DROP_HEADERS)
  139. )
  140. await resp.prepare(request)
  141. done_tracker: Optional[SseDoneTracker] = None
  142. if content_type_is_sse(upstream_resp.headers):
  143. done_tracker = SseDoneTracker()
  144. try:
  145. async for chunk in upstream_resp.content.iter_any():
  146. if chunk:
  147. if done_tracker is not None:
  148. done_tracker.feed(chunk)
  149. await resp.write(chunk)
  150. if done_tracker is not None and done_tracker.should_append_done():
  151. try:
  152. await resp.write(DONE_SSE_FRAME)
  153. except Exception as exc: # client hung up at EOF — harmless
  154. logger.debug("proxy: DONE append skipped: %s", exc)
  155. except (aiohttp.ClientError, asyncio.CancelledError, OSError) as exc:
  156. if done_tracker is not None:
  157. done_tracker.mark_interrupted()
  158. logger.warning("proxy: streaming interrupted: %s", exc)
  159. finally:
  160. upstream_resp.release()
  161. await session.close()
  162. await resp.write_eof()
  163. return resp
  164. def create_app(adapter: UpstreamAdapter, bound_host: str = DEFAULT_HOST) -> web.Application:
  165. """Build the aiohttp application bound to a specific upstream adapter.
  166. Every adapter method is synchronous and blocking (the Nous adapter takes the 15s cross-process
  167. ``_auth_store_lock()`` and may POST a token refresh; xAI rotates its key pool under a lock),
  168. so all three are run via ``asyncio.to_thread`` — a contended lock or refresh must never freeze
  169. the single loop and every other in-flight streaming completion.
  170. """
  171. _require_aiohttp()
  172. # The bind address is fixed for the app's lifetime, so derive the Host policy once.
  173. bound = _canonical_host(bound_host.strip("[]"))
  174. wildcard = _is_wildcard(bound)
  175. allowed_hosts = _LOOPBACK_HOSTS | {bound}
  176. @web.middleware
  177. async def local_only(request: web.Request, handler):
  178. refusal = _local_request_error(
  179. request.headers.get("Host", ""),
  180. request.headers.get("Origin"),
  181. request.headers.get("Sec-Fetch-Site"),
  182. wildcard=wildcard,
  183. allowed_hosts=allowed_hosts,
  184. )
  185. if refusal:
  186. return _json_error(
  187. 403,
  188. "Request refused: browser requests and foreign Host names are not accepted by this proxy.",
  189. code=refusal,
  190. )
  191. return await handler(request)
  192. app = web.Application(client_max_size=MAX_REQUEST_BYTES, middlewares=[local_only])
  193. # AppKey: forward-compat with aiohttp versions that strip bare-string keys.
  194. app[web.AppKey("adapter", UpstreamAdapter)] = adapter
  195. async def handle_health(request: web.Request) -> web.Response:
  196. authenticated = await asyncio.to_thread(adapter.is_authenticated)
  197. return web.json_response({"status": "ok", "upstream": adapter.display_name, "authenticated": authenticated})
  198. async def handle_proxy(request: web.Request) -> web.StreamResponse:
  199. rel_path = "/" + request.match_info.get("tail", "").lstrip("/")
  200. if rel_path not in adapter.allowed_paths:
  201. allowed = ", ".join(sorted(adapter.allowed_paths))
  202. return _json_error(
  203. 404, f"Path /v1{rel_path} is not forwarded by this proxy. Allowed: {allowed}", code="path_not_allowed"
  204. )
  205. try:
  206. cred = await asyncio.to_thread(adapter.get_credential)
  207. except Exception as exc:
  208. logger.warning("proxy: credential resolution failed: %s", exc)
  209. return _json_error(401, str(exc), code="upstream_auth_failed")
  210. # Body read into memory once (chat/embeddings payloads are small); switch to streaming
  211. # if large multipart uploads ever need forwarding.
  212. body = await request.read()
  213. session, upstream_resp = await _open_upstream(request, rel_path, body, cred)
  214. if upstream_resp is None:
  215. return session
  216. if upstream_resp.status in {401, 429}:
  217. # One-shot retry with a refreshed/rotated credential (Nous: unconditional refresh
  218. # POST under the auth lock; xAI: pool rotation).
  219. try:
  220. retry_cred = await asyncio.to_thread(
  221. adapter.get_retry_credential, failed_credential=cred, status_code=upstream_resp.status
  222. )
  223. except Exception as exc:
  224. logger.warning("proxy: retry credential resolution failed: %s", exc)
  225. retry_cred = None
  226. if retry_cred is not None:
  227. upstream_resp.release()
  228. await session.close()
  229. session, upstream_resp = await _open_upstream(request, rel_path, body, retry_cred)
  230. if upstream_resp is None:
  231. return session
  232. return await _stream_back(request, session, upstream_resp)
  233. app.router.add_get("/health", handle_health) # never goes upstream
  234. app.router.add_route("*", "/v1/{tail:.*}", handle_proxy) # forwards if the path is allowed
  235. return app
  236. async def run_server(
  237. adapter: UpstreamAdapter,
  238. host: str = DEFAULT_HOST,
  239. port: int = DEFAULT_PORT,
  240. shutdown_event: Optional[asyncio.Event] = None,
  241. ) -> None:
  242. """Run the proxy in the current event loop until shutdown_event is set."""
  243. _require_aiohttp()
  244. app = create_app(adapter, bound_host=host)
  245. runner = web.AppRunner(app, access_log=None)
  246. await runner.setup()
  247. site = web.TCPSite(runner, host=host, port=port)
  248. await site.start()
  249. logger.info("proxy: listening on http://%s:%d/v1 -> %s", host, port, adapter.display_name)
  250. stop_event = shutdown_event or asyncio.Event()
  251. if shutdown_event is None: # we own the loop's lifetime → wire signal handlers
  252. loop = asyncio.get_running_loop()
  253. for sig in (signal.SIGINT, signal.SIGTERM):
  254. try:
  255. loop.add_signal_handler(sig, stop_event.set) # windows-footgun: ok
  256. except NotImplementedError:
  257. pass # Windows / restricted envs — Ctrl+C still raises KeyboardInterrupt
  258. try:
  259. await stop_event.wait()
  260. finally:
  261. logger.info("proxy: shutting down")
  262. await runner.cleanup()
  263. __all__ = ["AIOHTTP_AVAILABLE", "DEFAULT_HOST", "DEFAULT_PORT", "create_app", "run_server"]