client.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559
  1. """HTTP client for managed gateway connector routes.
  2. A dispatch-local idempotency key permits one execute retry; connections are never retried because the gateway cannot deduplicate authorization starts.
  3. """
  4. from __future__ import annotations
  5. import logging
  6. import os
  7. import time
  8. import uuid
  9. from dataclasses import dataclass
  10. from typing import Any, Callable, Mapping, Optional, Protocol, Sequence
  11. import requests
  12. from pydantic import ValidationError
  13. from tools.connectors.gateway import wire
  14. from tools.connectors.gateway.config import session_platform
  15. from tools.connectors.gateway.errors import (
  16. GatewayAuthError,
  17. GatewayUnavailable,
  18. ToolGatewayError,
  19. parse_gateway_error,
  20. )
  21. from tools.connectors.gateway.merge import PlannedCall
  22. from tools.managed_gateway_identity import CallRef, connector_headers, log_connector_outcome, response_request_id
  23. logger = logging.getLogger(__name__)
  24. __all__ = ["ConnectorClient", "Transport", "return_to_args"]
  25. DEFAULT_TIMEOUT_SECONDS = 30.0
  26. EXECUTE_TIMEOUT_SECONDS = 60.0
  27. # Search is on every enabled tool-search path and must degrade to local-only results.
  28. SEARCH_TIMEOUT_SECONDS = 30.0
  29. SCHEMAS_TIMEOUT_SECONDS = 10.0
  30. STORAGE_TIMEOUT_SECONDS = 60.0
  31. _STORAGE_CHUNK_BYTES = 64 * 1024
  32. _STORAGE_PUT_ROUTE = "connectors.storage.put"
  33. _STORAGE_GET_ROUTE = "connectors.storage.get"
  34. _MAX_RETRIES = 1
  35. # Execute retries these once under the same idempotency key; the gateway replays a settled call.
  36. _RETRY_ONCE_STATUSES = frozenset({502, 503, 504})
  37. # A 429 asking for a longer wait is returned to the model rather than holding the turn.
  38. _MAX_RETRY_AFTER_SECONDS = 10.0
  39. # Gateway origins that rejected the single-call execute form; they get the array form for the process.
  40. _LEGACY_EXECUTE_ORIGINS: set[str] = set()
  41. _EXECUTE_ROUTE = "connectors.execute"
  42. @dataclass(frozen=True)
  43. class _Reply:
  44. """A 2xx answer before its body is checked; the outcome line is written once the body is read."""
  45. payload: Any
  46. status: int
  47. request_id: Optional[str]
  48. route: str
  49. class Transport(Protocol):
  50. def request(
  51. self,
  52. method: str,
  53. url: str,
  54. *,
  55. headers: Optional[dict] = None,
  56. json: Optional[dict] = None,
  57. timeout: Optional[float] = None,
  58. ) -> Any: ...
  59. class StorageTransport(Protocol):
  60. """Plain HTTP to presigned storage URLs: never carries the gateway's auth headers."""
  61. def request(
  62. self,
  63. method: str,
  64. url: str,
  65. *,
  66. headers: Optional[dict] = None,
  67. data: Optional[bytes] = None,
  68. timeout: Optional[float] = None,
  69. stream: bool = False,
  70. ) -> Any: ...
  71. def _default_transport() -> Transport:
  72. return requests
  73. def _default_endpoint_resolver() -> Optional[str]:
  74. """Resolve the connector deployment origin directly, not as a vendor passthrough."""
  75. from tools.managed_gateway_auth import connector_gateway_origin
  76. try:
  77. return connector_gateway_origin() or None
  78. except ValueError:
  79. return None
  80. def _default_header_provider(url: str) -> dict:
  81. from tools.managed_gateway_auth import managed_gateway_auth_headers
  82. return managed_gateway_auth_headers(url)
  83. class ConnectorClient:
  84. def __init__(
  85. self,
  86. *,
  87. transport: Optional[Transport] = None,
  88. endpoint_resolver: Optional[Callable[[], Optional[str]]] = None,
  89. header_provider: Optional[Callable[[str], dict]] = None,
  90. call: Optional[CallRef] = None,
  91. initiator: Optional[str] = None,
  92. sleep: Callable[[float], None] = time.sleep,
  93. storage: Optional[StorageTransport] = None,
  94. ) -> None:
  95. self._transport = transport or _default_transport()
  96. self._storage = storage or requests
  97. self._endpoint_resolver = endpoint_resolver or _default_endpoint_resolver
  98. self._header_provider = header_provider or _default_header_provider
  99. # The model call this client serves, and who asked for a connect; both go on the wire.
  100. self._call = call
  101. self._initiator = initiator
  102. self._sleep = sleep
  103. def search(self, queries: Sequence[dict[str, Any]]) -> dict[str, Any]:
  104. body = wire.ConnectorSearchRequest(
  105. queries=[wire.ConnectorSearchQuery(**q) for q in queries]
  106. ).model_dump(by_alias=True, exclude_none=True)
  107. reply = self._post(
  108. wire.CONNECTOR_SEARCH_PATH, body,
  109. timeout=SEARCH_TIMEOUT_SECONDS, retries=0,
  110. )
  111. return self._checked(wire.ConnectorSearchResponse, reply, "connector search").model_dump()
  112. def schemas(self, tools: Sequence[str]) -> dict[str, Any]:
  113. body = wire.ConnectorSchemasRequest(tools=list(tools)).model_dump(
  114. by_alias=True
  115. )
  116. reply = self._post(
  117. wire.CONNECTOR_SCHEMAS_PATH, body, timeout=SCHEMAS_TIMEOUT_SECONDS
  118. )
  119. return self._checked(wire.ConnectorSchemasResponse, reply, "connector schemas").model_dump()
  120. def connections(
  121. self, connectors: Sequence[str], *, reinitiate: bool = False, alias: Optional[str] = None,
  122. connection_id: Optional[str] = None, return_to: Optional[str] = None, op: Optional[str] = None,
  123. ) -> dict[str, Any]:
  124. """Never retry: the gateway cannot deduplicate authorization starts. ``alias`` names the one
  125. account the request connects (or, with ``reinitiate``, repairs); ``connection_id`` names an
  126. existing account to repair, named or not."""
  127. body = wire.ConnectorConnectionsRequest(
  128. connectors=list(connectors), reinitiate=reinitiate, alias=alias, connection_id=connection_id,
  129. return_to=return_to, op=op, initiator=self._initiator,
  130. ).model_dump(by_alias=True, exclude_none=True)
  131. reply = self._post(wire.CONNECTOR_CONNECTIONS_PATH, body, retries=0)
  132. return self._checked(wire.ConnectorConnectionsResponse, reply, "connector connections").model_dump()
  133. def list_connectors(self, *, timeout: float = DEFAULT_TIMEOUT_SECONDS) -> list[dict[str, Any]]:
  134. """Every page of the session's toolkit list, each typed whole. ``timeout`` applies per page: the
  135. watcher bounds it by the operation's remaining deadline so a stalled page cannot hold the
  136. operation open."""
  137. items: list[dict[str, Any]] = []
  138. cursor: Optional[str] = None
  139. for _ in range(20):
  140. path = f"{wire.CONNECTORS_PATH}?limit=50"
  141. if cursor:
  142. path += f"&cursor={cursor}"
  143. page = self._checked(wire.ConnectorListResponse, self._request("GET", path, None, timeout=timeout),
  144. "connector list page")
  145. items.extend(item.model_dump(by_alias=True) for item in page.items)
  146. cursor = page.next_cursor
  147. if not cursor:
  148. return items
  149. raise ToolGatewayError("connector list pagination incomplete", code="INVALID_RESPONSE")
  150. def account_status(
  151. self, connection_id: str, *, timeout: float = DEFAULT_TIMEOUT_SECONDS
  152. ) -> Optional[dict[str, Any]]:
  153. """One account's row; ``None`` when the gateway no longer knows it. A 429 raises ``RateLimited``.
  154. ``timeout`` is the watcher's remaining deadline, so a stalled read cannot outlive its operation."""
  155. try:
  156. reply = self._request("GET", f"{wire.CONNECTOR_ACCOUNTS_PATH}/{connection_id}", None, timeout=timeout)
  157. except GatewayUnavailable as exc:
  158. if exc.code == "connection_not_found":
  159. return None
  160. raise
  161. return self._checked(wire.ConnectorAccount, reply, "connector account").model_dump(by_alias=True)
  162. def stage_file(self, *, toolkit: str, tool: str, filename: str, mimetype: str, size: int) -> dict[str, Any]:
  163. """A presigned upload for one file input: ``{upload_url, headers, descriptor, expires_at}``.
  164. Never retried: each stage mints a new storage key."""
  165. body = wire.ConnectorFileUploadRequest(
  166. toolkit=toolkit, tool=tool, filename=filename, mimetype=mimetype, size=size,
  167. ).model_dump(by_alias=True)
  168. reply = self._post(wire.CONNECTOR_FILE_UPLOAD_PATH, body, retries=0)
  169. return self._checked(wire.ConnectorFileUploadResponse, reply, "connector file upload").model_dump()
  170. def put_file(self, url: str, headers: Mapping[str, str], body: bytes) -> None:
  171. """PUT the bytes to a presigned storage URL with exactly the headers the gateway returned."""
  172. try:
  173. response = self._storage.request("PUT", url, headers=dict(headers), data=body,
  174. timeout=STORAGE_TIMEOUT_SECONDS)
  175. except requests.RequestException as exc:
  176. log_connector_outcome(_STORAGE_PUT_ROUTE, None, code="TRANSPORT_ERROR", call=self._call)
  177. raise ToolGatewayError(f"storage upload failed: {type(exc).__name__}", code="TRANSPORT_ERROR") from exc
  178. status = int(getattr(response, "status_code", 0))
  179. failed = not 200 <= status < 300
  180. log_connector_outcome(_STORAGE_PUT_ROUTE, status, code=f"HTTP_{status}" if failed else None, call=self._call)
  181. if failed:
  182. raise ToolGatewayError(f"storage upload answered {status}", code=f"HTTP_{status}", status=status)
  183. def get_file(self, url: str, sink: Callable[[bytes], None], *, max_bytes: int) -> None:
  184. """Stream a presigned storage GET into ``sink``; more than ``max_bytes`` raises ``TOO_LARGE``."""
  185. try:
  186. response = self._storage.request("GET", url, headers={}, timeout=STORAGE_TIMEOUT_SECONDS, stream=True)
  187. except requests.RequestException as exc:
  188. log_connector_outcome(_STORAGE_GET_ROUTE, None, code="TRANSPORT_ERROR", call=self._call)
  189. raise ToolGatewayError(f"download failed: {type(exc).__name__}", code="TRANSPORT_ERROR") from exc
  190. status = int(getattr(response, "status_code", 0))
  191. try:
  192. if not 200 <= status < 300:
  193. log_connector_outcome(_STORAGE_GET_ROUTE, status, code=f"HTTP_{status}", call=self._call)
  194. raise ToolGatewayError(f"download answered {status}", code=f"HTTP_{status}", status=status)
  195. received = 0
  196. for chunk in response.iter_content(chunk_size=_STORAGE_CHUNK_BYTES):
  197. received += len(chunk)
  198. if received > max_bytes:
  199. log_connector_outcome(_STORAGE_GET_ROUTE, status, code="TOO_LARGE", call=self._call)
  200. raise ToolGatewayError(f"file is larger than {max_bytes // (1024 * 1024)} MiB",
  201. code="TOO_LARGE", status=status)
  202. sink(chunk)
  203. except requests.RequestException as exc:
  204. log_connector_outcome(_STORAGE_GET_ROUTE, status, code="TRANSPORT_ERROR", call=self._call)
  205. raise ToolGatewayError(f"download failed: {type(exc).__name__}", code="TRANSPORT_ERROR") from exc
  206. finally:
  207. response.close()
  208. log_connector_outcome(_STORAGE_GET_ROUTE, status, call=self._call)
  209. def _checked(self, model: Any, reply: _Reply, what: str) -> Any:
  210. """Parse a 2xx body and write its outcome line: a body that does not parse is a failure,
  211. logged with the gateway's request id like any failed status."""
  212. try:
  213. parsed = self._parse(model, reply.payload, what)
  214. except ToolGatewayError as exc:
  215. exc.status, exc.request_id = reply.status, reply.request_id
  216. self._log(reply, exc.code)
  217. raise
  218. if not isinstance(parsed, wire.ConnectorExecuteResponse):
  219. self._log(reply)
  220. return parsed
  221. # The execute envelope answers 200 even when a call failed; each failed call is a failure line.
  222. failed = [result.error.code for result in parsed.results if result.error is not None]
  223. for code in failed or [None]:
  224. self._log(reply, code)
  225. return parsed
  226. def _log(self, reply: _Reply, code: Optional[str] = None) -> None:
  227. log_connector_outcome(reply.route, reply.status, code=code, request_id=reply.request_id, call=self._call)
  228. @staticmethod
  229. def _parse(model: Any, payload: Any, what: str) -> Any:
  230. if not isinstance(payload, dict) or "error" in payload:
  231. raise ToolGatewayError(f"invalid {what}", code="INVALID_RESPONSE")
  232. try:
  233. return model.model_validate(payload)
  234. except ValidationError as exc:
  235. first = exc.errors()[0]
  236. raise ToolGatewayError(f"invalid {what}: {'.'.join(str(p) for p in first.get('loc', ()))}: {first.get('msg')}",
  237. code="INVALID_RESPONSE") from exc
  238. def execute(
  239. self, planned: Sequence[PlannedCall], *, return_to: Optional[str] = None, op: Optional[str] = None,
  240. ) -> list[dict[str, Any]]:
  241. """One result per planned call, in request order: ``{"data", "error"}``.
  242. Each call is its own single-call request; a failure status with an error body is that call's
  243. error entry. A gateway that predates the single-call form gets the legacy array once and is
  244. remembered for the process."""
  245. origin = self._endpoint_resolver()
  246. if origin and origin in _LEGACY_EXECUTE_ORIGINS:
  247. return self._execute_legacy(planned, return_to=return_to, op=op)
  248. results: list[dict[str, Any]] = []
  249. for index, plan in enumerate(planned):
  250. result = self._execute_one(plan, return_to=return_to, op=op)
  251. if result is None:
  252. if origin:
  253. _LEGACY_EXECUTE_ORIGINS.add(origin)
  254. logger.info("Connector gateway predates single-call execute; using the array form")
  255. return results + self._execute_legacy(planned[index:], return_to=return_to, op=op)
  256. results.append(result)
  257. return results
  258. def _execute_one(
  259. self, plan: PlannedCall, *, return_to: Optional[str], op: Optional[str],
  260. ) -> Optional[dict[str, Any]]:
  261. """``None`` when the gateway rejected the single-call form itself (an older gateway)."""
  262. body = wire.ConnectorExecuteSingleRequest(
  263. call=wire.ConnectorExecuteCall(connector=plan.connector, tool=plan.tool, arguments=plan.arguments),
  264. return_to=return_to, op=op,
  265. ).model_dump(by_alias=True, exclude_none=True)
  266. url = self._url(wire.CONNECTOR_EXECUTE_PATH)
  267. idempotency_key = str(uuid.uuid4())
  268. for attempt in range(1 + _MAX_RETRIES):
  269. response = self._send("POST", url, body, timeout=EXECUTE_TIMEOUT_SECONDS, idempotency_key=idempotency_key,
  270. retry_transport=attempt < _MAX_RETRIES)
  271. if response is None:
  272. continue
  273. status = int(getattr(response, "status_code", 0))
  274. payload = _safe_json(response)
  275. request_id = response_request_id(response)
  276. if 200 <= status < 300:
  277. reply = _Reply(payload, status, request_id, _EXECUTE_ROUTE)
  278. parsed = self._checked(wire.ConnectorExecuteSingleResponse, reply, "connector execute result")
  279. return _result_dict(parsed.result)
  280. headers = getattr(response, "headers", None)
  281. failure = _call_failure(payload)
  282. code = failure.code if failure else parse_gateway_error(status, payload, headers).code
  283. log_connector_outcome(_EXECUTE_ROUTE, status, code=code,
  284. request_id=(failure.request_id if failure else None) or request_id, call=self._call)
  285. if status == 400 and _rejects_single_call_form(payload):
  286. return None
  287. wait = _retry_wait(status, headers) if attempt < _MAX_RETRIES else None
  288. if wait is not None:
  289. logger.debug("Connector execute attempt %d got %d; retrying with same key", attempt + 1, status)
  290. if wait:
  291. self._sleep(wait)
  292. continue
  293. if failure is None:
  294. raise parse_gateway_error(status, payload, headers)
  295. return {"data": None, "error": _error_dict(failure, status=status, request_id=request_id)}
  296. raise ToolGatewayError("connector execute retries exhausted", code="TRANSPORT_ERROR", retryable=True)
  297. def _execute_legacy(
  298. self, planned: Sequence[PlannedCall], *, return_to: Optional[str] = None, op: Optional[str] = None,
  299. ) -> list[dict[str, Any]]:
  300. """The array form an older gateway accepts: a 200 envelope with per-call errors inside."""
  301. body = wire.ConnectorExecuteRequest(
  302. tools=[
  303. wire.ConnectorExecuteCall(
  304. connector=plan.connector, tool=plan.tool, arguments=plan.arguments
  305. )
  306. for plan in planned
  307. ],
  308. return_to=return_to, op=op,
  309. ).model_dump(by_alias=True, exclude_none=True)
  310. # Keep the key dispatch-local so its retry reuses it without a shared store.
  311. idempotency_key = str(uuid.uuid4())
  312. reply = self._post(
  313. wire.CONNECTOR_EXECUTE_PATH,
  314. body,
  315. timeout=EXECUTE_TIMEOUT_SECONDS,
  316. idempotency_key=idempotency_key,
  317. )
  318. parsed = self._checked(wire.ConnectorExecuteResponse, reply, "connector execute response")
  319. return [_result_dict(result) for result in parsed.results]
  320. def _post(
  321. self,
  322. path: str,
  323. body: dict[str, Any],
  324. *,
  325. timeout: float = DEFAULT_TIMEOUT_SECONDS,
  326. idempotency_key: Optional[str] = None,
  327. retries: int = _MAX_RETRIES,
  328. ) -> _Reply:
  329. return self._request(
  330. "POST", path, body,
  331. timeout=timeout, idempotency_key=idempotency_key, retries=retries,
  332. )
  333. def _url(self, path: str) -> str:
  334. origin = self._endpoint_resolver()
  335. if not origin:
  336. raise GatewayUnavailable(
  337. "no tool gateway origin resolves", code="NO_ORIGIN"
  338. )
  339. return f"{origin.rstrip('/')}/{path}"
  340. def _send(
  341. self, method: str, url: str, body: Optional[dict[str, Any]], *,
  342. timeout: float, idempotency_key: Optional[str], retry_transport: bool,
  343. ) -> Any:
  344. """One HTTP attempt. A transport failure returns ``None`` when the caller may retry, else raises."""
  345. headers = self._attempt_headers(url, idempotency_key)
  346. try:
  347. return self._transport.request(method, url, headers=headers, json=body, timeout=timeout)
  348. except Exception as exc:
  349. log_connector_outcome(_route_key(url.split("/", 3)[-1]), None, code="TRANSPORT_ERROR", call=self._call)
  350. if retry_transport:
  351. return None
  352. raise ToolGatewayError(
  353. f"transport failure: {exc}", code="TRANSPORT_ERROR", retryable=True
  354. ) from exc
  355. def _request(
  356. self,
  357. method: str,
  358. path: str,
  359. body: Optional[dict[str, Any]],
  360. *,
  361. timeout: float = DEFAULT_TIMEOUT_SECONDS,
  362. idempotency_key: Optional[str] = None,
  363. retries: int = _MAX_RETRIES,
  364. ) -> _Reply:
  365. """The 2xx reply, unchecked; the caller logs its outcome once the body is read. A failed
  366. status is logged and raised here."""
  367. url = self._url(path)
  368. route = _route_key(path)
  369. for attempt in range(1 + retries):
  370. response = self._send(method, url, body, timeout=timeout, idempotency_key=idempotency_key,
  371. retry_transport=attempt < retries)
  372. if response is None:
  373. continue
  374. status = int(getattr(response, "status_code", 0))
  375. request_id = response_request_id(response)
  376. if 200 <= status < 300:
  377. return _Reply(_safe_json(response), status, request_id, route)
  378. error = parse_gateway_error(status, _safe_json(response), getattr(response, "headers", None))
  379. error.request_id = error.request_id or request_id
  380. log_connector_outcome(route, status, code=error.code, request_id=error.request_id, call=self._call)
  381. if error.retryable and attempt < retries:
  382. logger.debug(
  383. "Connector %s attempt %d got %d; retrying with same key",
  384. path,
  385. attempt + 1,
  386. status,
  387. )
  388. continue
  389. raise error
  390. raise ToolGatewayError("connector request retries exhausted", code="TRANSPORT_ERROR", retryable=True)
  391. def _attempt_headers(self, url: str, idempotency_key: Optional[str]) -> dict[str, str]:
  392. """Every header of one HTTP attempt: auth, caller identity, and the idempotency key."""
  393. headers = dict(self._header_provider(url))
  394. if not headers:
  395. raise GatewayAuthError(
  396. "no portal access token available", code="NO_TOKEN", status=401
  397. )
  398. headers.update(connector_headers(self._call))
  399. headers["Content-Type"] = "application/json"
  400. if idempotency_key:
  401. headers["x-idempotency-key"] = idempotency_key
  402. return headers
  403. def return_to_args(*, op: Optional[str] = None) -> dict[str, Any]:
  404. """The ``returnTo`` / ``op`` arguments a connect or execute call carries so the vendor's done page
  405. can send the browser back to the app that asked for the connection.
  406. Only the desktop registers a URL scheme for that return, so every other surface sends neither
  407. field and keeps the vendor's own done page. The dev build registers ``hermes-dev://`` instead, and
  408. announces itself to its backend with ``HERMES_DESKTOP_DEV_SERVER``."""
  409. if session_platform() != "desktop":
  410. return {}
  411. target = "hermes-desktop-dev" if os.environ.get("HERMES_DESKTOP_DEV_SERVER") else "hermes-desktop"
  412. return {"return_to": target, "op": op} if op else {"return_to": target}
  413. def _route_key(path: str) -> str:
  414. """``v1/connectors/accounts/<id>?x`` -> ``connectors.accounts``: the route without ids or query."""
  415. parts = path.split("?", 1)[0].strip("/").split("/")[1:]
  416. if len(parts) > 2:
  417. parts = parts[:2]
  418. return ".".join(parts) or path
  419. def _result_dict(result: wire.ConnectorExecuteResult | wire.ConnectorExecuteSingleResult) -> dict[str, Any]:
  420. error = getattr(result, "error", None)
  421. entry = {"data": result.data, "error": _error_dict(error) if error is not None else None}
  422. if result.account:
  423. entry["account"] = result.account
  424. if result.data_use is not None and result.data_use.requires:
  425. entry["data_use"] = list(result.data_use.requires)
  426. return entry
  427. def _error_dict(
  428. failure: wire.ConnectorToolError, *, status: Optional[int] = None, request_id: Optional[str] = None,
  429. ) -> dict[str, Any]:
  430. """The plain error entry merge renders. ``status`` and ``request_id`` are for logs, never the model."""
  431. error: dict[str, Any] = {"code": failure.code, "message": failure.message}
  432. family = wire.execute_error_family(failure.code, failure.family)
  433. if family:
  434. error["family"] = family
  435. for key, value in (("connector", failure.connector), ("connect_url", failure.connect_url),
  436. ("connection_id", failure.connection_id), ("hint", failure.hint)):
  437. if value:
  438. error[key] = value
  439. if status is not None:
  440. error["status"] = status
  441. request_id = getattr(failure, "request_id", None) or request_id
  442. if request_id:
  443. error["request_id"] = str(request_id)
  444. return error
  445. def _call_failure(payload: Any) -> Optional[wire.ConnectorExecuteFailure]:
  446. """A single-call failure body: a flat ``{code, message}`` the gateway marks with a family or a code
  447. this client knows. Anything else (auth, dark route, idempotency conflict) stays a route failure."""
  448. if not isinstance(payload, Mapping) or not isinstance(payload.get("code"), str):
  449. return None
  450. if payload.get("family") is None and payload["code"] not in wire.EXECUTE_ERROR_CODES:
  451. return None
  452. try:
  453. return wire.ConnectorExecuteFailure.model_validate(
  454. {"message": "The connector gateway reported an error.", **payload})
  455. except ValidationError:
  456. return None
  457. def _rejects_single_call_form(payload: Any) -> bool:
  458. """An older gateway validates every body as the array form and names ``tools`` in its 400."""
  459. if not isinstance(payload, Mapping):
  460. return False
  461. issues = payload.get("issues")
  462. if isinstance(issues, list):
  463. for issue in issues:
  464. path = issue.get("path") if isinstance(issue, Mapping) else None
  465. if isinstance(path, list) and path[:1] == ["tools"]:
  466. return True
  467. envelope = payload.get("error")
  468. code = envelope.get("code") if isinstance(envelope, Mapping) else payload.get("code")
  469. if code != "VALIDATION_ERROR":
  470. return False
  471. text = envelope if isinstance(envelope, Mapping) else payload
  472. return "tools" in f"{text.get('message', '')} {text.get('details', '')}"
  473. def _retry_wait(status: int, headers: Optional[Mapping[str, Any]]) -> Optional[float]:
  474. """Seconds to wait before the one retry, or ``None`` when this status is not retried.
  475. A 429 is retried only when the gateway names a wait, and only a short one; a longer wait goes back
  476. to the model as the call's error instead of holding the turn."""
  477. if status in _RETRY_ONCE_STATUSES:
  478. return 0.0
  479. if status != 429:
  480. return None
  481. raw = (headers or {}).get("Retry-After")
  482. try:
  483. wait = float(raw)
  484. except (TypeError, ValueError):
  485. return None
  486. return wait if 0 <= wait <= _MAX_RETRY_AFTER_SECONDS else None
  487. def _safe_json(response: Any) -> Any:
  488. try:
  489. return response.json()
  490. except Exception:
  491. return getattr(response, "text", None)