sdk_transform_bypass.py 3.9 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980
  1. """Route bulk request payloads around the OpenAI SDK's request transform (#93650).
  2. ``responses.create`` and ``chat.completions.create`` both re-walk the whole request
  3. body against their TypedDict/union param graph client-side, with the GIL held,
  4. before any byte leaves the process. #93650 documents that walk wedging for 12+
  5. hours on a ~1.4 MB conversation — starving the TTFB/stale watchdogs whose job is
  6. to rescue this exact call; the hang is pre-network, so no socket kill helps.
  7. Hermes assembles these payloads from JSON round-trips, so they are already wire
  8. format and the walk has nothing to convert. The SDK merges ``extra_body`` into
  9. the JSON body *after* the transform (``_base_client._build_request``), so moving
  10. the bulk fields there skips the walk and yields the same request bytes.
  11. """
  12. from __future__ import annotations
  13. import os
  14. from typing import Any
  15. # Bulk request fields carrying the conversation payload, per API family. Everything
  16. # else is scalar configuration the SDK transform handles in microseconds.
  17. RESPONSES_BYPASS_FIELDS = ("input", "tools")
  18. CHAT_COMPLETIONS_BYPASS_FIELDS = ("messages", "tools")
  19. # One hatch for both API families (established by #93650); restores the typed SDK path.
  20. ESCAPE_HATCH_ENV = "HERMES_CODEX_SDK_TRANSFORM"
  21. def _is_plain_json_data(value: Any) -> bool:
  22. """True when ``value`` is purely JSON wire types; pydantic models / generators must keep the typed SDK path."""
  23. if value is None or isinstance(value, (str, int, float, bool)):
  24. return True
  25. if isinstance(value, dict):
  26. return all(isinstance(key, str) and _is_plain_json_data(item) for key, item in value.items())
  27. if isinstance(value, list):
  28. return all(_is_plain_json_data(item) for item in value)
  29. return False
  30. def bypass_sdk_request_transform(
  31. request_kwargs: dict,
  32. fields: tuple[str, ...] = RESPONSES_BYPASS_FIELDS,
  33. *,
  34. keep_slots: bool = False,
  35. ) -> dict:
  36. """Move wire-format bulk ``fields`` into ``extra_body``.
  37. Returns ``request_kwargs`` itself when nothing is safe to move, so callers use
  38. the result unconditionally. ``keep_slots`` leaves an empty-list placeholder in
  39. the typed kwargs for each moved field: it satisfies ``@required_args``
  40. (``messages`` on chat.completions) and keeps the field's position in the JSON
  41. body, so the bytes — and therefore any byte-keyed prompt cache — are unchanged.
  42. """
  43. if os.environ.get(ESCAPE_HATCH_ENV, "").strip().lower() in {"1", "true", "yes", "on"}:
  44. return request_kwargs
  45. moved = {f: request_kwargs[f] for f in fields
  46. if isinstance(request_kwargs.get(f), (dict, list)) and _is_plain_json_data(request_kwargs[f])}
  47. if not moved:
  48. return request_kwargs
  49. bypassed = {key: ([] if keep_slots else value) if key in moved else value
  50. for key, value in request_kwargs.items() if keep_slots or key not in moved}
  51. extra_body = bypassed.get("extra_body")
  52. merged = dict(extra_body) if isinstance(extra_body, dict) else {}
  53. # An explicit caller-provided extra_body entry keeps precedence (SDK post-transform merge).
  54. bypassed["extra_body"] = {**merged, **{f: v for f, v in moved.items() if f not in merged}}
  55. return bypassed
  56. def bypass_chat_sdk_request_transform(request_kwargs: dict, client: Any) -> dict:
  57. """Chat-completions bypass, gated on the real OpenAI SDK.
  58. Only the SDK performs the transform and only the SDK merges ``extra_body``
  59. afterwards. Hermes also drives chat-shaped facades that are NOT the SDK (the
  60. in-process MoA aggregator, test stand-ins); handing those an ``extra_body`` they
  61. never merge would silently send an empty conversation.
  62. """
  63. completions = getattr(getattr(client, "chat", None), "completions", None)
  64. if completions is None or not type(completions).__module__.startswith("openai."):
  65. return request_kwargs
  66. return bypass_sdk_request_transform(request_kwargs, CHAT_COMPLETIONS_BYPASS_FIELDS, keep_slots=True)