delegate_tool_prefix_gate.py 3.8 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182
  1. """Prompt-cache warm-up gate for batch subagents (``delegation.prefix_stagger_seconds``).
  2. Siblings in one batch send the same tools[] and system-prompt scaffold; only the goal (their first user
  3. turn) and, when tasks carry different ``context``, the tail of the system prompt differ. Launched together,
  4. every child's first request misses the provider cache and pays the full prefix. A provider cache entry is
  5. readable once the writing response starts streaming, so the gate lets the batch's first child (the leader)
  6. run at once and holds the siblings that share its route until the leader's first response has started, it
  7. has finished, or the cap elapses, whichever comes first. The wait is gone the moment the cache can be read,
  8. so the cost of opting in is one time-to-first-token, not a fixed per-child sleep.
  9. """
  10. from __future__ import annotations
  11. import threading
  12. import time
  13. from typing import Any, Callable, Optional
  14. # Set by the streaming paths on the first chunk of a response (chat completions and Responses).
  15. _FIRST_CHUNK_ACTIVITY = "receiving stream response"
  16. _POLL_S = 0.05
  17. def _route_key(child: Any) -> tuple:
  18. return tuple(getattr(child, attr, None) for attr in ("provider", "base_url", "model", "api_mode"))
  19. def _caches_prompts(child: Any) -> bool:
  20. """Only routes Hermes marks for prompt caching benefit; for the rest the wait would be pure latency."""
  21. return getattr(child, "_use_prompt_caching", False) is True
  22. def _first_response_started(child: Any) -> bool:
  23. return (getattr(child, "session_api_calls", 0) or 0) >= 1 or (
  24. getattr(child, "_last_activity_desc", None) == _FIRST_CHUNK_ACTIVITY)
  25. def _stop_requested(child: Any) -> bool:
  26. return getattr(child, "_interrupt_requested", False) is True
  27. class PrefixCacheGate:
  28. """Holds a batch's cache-sharing siblings until the leader's first response has started."""
  29. def __init__(self, leader: Any, max_wait_s: float) -> None:
  30. self.leader, self.max_wait_s = leader, max_wait_s
  31. self._leader_route = _route_key(leader)
  32. self._leader_done = threading.Event()
  33. @classmethod
  34. def for_children(cls, children: list, max_wait_s: float) -> Optional[PrefixCacheGate]:
  35. """A gate for ``children`` (``(index, task, child)`` tuples), or None when there is nothing to share."""
  36. # The leader is always task 0 (it is submitted first, so it always gets a worker): an uncached task 0
  37. # leaves the batch ungated even when later tasks share a cached route with each other.
  38. if max_wait_s <= 0 or len(children) < 2 or not _caches_prompts(children[0][2]):
  39. return None
  40. return cls(children[0][2], max_wait_s)
  41. def follows(self, child: Any) -> bool:
  42. return child is not self.leader and _caches_prompts(child) and _route_key(child) == self._leader_route
  43. def run(self, child: Any, run: Callable[[], Any], *, parent: Any = None) -> Optional[Any]:
  44. """Run ``run`` for ``child`` after the gate opens. Returns None, without running, when the child or
  45. ``parent`` was stopped during the wait (the caller reports it interrupted)."""
  46. if child is self.leader:
  47. try:
  48. return run()
  49. finally:
  50. self._leader_done.set()
  51. if self.follows(child) and not self._wait(child, parent):
  52. return None
  53. return run()
  54. def _wait(self, child: Any, parent: Any) -> bool:
  55. deadline = time.monotonic() + self.max_wait_s
  56. while not (self._leader_done.is_set() or _first_response_started(self.leader)):
  57. if _stop_requested(child) or (parent is not None and _stop_requested(parent)):
  58. return False
  59. remaining = deadline - time.monotonic()
  60. if remaining <= 0:
  61. break
  62. self._leader_done.wait(min(_POLL_S, remaining))
  63. return True