| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293 |
- """Pre-turn dead-connection sweep over an agent's OpenAI client pools (primary + cached request
- client). Sibling of ``agent.agent_runtime_helpers``, which owns the pool-socket walk."""
- from __future__ import annotations
- import contextlib
- import logging
- from typing import Any
- logger = logging.getLogger("run_agent") # origin module's logger name: log records / caplog filters unchanged
- def _socket_is_dead(sock) -> bool:
- """Probe socket health with a non-blocking recv peek."""
- import socket as _socket
- try:
- sock.setblocking(False)
- return sock.recv(1, _socket.MSG_PEEK | _socket.MSG_DONTWAIT) == b""
- except BlockingIOError:
- return False # no data available: socket is healthy
- except OSError:
- return True
- finally:
- with contextlib.suppress(OSError):
- sock.setblocking(True)
- def _count_dead_pool_sockets(client: Any) -> int:
- """Return the number of sockets in ``client``'s pool that have reached EOF (CLOSE-WAIT)."""
- from agent.agent_runtime_helpers import _iter_pool_sockets
- return sum(1 for sock in _iter_pool_sockets(client) if _socket_is_dead(sock))
- def cleanup_dead_connections(agent) -> bool:
- """Detect and clean up dead TCP connections owned by an agent.
- Long-lived chat-completion turns use a cached per-request OpenAI client
- for sequential calls, while ``agent.client`` remains the primary client
- used for runtime rebuilds and provider management. Inspect both pools:
- a provider FIN on the cached client otherwise stays in CLOSE-WAIT until
- session teardown and can be handed back to the next request.
- Dead sockets on the primary client rebuild that client. Dead sockets on
- the request cache go through the cache's ownership-aware teardown hook so
- an in-flight worker is aborted safely and an idle client is closed by its
- owner. Returns True if dead connections were found and cleaned up.
- """
- primary = getattr(agent, "client", None)
- cache = getattr(agent, "_request_client_cache", None)
- request_client = cache.get("client") if isinstance(cache, dict) else None
- clients: list[tuple[str, Any]] = []
- if primary is not None:
- clients.append(("primary", primary))
- if request_client is not None and request_client is not primary:
- clients.append(("request", request_client))
- if not clients:
- return False
- dead_clients: list[tuple[str, Any, int]] = []
- for label, client in clients:
- try:
- dead_count = _count_dead_pool_sockets(client)
- if dead_count:
- dead_clients.append((label, client, dead_count))
- except Exception:
- # Best-effort probe over private httpcore internals: one client's failure must not
- # skip the other's check or abort the turn.
- logger.debug("Dead connection check error for %s client", label, exc_info=True)
- if not dead_clients:
- return False
- total_dead = sum(count for _label, _client, count in dead_clients)
- labels = ", ".join(label for label, _client, _count in dead_clients)
- logger.warning(
- "Found %d dead connection(s) in %s client pool(s) — cleaning up",
- total_dead,
- labels,
- )
- for label, _client, _count in dead_clients:
- try:
- if label == "primary":
- agent._replace_primary_openai_client(reason="dead_connection_cleanup")
- else:
- # This hook clears the cache and chooses close() vs socket
- # shutdown based on whether another worker owns the client.
- agent._close_cached_request_openai_client(reason="dead_connection_cleanup")
- except Exception:
- # Cleaning one pool must not stop the other pool's cleanup.
- logger.debug("Dead connection cleanup error for %s client", label, exc_info=True)
- return True
|