telegram_network.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307
  1. """Telegram network helpers: a hostname-preserving fallback transport (Host + SNI stay
  2. api.telegram.org while TCP retries known IPv4 literals) plus DoH-based IP discovery."""
  3. from __future__ import annotations
  4. import asyncio
  5. import ipaddress
  6. import logging
  7. import socket
  8. from typing import Iterable, Optional
  9. import httpx
  10. from agent.ssl_verify import platform_ssl_context
  11. logger = logging.getLogger(__name__)
  12. _TELEGRAM_API_HOST = "api.telegram.org"
  13. def _describe_transport_error(error: Exception) -> str:
  14. """Return a non-empty, secret-safe exception representation for diagnostics."""
  15. try:
  16. from agent.redact import redact_sensitive_text
  17. return redact_sensitive_text(repr(error), force=True)
  18. except Exception:
  19. return type(error).__name__
  20. # TCP keepalive so a half-open/CLOSE-WAIT long-poll errors out instead of blocking getUpdates forever
  21. # (Windows leaves SO_KEEPALIVE off). Idle/interval knobs are best-effort per Python/OS combo.
  22. # Windows does not enable SO_KEEPALIVE on new sockets by default, so a dead api.telegram.org peer can hang
  23. # forever (#87057).
  24. _TCP_KEEPALIVE_IDLE_S = 30
  25. _TCP_KEEPALIVE_INTERVAL_S = 10
  26. _TCP_KEEPALIVE_COUNT = 3
  27. def tcp_keepalive_socket_options() -> list[tuple[int, int, int]]:
  28. """``setsockopt`` tuples for httpx ``socket_options``: always SO_KEEPALIVE, plus idle/interval/count
  29. when the interpreter exposes those option names."""
  30. options: list[tuple[int, int, int]] = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
  31. idle = getattr(socket, "TCP_KEEPIDLE", None) or getattr(socket, "TCP_KEEPALIVE", None)
  32. for opt, value in ((idle, _TCP_KEEPALIVE_IDLE_S), (getattr(socket, "TCP_KEEPINTVL", None), _TCP_KEEPALIVE_INTERVAL_S),
  33. (getattr(socket, "TCP_KEEPCNT", None), _TCP_KEEPALIVE_COUNT)):
  34. if opt is not None:
  35. options.append((socket.IPPROTO_TCP, opt, value))
  36. return options
  37. # DNS-over-HTTPS providers: discover Telegram API IPs the (possibly unreachable) local resolver may not
  38. # return. Bounded so connect() isn't delayed.
  39. _DOH_TIMEOUT = 4.0
  40. _DOH_PROVIDERS: list[dict] = [
  41. {"url": "https://dns.google/resolve", "params": {"name": _TELEGRAM_API_HOST, "type": "A"}, "headers": {}},
  42. {
  43. "url": "https://cloudflare-dns.com/dns-query", "params": {"name": _TELEGRAM_API_HOST, "type": "A"},
  44. "headers": {"Accept": "application/dns-json"}},
  45. ]
  46. # Last-resort IPv4 Bot API endpoints (149.154.160.0/20). Used when DoH is blocked AND as
  47. # first-try connect targets so a blackholed IPv6 AAAA for the hostname can't pin initialize().
  48. SEED_FALLBACK_IPS: list[str] = ["149.154.166.110", "149.154.167.220"]
  49. _UNSET = object()
  50. def _resolve_proxy_url(target_hosts=None) -> str | None:
  51. from gateway.platforms.base import resolve_proxy_url # env vars + macOS system proxy
  52. return resolve_proxy_url("TELEGRAM_PROXY", target_hosts=target_hosts)
  53. class TelegramFallbackTransport(httpx.AsyncBaseTransport):
  54. """Reach the Bot API via known IPv4 literals first, dual-stack hostname last. Host + SNI stay on
  55. api.telegram.org (like ``curl --resolve``) so a blackholed IPv6 AAAA can't pin initialize()."""
  56. # Bound every pool: httpx's 100-connection default × (wedged endpoint + seed IPs) can outgrow the fd limit.
  57. # See #63311.
  58. _POOL_LIMITS = httpx.Limits(max_connections=8, max_keepalive_connections=4)
  59. def __init__(self, fallback_ips: Iterable[str], **transport_kwargs):
  60. self._fallback_ips = list(dict.fromkeys(_normalize_fallback_ips(fallback_ips)))
  61. proxy_url = _resolve_proxy_url(target_hosts=[_TELEGRAM_API_HOST, *self._fallback_ips])
  62. if proxy_url and "proxy" not in transport_kwargs:
  63. transport_kwargs["proxy"] = proxy_url
  64. transport_kwargs.setdefault("limits", self._POOL_LIMITS)
  65. transport_kwargs.setdefault("socket_options", tcp_keepalive_socket_options())
  66. self._transport_kwargs = transport_kwargs
  67. self._primary = httpx.AsyncHTTPTransport(**transport_kwargs)
  68. self._primary_lock = asyncio.Lock()
  69. self._primary_closed = False
  70. # Built on demand and discarded on failure — see _reset_fallback.
  71. self._fallbacks: dict[str, httpx.AsyncHTTPTransport] = {}
  72. self._fallback_lock = asyncio.Lock()
  73. # ``_UNSET`` / ``None`` / ``str`` = no sticky yet / sticky hostname / sticky IPv4.
  74. self._sticky_ip: object = _UNSET
  75. self._sticky_lock = asyncio.Lock()
  76. self._last_failure: tuple[str, str] | None = None
  77. async def _get_fallback(self, ip: str) -> httpx.AsyncHTTPTransport:
  78. async with self._fallback_lock:
  79. transport = self._fallbacks.get(ip)
  80. if transport is None:
  81. transport = httpx.AsyncHTTPTransport(**self._transport_kwargs)
  82. self._fallbacks[ip] = transport
  83. return transport
  84. async def _reset_primary(self, transport: httpx.AsyncHTTPTransport) -> None:
  85. # Retryable primary failures leave half-closed sockets in the pool; replace the generation first.
  86. async with self._primary_lock:
  87. if self._primary_closed or transport is not self._primary:
  88. return
  89. self._primary = httpx.AsyncHTTPTransport(**self._transport_kwargs)
  90. try:
  91. await transport.aclose()
  92. except Exception as exc:
  93. logger.debug("[Telegram] Error closing primary transport: %s", exc)
  94. async def _reset_fallback(self, ip: str) -> None:
  95. """Discard a failed fallback pool: a peer-closed connect leaves a CLOSE_WAIT socket in it, and the
  96. poisoned pool would leak one fd per retry.
  97. Retaining the poisoned pool leaks one descriptor per retry until the process hits its file limit and
  98. can no longer accept connections or resolve DNS (#63311).
  99. """
  100. async with self._fallback_lock:
  101. transport = self._fallbacks.pop(ip, None)
  102. if transport is None:
  103. return
  104. try:
  105. await transport.aclose()
  106. except Exception as exc: # closing a broken pool must never mask the real error
  107. logger.debug("[Telegram] Error closing fallback transport %s: %s", ip, exc)
  108. def _attempt_order(self) -> list[Optional[str]]:
  109. """Sticky path first, then IPv4 literals, dual-stack hostname last (a blackholed IPv6 path never
  110. errors — Happy Eyeballs waits on AAAA until the OS TCP timeout and can pin the loop)."""
  111. order: list[Optional[str]] = []
  112. if self._sticky_ip is not _UNSET:
  113. order.append(None if self._sticky_ip is None else str(self._sticky_ip))
  114. order.extend(ip for ip in self._fallback_ips if ip not in order)
  115. if None not in order:
  116. order.append(None)
  117. return order
  118. async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
  119. if request.url.host != _TELEGRAM_API_HOST or not self._fallback_ips:
  120. return await self._primary.handle_async_request(request)
  121. last_error: Exception | None = None
  122. for ip in self._attempt_order():
  123. candidate = request if ip is None else _rewrite_request_for_ip(request, ip)
  124. transport = self._primary if ip is None else await self._get_fallback(ip)
  125. try:
  126. response = await transport.handle_async_request(candidate)
  127. if self._last_failure is not None:
  128. failed_path, failure = self._last_failure
  129. self._last_failure = None
  130. logger.info(
  131. "[Telegram] Telegram API transport recovered via %s after %s failed: %s",
  132. ip or _TELEGRAM_API_HOST,
  133. failed_path,
  134. failure,
  135. )
  136. if self._sticky_ip is _UNSET or self._sticky_ip != ip:
  137. async with self._sticky_lock:
  138. if self._sticky_ip is _UNSET or self._sticky_ip != ip:
  139. self._sticky_ip = ip
  140. if ip is not None:
  141. log = logger.warning if last_error is not None else logger.info
  142. log("[Telegram] Using sticky IPv4 Telegram API path %s (dual-stack hostname tried last — #87015)", ip)
  143. return response
  144. except Exception as exc:
  145. last_error = exc
  146. if not _is_retryable_connect_error(exc):
  147. raise
  148. path = ip or _TELEGRAM_API_HOST
  149. failure = _describe_transport_error(exc)
  150. self._last_failure = (path, failure)
  151. if self._sticky_ip is not _UNSET and ip == self._sticky_ip:
  152. async with self._sticky_lock:
  153. if self._sticky_ip is not _UNSET and self._sticky_ip == ip:
  154. self._sticky_ip = _UNSET
  155. logger.warning(
  156. "[Telegram] Sticky Telegram path %s failed; re-walking IPv4 literals before the hostname",
  157. ip if ip is not None else "api.telegram.org")
  158. if ip is None:
  159. await self._reset_primary(transport)
  160. logger.warning("[Telegram] Dual-stack api.telegram.org path failed (%s)", failure)
  161. continue
  162. logger.warning("[Telegram] IPv4 Telegram API IP %s failed: %s", ip, failure)
  163. await self._reset_fallback(ip)
  164. continue
  165. if last_error is None:
  166. raise RuntimeError("All Telegram fallback IPs exhausted but no error was recorded")
  167. raise last_error
  168. async def aclose(self) -> None:
  169. async with self._primary_lock:
  170. self._primary_closed = True
  171. primary = self._primary
  172. await primary.aclose()
  173. async with self._fallback_lock:
  174. transports = list(self._fallbacks.values())
  175. self._fallbacks.clear()
  176. for transport in transports:
  177. await transport.aclose()
  178. def _normalize_fallback_ips(values: Iterable[str]) -> list[str]:
  179. normalized: list[str] = []
  180. for value in values:
  181. raw = str(value).strip()
  182. if not raw:
  183. continue
  184. try:
  185. addr = ipaddress.ip_address(raw)
  186. except ValueError:
  187. logger.warning("Ignoring invalid Telegram fallback IP: %r", raw)
  188. continue
  189. if addr.version != 4:
  190. logger.warning("Ignoring non-IPv4 Telegram fallback IP: %s", raw)
  191. elif addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_unspecified:
  192. logger.warning("Ignoring private/internal Telegram fallback IP: %s", raw)
  193. else:
  194. normalized.append(str(addr))
  195. return normalized
  196. def parse_fallback_ip_env(value: str | None) -> list[str]:
  197. return _normalize_fallback_ips(part.strip() for part in value.split(",")) if value else []
  198. def _resolve_system_dns() -> set[str]:
  199. """Return the IPv4 addresses that the OS resolver gives for api.telegram.org."""
  200. try:
  201. results = socket.getaddrinfo(_TELEGRAM_API_HOST, 443, socket.AF_INET)
  202. return {addr[4][0] for addr in results}
  203. except Exception:
  204. return set()
  205. async def _query_doh_provider(client: httpx.AsyncClient, provider: dict) -> list[str]:
  206. """Query one DoH provider and return A-record IPs."""
  207. try:
  208. resp = await client.get(provider["url"], params=provider["params"], headers=provider["headers"])
  209. resp.raise_for_status()
  210. data = resp.json()
  211. ips: list[str] = []
  212. for answer in data.get("Answer", []):
  213. if answer.get("type") != 1: # A record
  214. continue
  215. raw = answer.get("data", "").strip()
  216. try:
  217. ipaddress.ip_address(raw)
  218. except ValueError:
  219. continue
  220. ips.append(raw)
  221. return ips
  222. except Exception as exc:
  223. logger.debug("DoH query to %s failed: %s", provider["url"], exc)
  224. return []
  225. async def discover_fallback_ips() -> list[str]:
  226. """Resolve api.telegram.org via Google + Cloudflare DoH; unique A records, in order. IPs matching the
  227. system resolver are deliberately KEPT (often the most reliable path). Falls back to
  228. ``SEED_FALLBACK_IPS`` only when DoH yields nothing usable.
  229. IPs that match the local system resolver are kept rather than excluded: in many networks the system-DNS
  230. IP is the most reliable path to api.telegram.org and a transient primary-path failure should be retried
  231. against the same address via the IP-rewrite path before the seed list is consulted (#14520).
  232. """
  233. async with httpx.AsyncClient(timeout=httpx.Timeout(_DOH_TIMEOUT), verify=platform_ssl_context()) as client:
  234. system_dns_task = asyncio.ensure_future(asyncio.to_thread(_resolve_system_dns))
  235. results = await asyncio.gather(*[_query_doh_provider(client, p) for p in _DOH_PROVIDERS], return_exceptions=True)
  236. # The getaddrinfo leg has no timeout of its own and only feeds the log line below — bound it.
  237. # The system-resolver leg runs socket.getaddrinfo in a worker thread with no timeout of its own — a
  238. # wedged OS resolver (broken VPN/DNS) can sit for minutes. Its result only feeds the no-usable-answers
  239. # log line below, so it must never gate discovery: bound it and move on (#63309). The DoH legs are
  240. # already bounded by the client timeout above.
  241. system_ips: set[str] = set()
  242. try:
  243. system_result = await asyncio.wait_for(system_dns_task, timeout=_DOH_TIMEOUT)
  244. if isinstance(system_result, set):
  245. system_ips = system_result
  246. except Exception:
  247. logger.debug("System-DNS resolution for %s did not complete in time", _TELEGRAM_API_HOST)
  248. doh_ips = [ip for r in results if isinstance(r, list) for ip in r]
  249. validated = _normalize_fallback_ips(list(dict.fromkeys(doh_ips))) # dedupe, keep order
  250. if validated:
  251. logger.debug("Discovered Telegram fallback IPs via DoH: %s", ", ".join(validated))
  252. return validated
  253. logger.info(
  254. "DoH discovery yielded no usable IPs (system DNS: %s); using seed fallback IPs %s",
  255. ", ".join(system_ips) or "unknown", ", ".join(SEED_FALLBACK_IPS))
  256. return list(SEED_FALLBACK_IPS)
  257. def _rewrite_request_for_ip(request: httpx.Request, ip: str) -> httpx.Request:
  258. original_host = request.url.host or _TELEGRAM_API_HOST
  259. url = request.url.copy_with(host=ip)
  260. headers = request.headers.copy()
  261. headers["host"] = original_host
  262. extensions = dict(request.extensions)
  263. extensions["sni_hostname"] = original_host
  264. return httpx.Request(method=request.method, url=url, headers=headers, stream=request.stream, extensions=extensions)
  265. def _is_retryable_connect_error(exc: Exception) -> bool:
  266. return isinstance(exc, (httpx.ConnectTimeout, httpx.ConnectError))