network.py 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. """Bounded retries for PM-owned, idempotent network operations."""
  2. from __future__ import annotations
  3. import errno
  4. import http.client
  5. import logging
  6. import socket
  7. import ssl
  8. import time
  9. import urllib.error
  10. import urllib.parse
  11. from datetime import timezone, UTC
  12. from email.utils import parsedate_to_datetime
  13. from typing import Callable, TypeVar
  14. _T = TypeVar("_T")
  15. _ATTEMPTS = 4
  16. _MAX_DELAY = 30.0
  17. _MAX_RECOVERY_DELAY = 300.0
  18. _HTTP_RETRY = frozenset({408, 429, 500, 502, 503, 504})
  19. _R2_HOSTS = frozenset({"hermes-assets.nousresearch.com"})
  20. _NETWORK_ERRNOS = frozenset({
  21. errno.ECONNABORTED, errno.ECONNREFUSED, errno.ECONNRESET,
  22. errno.ENETDOWN, errno.ENETRESET, errno.ENETUNREACH,
  23. errno.EHOSTUNREACH, errno.EPIPE, errno.ETIMEDOUT,
  24. })
  25. def is_transient(exc: Exception) -> bool:
  26. if isinstance(exc, urllib.error.HTTPError):
  27. return exc.code in _HTTP_RETRY
  28. if isinstance(exc, urllib.error.URLError):
  29. return isinstance(exc.reason, Exception) and is_transient(exc.reason)
  30. if isinstance(exc, (ssl.SSLEOFError, ssl.SSLZeroReturnError)):
  31. return True
  32. if isinstance(exc, ssl.SSLError):
  33. return False
  34. if isinstance(exc, socket.gaierror):
  35. return exc.errno == socket.EAI_AGAIN
  36. if isinstance(exc, (TimeoutError, ConnectionError, http.client.IncompleteRead)):
  37. return True
  38. return isinstance(exc, OSError) and exc.errno in _NETWORK_ERRNOS
  39. def edge_block(exc: BaseException) -> str | None:
  40. """Explain a 403/429 that a CDN firewall sent instead of the server, else None.
  41. A bare "HTTP 403" from a regional block reads as a broken install or a missing
  42. file (#128295). Cloudflare marks challenges with ``cf-mitigated`` and Vercel's
  43. firewall sets ``x-vercel-mitigated``. Cloudflare's own block page carries
  44. neither, but on our R2 custom domain a response without ``cf-cache-status``
  45. never reached the bucket (Cloudflare R2 troubleshooting docs); other hosts
  46. behind Cloudflare make no such promise, so that fallback is R2-only.
  47. """
  48. if not isinstance(exc, urllib.error.HTTPError) or exc.code not in (403, 429):
  49. return None
  50. headers = exc.headers or {}
  51. host = urllib.parse.urlsplit(exc.filename or "").hostname or ""
  52. if headers.get("x-vercel-mitigated"):
  53. cdn = "Vercel"
  54. elif headers.get("cf-mitigated") or (
  55. host in _R2_HOSTS and (headers.get("server") or "").lower() == "cloudflare"
  56. and not headers.get("cf-cache-status")):
  57. cdn = "Cloudflare"
  58. else:
  59. return None
  60. return (f"{host or 'the server'}'s CDN firewall ({cdn}) refused this network. Your install is fine; "
  61. "retry from another network")
  62. def _delay(exc: Exception, attempt: int) -> float:
  63. delay = float(2 ** attempt)
  64. if isinstance(exc, urllib.error.HTTPError):
  65. value = exc.headers.get("Retry-After", "") if exc.headers else ""
  66. if value:
  67. try:
  68. retry_after = int(value)
  69. except ValueError:
  70. try:
  71. when = parsedate_to_datetime(value)
  72. if when.tzinfo is None:
  73. when = when.replace(tzinfo=UTC)
  74. retry_after = when.timestamp() - time.time()
  75. except (TypeError, ValueError, OverflowError):
  76. retry_after = 0.0
  77. delay = max(delay, retry_after)
  78. return delay
  79. def retry_network(
  80. operation: Callable[[], _T], *, wait: Callable[[float], None] | None = None,
  81. progress: Callable[[], int] | None = None, fallback_available: bool = False,
  82. ) -> _T:
  83. """Retry the complete request and body read, never an install or publish.
  84. The caller resets attempt-local buffers and keeps any resumable bytes.
  85. A download supplies an interruptible wait so pause does not wait for backoff.
  86. Downloads supply ``progress``: durable forward progress renews their
  87. budget. Metadata and non-resumable requests still get four attempts.
  88. A resumable request yields to an available fallback before a long wait.
  89. """
  90. failures_since_progress = 0
  91. backoff = 0
  92. saved = progress() if progress is not None else 0
  93. while True:
  94. try:
  95. return operation()
  96. except Exception as exc:
  97. if isinstance(exc, urllib.error.HTTPError):
  98. exc.close()
  99. if not is_transient(exc):
  100. raise
  101. current = progress() if progress is not None else saved
  102. if current > saved:
  103. failures_since_progress = 0
  104. backoff = 0
  105. saved = current
  106. else:
  107. failures_since_progress += 1
  108. if failures_since_progress >= _ATTEMPTS:
  109. raise
  110. delay = _delay(exc, backoff)
  111. if progress is None:
  112. delay = min(delay, _MAX_DELAY)
  113. elif delay > (_MAX_DELAY if fallback_available else _MAX_RECOVERY_DELAY):
  114. raise
  115. backoff += 1
  116. logging.getLogger(__name__).warning(
  117. "network request failed (%s); retrying in %gs (%d/%d failures without progress)",
  118. exc, delay, failures_since_progress, _ATTEMPTS,
  119. )
  120. (wait or time.sleep)(delay)