http_diagnostics.py 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105
  1. """Describe a non-2xx answer on the gateway -> connector HTTP routes. EXPERIMENTAL.
  2. An egress proxy between the gateway and the connector answers with its OWN
  3. status codes. A proxy's 403 page looks exactly like a connector refusal unless
  4. the log says who sent it. The connector always answers a refusal with a JSON
  5. ``{"error": "<reason>"}`` body and sets none of the headers below. A proxy
  6. answers with an HTML page (some, Cloudflare/Envoy style, with JSON) and identifies
  7. itself in ``Server`` / ``Via`` / ``Proxy-Status`` (RFC 9209) / ``X-Squid-Error``;
  8. ``answered_by_proxy`` reads those so a proxy's JSON is not taken for a refusal.
  9. Only RESPONSE data is described: request headers (the bearer) are never read
  10. here, and the body excerpt is short and redacted. A hop can still echo the
  11. request's bearer into its body bare (``token <x> rejected``), where no generic
  12. pattern recognises it, so callers pass the exact credentials they sent as
  13. ``secrets`` and those are replaced literally first.
  14. """
  15. from __future__ import annotations
  16. import http.client
  17. import json
  18. import urllib.error
  19. from typing import Iterable
  20. from agent.redact import redact_sensitive_text
  21. # Response headers that name the hop that answered.
  22. _ORIGIN_HEADERS = ("Server", "Via", "Proxy-Status", "X-Squid-Error")
  23. # Headers only an intermediary sets (the connector sets none of them, not even ``Server``).
  24. _PROXY_ONLY_HEADERS = ("Via", "Proxy-Status", "X-Squid-Error")
  25. # ``Server`` values naming an egress/CDN/mesh proxy that can answer on its own with a
  26. # JSON ``{"error"}`` body. A hosting edge's own name (e.g. ``railway-edge``) is not here.
  27. _PROXY_SERVER_NAMES = ("squid", "envoy", "cloudflare", "zscaler", "bluecoat", "proxysg", "netskope", "forcepoint")
  28. BODY_EXCERPT_CHARS = 200
  29. _BODY_READ_LIMIT = 4096
  30. def read_error_body(exc: urllib.error.HTTPError) -> bytes:
  31. """The (bounded) body of an HTTP error response; ``b""`` when unreadable."""
  32. try:
  33. return exc.read(_BODY_READ_LIMIT) or b""
  34. except (OSError, ValueError, http.client.HTTPException):
  35. return b""
  36. def connector_reason(body: bytes) -> str:
  37. """The connector's own refusal reason (JSON ``{"error": ...}``), else ``""``."""
  38. try:
  39. parsed = json.loads(body.decode("utf-8"))
  40. except ValueError: # not UTF-8 / not JSON: a proxy page, not the connector
  41. return ""
  42. reason = parsed.get("error") if isinstance(parsed, dict) else None
  43. return reason.strip() if isinstance(reason, str) else ""
  44. def answered_by_proxy(exc: urllib.error.HTTPError) -> bool:
  45. """A response header names a hop other than the connector, so a JSON ``{"error"}``
  46. body is that hop's (a Cloudflare/Envoy-style 403), not a connector refusal."""
  47. headers = getattr(exc, "headers", None)
  48. if headers is None:
  49. return False
  50. if any(headers.get(name) for name in _PROXY_ONLY_HEADERS):
  51. return True
  52. server = str(headers.get("Server") or "").lower()
  53. return any(name in server for name in _PROXY_SERVER_NAMES)
  54. def redact_excerpt(text: str, secrets: Iterable[str] = ()) -> str:
  55. """``text`` with every exact credential in ``secrets`` replaced, then generically redacted."""
  56. return redact_sensitive_text(_scrub(text, secrets), force=True)
  57. def _scrub(text: str, secrets: Iterable[str]) -> str:
  58. for secret in secrets:
  59. if secret:
  60. text = text.replace(secret, "***")
  61. return text
  62. def describe_http_error(exc: urllib.error.HTTPError, body: bytes, *, secrets: Iterable[str] = ()) -> str:
  63. """``HTTP 403 (server=squid/6.10 x-squid-error=ERR_DNS_FAIL 0 body='<html>…')``.
  64. Header values are redacted whole, then cut. The body is reduced to one line, stripped of the exact credentials the caller sent
  65. (``secrets``) before it is cut, so a token straddling the cut cannot leave a prefix,
  66. then its first ``BODY_EXCERPT_CHARS`` characters are redacted.
  67. """
  68. parts = []
  69. secrets = tuple(secrets)
  70. headers = getattr(exc, "headers", None)
  71. for name in _ORIGIN_HEADERS:
  72. value = headers.get(name) if headers is not None else None
  73. if value:
  74. # A hop can echo the bearer into a header too: redact the whole value before the cut.
  75. parts.append(f"{name.lower()}={redact_excerpt(' '.join(str(value).split()), secrets)[:120]}")
  76. text = _scrub(" ".join(body.decode("utf-8", "replace").split()), secrets)
  77. if text:
  78. excerpt = text[:BODY_EXCERPT_CHARS] + ("…" if len(text) > BODY_EXCERPT_CHARS else "")
  79. parts.append(f"body={redact_excerpt(excerpt, secrets)!r}")
  80. return f"HTTP {exc.code}" + (f" ({' '.join(parts)})" if parts else "")
  81. __all__ = [
  82. "BODY_EXCERPT_CHARS", "answered_by_proxy", "connector_reason", "describe_http_error", "read_error_body",
  83. "redact_excerpt",
  84. ]