har_to_client.py 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147
  1. #!/usr/bin/env python3
  2. """Distill a HAR file into an API summary an agent can turn into a client.
  3. Usage:
  4. python3 har_to_client.py <input.har> [--include-static] [--host SUBSTRING] [--max-body 600]
  5. Filters to XHR/fetch/JSON traffic by default, groups by (method, host, path
  6. template), and prints per-endpoint: query params, interesting request headers,
  7. request body sample, response content-type/status, and a response body sample.
  8. Numeric/UUID-ish path segments are collapsed to {id} so repeated calls group.
  9. Also prints "### Replay hints": the browser User-Agent plus whether cookies or
  10. auth/token headers were present -- send those in the derived client or you may
  11. get a 403/401.
  12. """
  13. import argparse
  14. import json
  15. import re
  16. import sys
  17. from collections import OrderedDict
  18. from urllib.parse import urlsplit
  19. BORING_HEADERS = {
  20. "accept-encoding", "accept-language", "connection", "content-length",
  21. "host", "origin", "referer", "sec-ch-ua", "sec-ch-ua-mobile",
  22. "sec-ch-ua-platform", "sec-fetch-dest", "sec-fetch-mode", "sec-fetch-site",
  23. "user-agent", "pragma", "cache-control", "priority", "te",
  24. "upgrade-insecure-requests", "cookie",
  25. }
  26. ID_SEG = re.compile(r"^(\d+|[0-9a-f]{8}-[0-9a-f-]{27,}|[0-9a-f]{16,})$", re.IGNORECASE)
  27. STATIC_EXT = re.compile(r"\.(js|css|png|jpe?g|gif|svg|webp|ico|woff2?|ttf|mp4|map)$", re.IGNORECASE)
  28. def path_template(path: str) -> str:
  29. segs = path.split("/")
  30. return "/".join("{id}" if ID_SEG.match(s) else s for s in segs)
  31. def is_api_entry(entry: dict) -> bool:
  32. req = entry["request"]
  33. resp = entry.get("response", {})
  34. rtype = (entry.get("_resourceType") or "").lower()
  35. mime = (resp.get("content", {}).get("mimeType") or "").lower()
  36. if rtype in ("xhr", "fetch"):
  37. return True
  38. if "json" in mime:
  39. return True
  40. if req["method"] not in ("GET", "HEAD") and not STATIC_EXT.search(urlsplit(req["url"]).path):
  41. return True
  42. return False
  43. def trunc(text, n: int) -> str:
  44. text = text if isinstance(text, str) else str(text)
  45. return text if len(text) <= n else text[:n] + f"... [{len(text)} chars total]"
  46. def main() -> int:
  47. ap = argparse.ArgumentParser()
  48. ap.add_argument("har")
  49. ap.add_argument("--include-static", action="store_true")
  50. ap.add_argument("--host", default=None, help="only endpoints whose host contains this")
  51. ap.add_argument("--max-body", type=int, default=600)
  52. args = ap.parse_args()
  53. with open(args.har, encoding="utf-8") as f:
  54. har = json.load(f)
  55. groups = OrderedDict()
  56. for entry in har["log"]["entries"]:
  57. req = entry["request"]
  58. url = urlsplit(req["url"])
  59. if url.scheme not in ("http", "https"):
  60. continue
  61. if args.host and args.host not in url.netloc:
  62. continue
  63. if not args.include_static:
  64. if STATIC_EXT.search(url.path) or not is_api_entry(entry):
  65. continue
  66. key = (req["method"], url.netloc, path_template(url.path))
  67. g = groups.setdefault(key, {"count": 0, "queries": set(), "headers": {},
  68. "req_body": None, "resp": None})
  69. g["count"] += 1
  70. for q in req.get("queryString", []):
  71. g["queries"].add((q["name"], trunc(q["value"], 80)))
  72. for h in req.get("headers", []):
  73. name = h["name"].lower().lstrip(":")
  74. if name in BORING_HEADERS or name in ("method", "path", "scheme", "authority"):
  75. continue
  76. g["headers"][name] = trunc(h["value"], 120)
  77. post = req.get("postData", {})
  78. if post.get("text") and g["req_body"] is None:
  79. g["req_body"] = (post.get("mimeType", ""), trunc(post["text"], args.max_body))
  80. resp = entry.get("response", {})
  81. if g["resp"] is None and resp:
  82. content = resp.get("content", {})
  83. g["resp"] = (resp.get("status"), content.get("mimeType", ""),
  84. trunc(content.get("text") or "", args.max_body))
  85. if not groups:
  86. print("No API-looking entries found. Re-run with --include-static to see everything.")
  87. return 1
  88. # Surface the browser identity so the replay client can match it (many
  89. # sites 403 a default library User-Agent).
  90. ua = None
  91. saw_cookie = saw_auth = False
  92. for entry in har["log"]["entries"]:
  93. for h in entry["request"].get("headers", []):
  94. n = h["name"].lower()
  95. if n == "user-agent" and ua is None:
  96. ua = h["value"]
  97. if n == "cookie":
  98. saw_cookie = True
  99. if n in ("authorization", "x-api-key") or "token" in n:
  100. saw_auth = True
  101. print("### Replay hints")
  102. if ua:
  103. print(f" User-Agent (send this): {ua}")
  104. if saw_cookie:
  105. print(" Cookies present -> session may be auth-gated; capture & resend the Cookie header.")
  106. if saw_auth:
  107. print(" Authorization/token header present -> extract and resend it.")
  108. for (method, host, path), g in groups.items():
  109. print(f"\n=== {method} https://{host}{path} (x{g['count']})")
  110. if g["queries"]:
  111. print(" query params:")
  112. for name, val in sorted(g["queries"]):
  113. print(f" {name} = {val}")
  114. if g["headers"]:
  115. print(" request headers (non-boring):")
  116. for name, val in sorted(g["headers"].items()):
  117. print(f" {name}: {val}")
  118. if g["req_body"]:
  119. print(f" request body ({g['req_body'][0]}):")
  120. print(f" {g['req_body'][1]}")
  121. if g["resp"]:
  122. status, mime, body = g["resp"]
  123. print(f" response: {status} {mime}")
  124. if body:
  125. print(f" {body}")
  126. print(f"\n{len(groups)} distinct endpoints.")
  127. return 0
  128. if __name__ == "__main__":
  129. sys.exit(main())