downloader.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708
  1. """Resumable, hash-verified, multi-connection downloads.
  2. Every large fetch in Hermes goes through this one downloader: pm
  3. packages (pinned sha256 from lock.json) and local models (deliberately
  4. unverified — catalog sizes may lag an upstream re-upload, so sha256 is
  5. optional per source).
  6. Partial state is the downloader's own. Files land in a managed
  7. partials area keyed by sha256(url). One process owns each key while it
  8. transfers or publishes. Complete bytes are copied to destination-local
  9. staging before atomic replacement; a failed copy leaves the old destination
  10. intact. An interrupted download keeps its durable ranges bound to the remote
  11. length, strong ETag and optional pinned hash. Unidentified data restarts
  12. instead of mixing bytes from different upstream versions.
  13. The progress callback reports the whole job AND the per-dest bitmap:
  14. ``progress(overall_done, overall_total, ranges)`` where ``ranges`` maps
  15. the destination path to half-open [start, end) runs. ``done_bytes`` is the sum;
  16. ``ranges`` is the shape — one datum, two resolutions.
  17. """
  18. from __future__ import annotations
  19. import hashlib
  20. import http.client
  21. import json
  22. import os
  23. import re
  24. import shutil
  25. import logging
  26. import socket
  27. import ssl
  28. import threading
  29. import time
  30. import urllib.error
  31. import urllib.request
  32. from contextlib import suppress
  33. from dataclasses import dataclass
  34. from pathlib import Path
  35. from typing import Callable, Optional, Sequence
  36. from pm.network import edge_block, is_transient, retry_network
  37. # GitHub's release-asset CDN (release-assets.githubusercontent.com, which
  38. # TUR's pool 302s to) 403s unknown tool UAs from CI runner IP ranges --
  39. # their docs require a real User-Agent. A browser-shaped one is the
  40. # least-privileged string every asset CDN accepts.
  41. _UA = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) hermes-pm/1.0", "Accept-Encoding": "identity"}
  42. _LOOPBACK = ("http://127.0.0.1:", "http://localhost:", "http://[::1]:")
  43. _CHUNK = 1 << 20 # read/write block, also the minimum range size
  44. _CHECKPOINT_SECONDS = 5.0
  45. class _HttpsRedirectHandler(urllib.request.HTTPRedirectHandler):
  46. """The https-only gate must hold across redirects, not just the first
  47. hop — an https URL could otherwise bounce to http mid-download and
  48. carry the payload in the clear."""
  49. def redirect_request(self, req, fp, code, msg, headers, newurl):
  50. # Plain-http loopback exists for test servers, and only a download that
  51. # started on loopback may stay there: an https origin bouncing to a local
  52. # listener would hand an unpinned model file to whatever is bound there.
  53. loopback = req.full_url.startswith(_LOOPBACK) and newurl.startswith(_LOOPBACK)
  54. if not (newurl.startswith("https://") or loopback):
  55. raise DownloadError(f"refusing redirect to non-https url: {newurl}")
  56. # urllib preserves our User-Agent and range headers. Do not re-add
  57. # arbitrary headers after its redirect policy has processed them.
  58. return super().redirect_request(req, fp, code, msg, headers, newurl)
  59. def _https_context() -> ssl.SSLContext:
  60. # Trust a CA the store holds even when its own issuer is absent: Windows can
  61. # cache Let's Encrypt "Root YR" without ISRG Root X1. Python 3.13+ does this
  62. # by default; the older interpreter that starts a cold update does not.
  63. context = ssl.create_default_context()
  64. context.verify_flags |= ssl.VERIFY_X509_PARTIAL_CHAIN
  65. return context
  66. _OPENER = urllib.request.build_opener(
  67. _HttpsRedirectHandler(), urllib.request.HTTPSHandler(context=_https_context())
  68. )
  69. class DownloadError(RuntimeError):
  70. """Base class for downloader failures."""
  71. def _validate_range(response, start: int, end: int, total: int | None = None) -> int:
  72. """A range body is useful only for the exact requested interval."""
  73. match = re.fullmatch(r"bytes (\d+)-(\d+)/(\d+)", response.headers.get("Content-Range", ""))
  74. if response.status != 206 or match is None:
  75. raise _RangeError("server did not honor the requested byte range")
  76. first, last, size = map(int, match.groups())
  77. if (first, last) != (start, end - 1) or size < end or (total is not None and size != total):
  78. raise _RangeError("server returned a different byte range or representation size")
  79. length = response.headers.get("Content-Length")
  80. if length is not None and (not length.isdecimal() or int(length) != end - start):
  81. raise _RangeError("range Content-Length does not match its bounds")
  82. if response.headers.get("Content-Encoding", "identity").lower() != "identity":
  83. raise _RangeError("encoded response cannot be written into an identity byte range")
  84. return size
  85. class _RangeError(DownloadError):
  86. """Range or representation identity changed; partial bytes cannot be reused."""
  87. @dataclass(frozen=True)
  88. class _Remote:
  89. total: int
  90. ranged: bool
  91. etag: str = ""
  92. def _strong_etag(response) -> str:
  93. value = response.headers.get("ETag", "")
  94. return value if value.startswith('"') and value.endswith('"') else ""
  95. class HashError(DownloadError):
  96. """The downloaded bytes did not match the pinned sha256."""
  97. class DownloadPaused(DownloadError):
  98. """pause() was called mid-download; partials were left intact."""
  99. class DownloadTransportError(DownloadError):
  100. """An exhausted network request, with its original status and URL."""
  101. def __init__(self, url: str, cause: Exception):
  102. self.url = url
  103. self.status = cause.code if isinstance(cause, urllib.error.HTTPError) else None
  104. # A host that does not resolve is not worth a retry, but the pinned
  105. # mirror is a different host and the hash still guards the bytes.
  106. unresolved = isinstance(getattr(cause, "reason", cause), socket.gaierror)
  107. # Any 5xx is the upstream host failing, not its bytes changing: a CDN's
  108. # origin-down codes (520-527) are what a dead third-party host returns.
  109. self.fallback_allowed = (
  110. self.status in (401, 403, 404, 410) or (self.status or 0) >= 500
  111. or unresolved or is_transient(cause)
  112. )
  113. refused = edge_block(cause) or ("the host refused access" if self.status in (401, 403) else None)
  114. reason = f"{cause}; {refused}" if refused else str(cause)
  115. super().__init__(f"download failed from {url}: {reason}")
  116. @dataclass(frozen=True)
  117. class Source:
  118. url: str
  119. dest: Path
  120. sha256: str = "" # "" = no integrity check (model catalog policy)
  121. fallbacks: tuple[str, ...] = ()
  122. _Ranges = list[tuple[int, int]]
  123. ProgressFn = Callable[[int, int, dict[str, _Ranges]], None]
  124. def _coalesce(ranges: _Ranges) -> _Ranges:
  125. """Merge half-open [start, end) ranges into sorted, disjoint runs."""
  126. runs = sorted((a, b) for a, b in ranges if b > a)
  127. if not runs:
  128. return []
  129. out: _Ranges = []
  130. a, b = runs[0]
  131. for x, y in runs[1:]:
  132. if x <= b:
  133. b = max(b, y)
  134. else:
  135. out.append((a, b))
  136. a, b = x, y
  137. out.append((a, b))
  138. return out
  139. def _missing(total: int, covered: _Ranges) -> _Ranges:
  140. """The gaps in [0, total) not covered by the coalesced bitmap."""
  141. missing: _Ranges = []
  142. cursor = 0
  143. for a, b in _coalesce(covered):
  144. if a > cursor:
  145. missing.append((cursor, a))
  146. cursor = max(cursor, b)
  147. if cursor < total:
  148. missing.append((cursor, total))
  149. return missing
  150. def _sha256_file(path: Path) -> str:
  151. digest = hashlib.sha256()
  152. with open(path, "rb") as f:
  153. for block in iter(lambda: f.read(_CHUNK), b""):
  154. digest.update(block)
  155. return digest.hexdigest()
  156. # How long a finished download may wait for another process to let go of it before the
  157. # publication is reported as failed.
  158. _RELEASE_WAIT_SECONDS = 60.0
  159. def replace_when_released(tmp: Path, dest: Path, *, timeout: float = _RELEASE_WAIT_SECONDS) -> None:
  160. """Rename a finished download into place, waiting out a transient hold on the file.
  161. On Windows a multi-gigabyte file whose last write handle just closed is often still open
  162. to an antivirus or indexing scan, and renaming it fails with a permission error until the
  163. scan lets go — which for a 22 GB model can take many seconds. ``os.replace`` is retried
  164. through that window; it never falls back to copying (``shutil.move`` does, which duplicates
  165. the whole file and then reports the leftover's failed delete as the download's failure).
  166. A hold that outlasts the window raises a plain-language error.
  167. """
  168. deadline = time.monotonic() + timeout
  169. delay = 0.1
  170. while True:
  171. try:
  172. os.replace(tmp, dest)
  173. return
  174. except PermissionError as exc:
  175. if time.monotonic() >= deadline:
  176. raise RuntimeError(
  177. f"The download finished, but another program (usually an antivirus scan) kept "
  178. f"{tmp.name} open and it could not be renamed into place. Please try again.") from exc
  179. time.sleep(delay)
  180. delay = min(delay * 2, 2.0)
  181. def _existing_dest_ok(source: Source) -> bool:
  182. """Pinned destinations are rehashed; unpinned model files are accepted as-is.
  183. Catalog policy does not supply their expected hash or stable length.
  184. Only this downloader's publication path guarantees complete new files.
  185. """
  186. if not source.dest.exists():
  187. return False
  188. if not source.dest.is_file():
  189. raise IsADirectoryError(f"download destination is not a file: {source.dest}")
  190. if not source.sha256:
  191. return True
  192. try:
  193. return _sha256_file(source.dest) == source.sha256
  194. except OSError:
  195. return False
  196. class Download:
  197. """One resumable download job: a plan of sources, run in parallel.
  198. Per source: probe Range support, preallocate a flat .part in the
  199. managed partials area, split into at most ``connections`` byte
  200. ranges, and stream each with a ``Range:`` header from a worker
  201. thread. A lock-protected bitmap records which ranges are actually
  202. durable; on resume only the missing ranges are re-fetched.
  203. ``Source.dest`` is atomically replaced only after successful transfer.
  204. ``pause()`` stops between chunks or while waiting for a partial's owner;
  205. ``run()`` raises :class:`DownloadPaused` and preserves resumable bytes.
  206. """
  207. CONNECTIONS = 8
  208. def __init__(
  209. self,
  210. sources: Sequence[Source],
  211. *,
  212. resume: bool = True,
  213. connections: int = CONNECTIONS,
  214. partials_dir: Optional[Path] = None,
  215. pause_event: Optional[threading.Event] = None,
  216. ):
  217. self.sources = [Source(s.url, Path(s.dest), s.sha256, tuple(s.fallbacks)) for s in sources]
  218. self.resume = resume
  219. self.connections = max(1, int(connections))
  220. if partials_dir:
  221. self.partials_dir = Path(partials_dir)
  222. else:
  223. from pm import paths
  224. self.partials_dir = paths.partials_root()
  225. self._owns_pause_event = pause_event is None
  226. self._paused = pause_event if pause_event is not None else threading.Event()
  227. self._durable_done = 0
  228. def pause(self) -> None:
  229. """Request a stop between chunks; run() raises DownloadPaused."""
  230. self._paused.set()
  231. def run(self, progress: Optional[ProgressFn] = None) -> list[Path]:
  232. """Fetch every source; return the moved destination paths."""
  233. if self._owns_pause_event:
  234. self._paused.clear()
  235. self._check_pause()
  236. self.partials_dir.mkdir(parents=True, exist_ok=True)
  237. for source in self.sources:
  238. if source.fallbacks and not re.fullmatch(r"[a-f0-9]{64}", source.sha256):
  239. raise ValueError("mirror fallback requires a full lowercase SHA256")
  240. for url in (source.url, *source.fallbacks):
  241. if not (url.startswith("https://") or url.startswith(_LOOPBACK)):
  242. raise ValueError(f"refusing non-https url: {url}")
  243. # Probe the whole plan before reporting a denominator. A missing
  244. # Content-Length keeps the bar indeterminate until that file ends.
  245. from pm.download_state import partial_lock
  246. totals: dict[str, int] = {}
  247. unknown: set[str] = set()
  248. coverage: dict[str, _Ranges] = {}
  249. selected: list[Source] = []
  250. failures: dict[str, list[DownloadTransportError]] = {}
  251. for source in self.sources:
  252. self._check_pause()
  253. key = str(source.dest)
  254. failures[key] = []
  255. if _existing_dest_ok(source):
  256. remote = _Remote(source.dest.stat().st_size, False)
  257. covered = [(0, remote.total)]
  258. else:
  259. source, remote = self._try_sources(source, lambda candidate: self._probe(candidate.url), failures[key])
  260. covered = []
  261. with partial_lock(self.partials_dir, self._key(source.url), wait=False) as acquired:
  262. if acquired and remote.ranged:
  263. covered = self._partial_ranges(source, remote)
  264. if not remote.total:
  265. unknown.add(key)
  266. selected.append(source)
  267. totals[key] = remote.total
  268. coverage[key] = covered
  269. def report(key: str, written: _Ranges, total: int, *, complete: bool = False) -> None:
  270. totals[key] = total
  271. if total or complete:
  272. unknown.discard(key)
  273. else:
  274. unknown.add(key)
  275. # The last key identifies the source reporting this tick.
  276. coverage.pop(key, None)
  277. coverage[key] = list(written)
  278. if progress is not None:
  279. done = sum(b - a for rows in coverage.values() for a, b in rows)
  280. progress(done, 0 if unknown else sum(totals.values()), dict(coverage))
  281. for source in selected:
  282. key = str(source.dest)
  283. def tick(written: _Ranges, total: int) -> None:
  284. report(key, written, total)
  285. _, size = self._try_sources(source, lambda candidate: self._transfer(candidate, tick), failures[key])
  286. report(key, [(0, size)], size, complete=True)
  287. return [source.dest for source in self.sources]
  288. def _try_sources(self, source: Source, operation, failures: list[DownloadTransportError]):
  289. urls = tuple(dict.fromkeys((source.url, *source.fallbacks)))
  290. for index, url in enumerate(urls):
  291. self._check_pause()
  292. candidate = Source(url, source.dest, source.sha256, urls[index + 1:])
  293. try:
  294. return candidate, operation(candidate)
  295. except DownloadTransportError as exc:
  296. failures.append(exc)
  297. if not exc.fallback_allowed or index == len(urls) - 1:
  298. if len(failures) == 1:
  299. raise
  300. raise DownloadError("\n".join(str(error) for error in failures)) from exc
  301. logging.getLogger(__name__).debug("%s; trying pinned mirror %s", exc, urls[index + 1])
  302. def _transfer(self, source: Source, tick) -> int:
  303. from pm.download_state import partial_lock
  304. with partial_lock(self.partials_dir, self._key(source.url), cancelled=self._paused.is_set) as acquired:
  305. if not acquired:
  306. raise DownloadPaused(source.url)
  307. self._check_pause()
  308. if _existing_dest_ok(source):
  309. return source.dest.stat().st_size
  310. # Recheck identity after waiting for another process's partial.
  311. remote = self._probe(source.url)
  312. covered = self._partial_ranges(source, remote) if remote.ranged else []
  313. self._durable_done = sum(end - start for start, end in covered)
  314. tick(covered, remote.total)
  315. resumable = bool(remote.total and remote.ranged and (source.sha256 or remote.etag))
  316. def report(written: _Ranges) -> None:
  317. tick(written, remote.total)
  318. def fetch() -> int:
  319. self._check_pause()
  320. if resumable:
  321. return self._fetch_ranged(source, remote, report)
  322. return self._fetch_single(source, remote, report)
  323. configured_connections = self.connections
  324. try:
  325. size = retry_network(fetch, wait=self._wait_retry,
  326. progress=(lambda: self._durable_done) if self.resume and resumable else None,
  327. fallback_available=bool(source.fallbacks))
  328. except (OSError, http.client.HTTPException) as exc:
  329. if isinstance(exc, urllib.error.URLError) or is_transient(exc):
  330. raise DownloadTransportError(source.url, exc) from exc
  331. raise
  332. finally:
  333. self.connections = configured_connections
  334. partial_key = self._key(source.url)
  335. self._finalize(source, self.partials_dir / f"{partial_key}.part",
  336. self.partials_dir / f"{partial_key}.ranges")
  337. return size
  338. # ── internals ─────────────────────────────────────────────
  339. def _check_pause(self) -> None:
  340. if self._paused.is_set():
  341. raise DownloadPaused("download paused")
  342. def _wait_retry(self, delay: float) -> None:
  343. if self._paused.wait(delay):
  344. raise DownloadPaused("download paused during retry backoff")
  345. def _probe(self, url: str) -> _Remote:
  346. def request():
  347. self._check_pause()
  348. req = urllib.request.Request(url, headers={**_UA, "Range": "bytes=0-0"})
  349. with _OPENER.open(req, timeout=60) as response:
  350. etag = _strong_etag(response)
  351. if response.status == 206:
  352. total = _validate_range(response, 0, 1)
  353. body = response.read(2)
  354. if not body:
  355. raise http.client.IncompleteRead(b"", 1)
  356. if len(body) != 1:
  357. raise _RangeError("range probe returned the wrong byte count")
  358. return _Remote(total, True, etag)
  359. if response.status != 200:
  360. raise DownloadError(f"unexpected download probe status: {response.status}")
  361. return _Remote(int(response.headers.get("Content-Length") or 0), False, etag)
  362. try:
  363. return retry_network(request, wait=self._wait_retry)
  364. except (OSError, http.client.HTTPException) as exc:
  365. raise DownloadTransportError(url, exc) from exc
  366. except DownloadError:
  367. raise
  368. except ValueError:
  369. return _Remote(0, False)
  370. def _key(self, url: str) -> str:
  371. return hashlib.sha256(url.encode("utf-8")).hexdigest()
  372. def _load_sidecar(self, side: Path, part: Path, remote: _Remote, sha256: str) -> _Ranges:
  373. if not self.resume or not (sha256 or remote.etag):
  374. return []
  375. try:
  376. data = json.loads(side.read_text(encoding="utf-8"))
  377. if (data["total"], data["etag"], data["sha256"]) != (remote.total, remote.etag, sha256):
  378. return []
  379. size = part.stat().st_size
  380. ranges = data["ranges"]
  381. if not isinstance(ranges, list) or any(
  382. not isinstance(row, list) or len(row) != 2
  383. or any(type(value) is not int for value in row)
  384. or not 0 <= row[0] < row[1] <= min(size, remote.total)
  385. for row in ranges
  386. ):
  387. return []
  388. return _coalesce([tuple(row) for row in ranges])
  389. except (OSError, ValueError, KeyError, TypeError):
  390. return []
  391. def _partial_ranges(self, source: Source, remote: _Remote) -> _Ranges:
  392. key = self._key(source.url)
  393. return self._load_sidecar(self.partials_dir / f"{key}.ranges",
  394. self.partials_dir / f"{key}.part", remote, source.sha256)
  395. @staticmethod
  396. def _write_sidecar(side: Path, part: Path, covered: _Ranges, remote: _Remote, sha256: str) -> None:
  397. from pm.filesystem import durable_write_bytes, retry_held
  398. # Workers flush before announcing coverage. Snapshot those ranges,
  399. # fsync the data, then atomically publish only that durable snapshot.
  400. with part.open("r+b") as stream:
  401. os.fsync(stream.fileno())
  402. record = {"total": remote.total, "etag": remote.etag, "sha256": sha256, "ranges": covered}
  403. retry_held(lambda: durable_write_bytes(side, json.dumps(record).encode("utf-8")))
  404. def _fetch_ranged(self, source: Source, remote: _Remote, tick) -> int:
  405. key = self._key(source.url)
  406. part = self.partials_dir / f"{key}.part"
  407. side = self.partials_dir / f"{key}.ranges"
  408. covered = self._load_sidecar(side, part, remote, source.sha256)
  409. with part.open("r+b" if covered else "w+b") as stream:
  410. stream.truncate(remote.total)
  411. while True:
  412. gap_count = len(_missing(remote.total, covered)) if covered else (remote.total + _CHUNK - 1) // _CHUNK
  413. connections = max(1, min(self.connections, gap_count))
  414. covered, errors = self._ranged_attempt(source, remote, part, covered, connections, tick)
  415. protocol_error = next((error for error in errors if isinstance(error, _RangeError)), None)
  416. if protocol_error is not None:
  417. part.unlink(missing_ok=True)
  418. side.unlink(missing_ok=True)
  419. raise protocol_error
  420. self._write_sidecar(side, part, covered, remote, source.sha256)
  421. self._durable_done = sum(end - start for start, end in covered)
  422. if self._paused.is_set():
  423. raise DownloadPaused(source.url)
  424. if errors:
  425. if self.connections > 1 and all(
  426. isinstance(error, urllib.error.HTTPError) and error.code in (403, 404)
  427. for error in errors
  428. ):
  429. logging.getLogger(__name__).warning(
  430. "parallel range fetch refused for %s; retrying with one connection", source.url,
  431. )
  432. self.connections = 1
  433. continue
  434. raise next((error for error in errors if not is_transient(error)), errors[0])
  435. written = self._durable_done
  436. if written != remote.total:
  437. raise http.client.IncompleteRead(b"", remote.total - written)
  438. return written
  439. def _read_range(self, source: Source, remote: _Remote, part: Path,
  440. start: int, end: int, stop: threading.Event, record) -> None:
  441. if self._paused.is_set() or stop.is_set():
  442. return
  443. headers = {**_UA, "Range": f"bytes={start}-{end - 1}"}
  444. if remote.etag:
  445. headers["If-Range"] = remote.etag
  446. request = urllib.request.Request(source.url, headers=headers)
  447. with _OPENER.open(request, timeout=120) as response, part.open("r+b") as stream:
  448. _validate_range(response, start, end, remote.total)
  449. if remote.etag and _strong_etag(response) != remote.etag:
  450. raise _RangeError("remote representation changed during download")
  451. stream.seek(start)
  452. position = start
  453. while position < end:
  454. if self._paused.is_set() or stop.is_set():
  455. return
  456. chunk = response.read(min(_CHUNK, end - position))
  457. if not chunk:
  458. raise http.client.IncompleteRead(b"", end - position)
  459. stream.write(chunk)
  460. stream.flush()
  461. position += len(chunk)
  462. # A complete checkpoint must not bypass the body-bounds check
  463. # if the process dies between the last write and EOF validation.
  464. if position == end and response.read(1):
  465. raise _RangeError("range body exceeds its declared bounds")
  466. record(start, position)
  467. def _ranged_attempt(self, source: Source, remote: _Remote, part: Path,
  468. covered: _Ranges, connections: int, tick) -> tuple[_Ranges, list[Exception]]:
  469. from concurrent.futures import ThreadPoolExecutor
  470. ranges = _missing(remote.total, covered)
  471. if not covered:
  472. count = max(1, min(connections, (remote.total + _CHUNK - 1) // _CHUNK))
  473. ranges = [(index * remote.total // count, (index + 1) * remote.total // count)
  474. for index in range(count)]
  475. lock = threading.Lock()
  476. errors: list[Exception] = []
  477. stop = threading.Event()
  478. updated = threading.Event()
  479. pending = len(ranges)
  480. covered = list(covered)
  481. reported = list(covered)
  482. checkpointed = list(covered)
  483. next_checkpoint = time.monotonic() + _CHECKPOINT_SECONDS
  484. def record(start: int, position: int) -> None:
  485. with lock:
  486. covered[:] = _coalesce(covered + [(start, position)])
  487. updated.set()
  488. def worker(start: int, end: int) -> None:
  489. nonlocal pending
  490. try:
  491. self._read_range(source, remote, part, start, end, stop, record)
  492. except Exception as exc:
  493. if isinstance(exc, urllib.error.HTTPError):
  494. exc.close()
  495. with lock:
  496. errors.append(exc)
  497. stop.set()
  498. finally:
  499. with lock:
  500. pending -= 1
  501. updated.set()
  502. with ThreadPoolExecutor(max_workers=connections, thread_name_prefix="hermes-download") as pool:
  503. futures = [pool.submit(worker, start, end) for start, end in ranges]
  504. observer_failed = False
  505. finished = not ranges
  506. while not finished:
  507. updated.wait(max(0.0, next_checkpoint - time.monotonic()))
  508. with lock:
  509. snapshot = list(covered)
  510. finished = pending == 0
  511. updated.clear()
  512. now = time.monotonic()
  513. if now >= next_checkpoint:
  514. if snapshot != checkpointed:
  515. try:
  516. self._write_sidecar(part.with_suffix(".ranges"), part, snapshot, remote, source.sha256)
  517. except OSError as exc:
  518. logging.getLogger(__name__).warning(
  519. "could not save download progress for %s; continuing: %s", source.dest.name, exc,
  520. )
  521. except BaseException:
  522. stop.set()
  523. raise
  524. else:
  525. self._durable_done = sum(end - start for start, end in snapshot)
  526. checkpointed = snapshot
  527. next_checkpoint = time.monotonic() + _CHECKPOINT_SECONDS
  528. # Only the coordinator observes progress. Slow UI/worker IPC
  529. # coalesces intermediate snapshots, never holds up range writes.
  530. if not observer_failed and snapshot != reported:
  531. try:
  532. tick(snapshot)
  533. except Exception as exc:
  534. with lock:
  535. errors.append(exc)
  536. observer_failed = True
  537. stop.set()
  538. except BaseException:
  539. stop.set()
  540. raise
  541. reported = snapshot
  542. for future in futures:
  543. future.result()
  544. return covered, errors
  545. def _fetch_single(self, source: Source, remote: _Remote, tick) -> int:
  546. # A stream with no strong validator cannot safely reuse earlier bytes.
  547. key = self._key(source.url)
  548. part = self.partials_dir / f"{key}.part"
  549. side = self.partials_dir / f"{key}.ranges"
  550. covered: _Ranges = []
  551. request = urllib.request.Request(source.url, headers=_UA)
  552. try:
  553. with _OPENER.open(request, timeout=120) as response, part.open("wb") as stream:
  554. if response.status != 200:
  555. raise DownloadError(f"unexpected download status: {response.status}")
  556. if remote.etag and _strong_etag(response) != remote.etag:
  557. raise _RangeError("remote representation changed during download")
  558. if response.headers.get("Content-Encoding", "identity").lower() != "identity":
  559. raise DownloadError("encoded response cannot be used as an identity download")
  560. declared = int(response.headers.get("Content-Length") or 0)
  561. position = 0
  562. while True:
  563. if self._paused.is_set():
  564. raise DownloadPaused(source.url)
  565. chunk = response.read(_CHUNK)
  566. if not chunk:
  567. break
  568. stream.write(chunk)
  569. position += len(chunk)
  570. covered = [(0, position)]
  571. tick(list(covered))
  572. if self._paused.is_set():
  573. raise DownloadPaused(source.url)
  574. if position < (declared or remote.total):
  575. raise http.client.IncompleteRead(b"", (declared or remote.total) - position)
  576. if (declared and position != declared) or (remote.total and position != remote.total):
  577. raise DownloadError(f"download incomplete ({position} bytes, expected {declared or remote.total})")
  578. except BaseException:
  579. if part.exists():
  580. self._write_sidecar(side, part, covered, remote, source.sha256)
  581. raise
  582. self._write_sidecar(side, part, covered, remote, source.sha256)
  583. return position
  584. def _finalize(self, source: Source, part: Path, side: Path) -> None:
  585. self._check_pause()
  586. if source.sha256:
  587. actual = _sha256_file(part)
  588. if actual != source.sha256:
  589. # Best effort: a leftover that cannot be removed must not mask the hash error.
  590. with suppress(OSError):
  591. part.unlink(missing_ok=True)
  592. side.unlink(missing_ok=True)
  593. raise HashError(
  594. f"sha256 mismatch for {source.url}: pinned "
  595. f"{source.sha256}, got {actual}")
  596. import tempfile
  597. self._check_pause()
  598. source.dest.parent.mkdir(parents=True, exist_ok=True)
  599. fd, name = tempfile.mkstemp(prefix=f".{source.dest.name}-", suffix=".download", dir=source.dest.parent)
  600. staged = Path(name)
  601. try:
  602. with os.fdopen(fd, "wb") as target, part.open("rb") as incoming:
  603. shutil.copyfileobj(incoming, target, _CHUNK)
  604. target.flush()
  605. os.fsync(target.fileno())
  606. if staged.stat().st_size != part.stat().st_size:
  607. raise DownloadError("destination copy did not preserve the complete download")
  608. self._check_pause()
  609. replace_when_released(staged, source.dest)
  610. finally:
  611. # Best effort: a leftover that cannot be removed must not hide the error that left it.
  612. with suppress(OSError):
  613. staged.unlink(missing_ok=True)
  614. # The published file is what the caller asked for; a partial the OS still holds must not
  615. # turn a completed download into a failure.
  616. with suppress(OSError):
  617. part.unlink(missing_ok=True)
  618. side.unlink(missing_ok=True)