termux_libs.py 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220
  1. """Termux pool pins: the runtime-lib table and lock.json's bionic rows.
  2. ``scripts/termux/stage_runtime_libs.py`` merges their .debs into one flat
  3. ``lib/`` inside the sealed Termux payload, and ``scripts/ci/archive_inputs.py``
  4. mirrors the same bytes into R2. Both are chained to the pin table that lives
  5. beside this module.
  6. The pool is rolling: rebuilding a package DELETES the previous archive from
  7. ``pool/main/``, so a pin rots silently until a payload build downloads it and
  8. gets a 404. The library set is the payload's recursive DT_NEEDED closure, so a
  9. pin may not simply track "newest" — a repin is a repair:
  10. hermes pm update --termux --check # what the pool no longer serves
  11. hermes pm update --termux # repin exactly those rows
  12. Nothing here resolves versions. "Move to the newest" is a different job with a
  13. different blast radius (a repin can land a rebuilt library under a moved
  14. soname), so an alive pin is left alone even when the pool has moved on.
  15. """
  16. from __future__ import annotations
  17. import json
  18. import re
  19. from dataclasses import dataclass
  20. from pathlib import Path
  21. from typing import Optional
  22. from urllib.parse import urlsplit
  23. from pm.store import hash_url
  24. from pm.update import _get_text
  25. POOL = "https://packages.termux.dev/apt/termux-main/"
  26. POOL_HOST = urlsplit(POOL).hostname
  27. INDEX_URL = f"{POOL}dists/stable/main/binary-aarch64/Packages"
  28. POOL_PATH_PREFIX = "/apt/termux-main/pool/main/"
  29. TABLE_NAME = "termux_runtime_libs.json"
  30. class PinMismatch(RuntimeError):
  31. """The pool serves bytes the package index does not describe."""
  32. def table_path() -> Path:
  33. return Path(__file__).resolve().parent / TABLE_NAME
  34. def load_table(path: Optional[Path] = None) -> dict:
  35. return json.loads(Path(path or table_path()).read_text(encoding="utf-8-sig"))
  36. def save_table(table: dict, path: Optional[Path] = None) -> None:
  37. Path(path or table_path()).write_text(
  38. json.dumps(table, indent=2) + "\n", encoding="utf-8", newline="\n"
  39. )
  40. # ---------------------------------------------------------------------------
  41. # The pool's own index (pure parse + one cached GET)
  42. # ---------------------------------------------------------------------------
  43. @dataclass(frozen=True)
  44. class PoolPackage:
  45. name: str
  46. version: str
  47. url: str
  48. sha256: str
  49. _FIELD = re.compile(r"^([A-Za-z][A-Za-z0-9-]*):[ ]?(.*)$")
  50. def parse_index(text: str) -> dict[str, PoolPackage]:
  51. """Package -> the archive the pool currently serves.
  52. RFC822-ish stanzas: one field per line, indented continuation lines
  53. (long descriptions) ignored. A stanza without a filename or hash is not
  54. installable evidence and is dropped.
  55. """
  56. pool: dict[str, PoolPackage] = {}
  57. fields: dict[str, str] = {}
  58. for line in [*text.splitlines(), ""]:
  59. if not line.strip():
  60. row = _package(fields)
  61. if row is not None:
  62. pool[row.name] = row
  63. fields = {}
  64. elif not line[0].isspace():
  65. match = _FIELD.match(line)
  66. if match:
  67. fields[match.group(1)] = match.group(2).strip()
  68. return pool
  69. def _package(fields: dict[str, str]) -> Optional[PoolPackage]:
  70. name = fields.get("Package")
  71. filename = fields.get("Filename")
  72. sha256 = fields.get("SHA256")
  73. if not name or not filename or not sha256:
  74. return None
  75. return PoolPackage(name, fields.get("Version", ""), POOL + filename, sha256)
  76. def index() -> dict[str, PoolPackage]:
  77. return parse_index(_get_text(INDEX_URL))
  78. def package_of(url: str) -> Optional[str]:
  79. """The pool package a pinned URL names, or None for any other source."""
  80. parsed = urlsplit(url)
  81. if parsed.scheme != "https" or parsed.netloc != POOL_HOST:
  82. return None
  83. if not parsed.path.startswith(POOL_PATH_PREFIX):
  84. return None
  85. parts = parsed.path[len(POOL_PATH_PREFIX):].split("/")
  86. if len(parts) != 3 or not all(parts):
  87. return None
  88. return parts[1]
  89. def _filename(url: str) -> str:
  90. return url.rsplit("/", 1)[-1]
  91. # ---------------------------------------------------------------------------
  92. # Pins, and which of them the pool has retired
  93. # ---------------------------------------------------------------------------
  94. @dataclass(frozen=True)
  95. class Pin:
  96. scope: str # "library", "license", or "tool"
  97. name: str
  98. url: str
  99. version: str
  100. @dataclass(frozen=True)
  101. class Retired:
  102. pin: Pin
  103. # None: the pool no longer carries the package at all — a rename or a
  104. # drop, which needs a human, not a repin.
  105. replacement: Optional[PoolPackage]
  106. def pins(table: dict, lockfile) -> list[Pin]:
  107. """Every pinned URL the Termux pool serves, from both authorities.
  108. The table owns the payload's libraries and its license package; the
  109. lockfile owns the cross-target tools whose bionic arm is a pool .deb
  110. (python, node, uv). Other lock sources — GitHub tarballs, OCI digests —
  111. are not the pool's business.
  112. """
  113. out = [
  114. Pin("library", name, row["url"], row.get("version", ""))
  115. for name, row in (table.get("libs") or {}).items()
  116. ]
  117. licenses = table.get("licenses")
  118. if licenses:
  119. out.append(Pin("license", "termux-licenses", licenses["url"], licenses.get("version", "")))
  120. for name in lockfile.names():
  121. for target, rows in lockfile.pinned_artifacts(name).items():
  122. for row in rows if isinstance(rows, list) else [rows]:
  123. if isinstance(row, dict) and package_of(row.get("url", "")):
  124. out.append(Pin("tool", f"{name}@{target}", row["url"], lockfile.version(name) or ""))
  125. return out
  126. def retired(table: dict, lockfile, pool: dict[str, PoolPackage]) -> list[Retired]:
  127. """Pins whose archive the pool has replaced (or dropped)."""
  128. out = []
  129. for pin in pins(table, lockfile):
  130. entry = pool.get(package_of(pin.url) or "")
  131. if entry is None or _filename(entry.url) != _filename(pin.url):
  132. out.append(Retired(pin, entry))
  133. return out
  134. def repair(table: dict, lockfile, stale: list[Retired], *, verify=None) -> int:
  135. """Repin every retired row to the pool's current archive.
  136. The index's SHA256 is not taken on faith: the replacement's bytes are
  137. fetched and hashed first — the same evidence `pm lock` pins with. Rows
  138. the pool no longer carries are left exactly as they are.
  139. """
  140. verify = verify or hash_url
  141. applied = 0
  142. for entry in stale:
  143. if entry.replacement is None:
  144. continue
  145. served = verify(entry.replacement.url)
  146. if served != entry.replacement.sha256:
  147. raise PinMismatch(
  148. f"{package_of(entry.pin.url)}: the pool serves {served}, "
  149. f"the index declares {entry.replacement.sha256}"
  150. )
  151. _repoint(table, lockfile, entry)
  152. applied += 1
  153. return applied
  154. def _repoint(table: dict, lockfile, entry: Retired) -> None:
  155. pin, replacement = entry.pin, entry.replacement
  156. if pin.scope == "tool":
  157. name, target = pin.name.split("@", 1)
  158. artifacts = lockfile.pinned_artifacts(name)
  159. rows = artifacts[target]
  160. for row in rows if isinstance(rows, list) else [rows]:
  161. if package_of(row.get("url", "")) == package_of(pin.url):
  162. row["url"], row["sha256"] = replacement.url, replacement.sha256
  163. lockfile.set_pin(name, lockfile.version(name), artifacts)
  164. return
  165. row = table["libs"][pin.name] if pin.scope == "library" else table["licenses"]
  166. row["url"], row["sha256"], row["version"] = (
  167. replacement.url,
  168. replacement.sha256,
  169. replacement.version,
  170. )