lock.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317
  1. """The lockfile (versions + hashes, machine-written) and the installed-state
  2. file (what is actually on this machine).
  3. lock.json: {"schema": 1, "packages": {name: {"version": ..., "artifacts": {target: {"url": ..., "sha256": ...}}}}}
  4. facts.json: {"schema": 1, "packages": {name: {"entry": ..., "version": ..., "env": ..., "stamp": ...}}}
  5. A target's value is one {"url", "sha256"} object, or a LIST of them when the
  6. package is split across several archives that must land in one directory
  7. (llama.cpp's engine zip and its cudart zip: Windows resolves a DLL from the
  8. loading executable's own directory). Both shapes read back as a list.
  9. lock.json artifacts carry the RESOLVED url beside the hash: the lockfile is
  10. the complete machine interface (nix reads it as pure data), and the python
  11. url templates are consulted only at `pm lock --bump` time. The "any" target
  12. key covers target-independent artifacts (npm's tarball).
  13. facts.json env values hold {{store}} templates so a CI-built file adopts onto
  14. any machine by substitution.
  15. """
  16. from __future__ import annotations
  17. import json
  18. import os
  19. import re
  20. from copy import deepcopy
  21. from pathlib import Path
  22. SCHEMA = 1
  23. STORE_TOKEN = "{{store}}"
  24. def _read(path: Path, *, strict: bool = False) -> dict:
  25. try:
  26. text = path.read_text(encoding="utf-8-sig")
  27. except FileNotFoundError:
  28. return {"schema": SCHEMA, "packages": {}}
  29. except OSError:
  30. if strict:
  31. raise
  32. return {"schema": SCHEMA, "packages": {}}
  33. try:
  34. data = json.loads(text)
  35. if isinstance(data, dict) and data.get("schema") == SCHEMA and isinstance(data.get("packages"), dict):
  36. return data
  37. except ValueError:
  38. pass
  39. if strict:
  40. raise ValueError(f"cannot read recorded package state: {path}")
  41. if text.strip():
  42. # An unparsable-but-nonempty state file is evidence, not garbage:
  43. # the next _write would silently discard every installed-state
  44. # record. Keep the bytes for post-mortem.
  45. try:
  46. from pm.filesystem import long_root
  47. from pm.paths import store_root
  48. # A Store-built facts path carries the store's long spelling; compare like with like.
  49. if (long_root(path.parent) == long_root(store_root())
  50. and (store_root().parent / "manifest.json").is_file()):
  51. import logging
  52. logging.getLogger(__name__).warning("invalid shipped state file: %s", path)
  53. else:
  54. path.with_suffix(".corrupt").write_text(text, encoding="utf-8")
  55. except OSError:
  56. pass
  57. return {"schema": SCHEMA, "packages": {}}
  58. def _write(path: Path, data: dict) -> None:
  59. from pm.filesystem import durable_write_bytes
  60. durable_write_bytes(path, (json.dumps(data, indent=2, sort_keys=True) + "\n").encode("utf-8"))
  61. class StaleLockRow(RuntimeError):
  62. """A concurrent editor changed a row that this writer plans to publish."""
  63. class Lockfile:
  64. """Snapshot of lock.json with conflict-checked publication of changed pins."""
  65. def __init__(self, path: Path):
  66. self.path = Path(path).resolve()
  67. self._packages = _read(self.path)["packages"]
  68. self._base = deepcopy(self._packages)
  69. self._touched: set[str] = set()
  70. def version(self, name: str) -> str | None:
  71. return (self._packages.get(name) or {}).get("version")
  72. def artifacts(self, name: str, target: str) -> list[dict]:
  73. """Every archive this target needs, in extraction order. One
  74. artifact and a list of them are the same thing here — the single
  75. form is just the common case written short."""
  76. artifacts = (self._packages.get(name) or {}).get("artifacts") or {}
  77. found = artifacts.get(target)
  78. if found is None:
  79. found = artifacts.get("any")
  80. if found is None:
  81. return []
  82. return list(found) if isinstance(found, list) else [found]
  83. def pinned_artifacts(self, name: str) -> dict:
  84. """Return the complete pin table, including targets without discovery."""
  85. return deepcopy((self._packages.get(name) or {}).get("artifacts") or {})
  86. def names(self) -> list[str]:
  87. return sorted(self._packages)
  88. def set_pin(self, name: str, version: str, artifacts: dict[str, dict]) -> None:
  89. self._packages[name] = {"version": version, "artifacts": deepcopy(artifacts)}
  90. self._touched.add(name)
  91. def save(self) -> None:
  92. """Merge touched rows under the file's lock, or leave all rows unchanged."""
  93. from pm.filesystem import lock_fd
  94. self.path.parent.mkdir(parents=True, exist_ok=True)
  95. fd = os.open(self.path.with_name(f".{self.path.name}.lock"), os.O_CREAT | os.O_RDWR, 0o600)
  96. try:
  97. lock_fd(fd, wait=True)
  98. current = _read(self.path, strict=True)["packages"]
  99. updated = dict(current)
  100. for name in self._touched:
  101. intended = self._packages[name]
  102. if current.get(name) != self._base.get(name) and current.get(name) != intended:
  103. raise StaleLockRow(f"{name} changed in {self.path}; read the lockfile again and retry")
  104. updated[name] = intended
  105. if updated != current or not self.path.exists():
  106. _write(self.path, {"schema": SCHEMA, "packages": updated})
  107. self._packages = updated
  108. self._base = deepcopy(updated)
  109. self._touched.clear()
  110. finally:
  111. os.close(fd)
  112. def termux_docker_digest() -> str:
  113. """The pinned termux-docker image digest (pm package `termux-docker`).
  114. Single reader for every consumer -- the build/deb scripts, the builder
  115. image script, and the release workflow -- so a lock-schema change lands
  116. once.
  117. """
  118. from pm.paths import lockfile_path
  119. return Lockfile(lockfile_path()).version("termux-docker")
  120. class Facts:
  121. """The installed-state file. Written only by pm."""
  122. def __init__(self, path: Path, *, strict: bool = False):
  123. self.path = path
  124. self._packages = _read(path, strict=strict)["packages"]
  125. def reload(self) -> None:
  126. self._packages = _read(self.path, strict=True)["packages"]
  127. def get(self, name: str) -> dict | None:
  128. return self._packages.get(name)
  129. def refresh_digests(self, store_root: Path) -> int:
  130. """Publish all tool digests after packaging finishes changing their bytes."""
  131. from pm.store import tree_digest
  132. from pm.filesystem import long_root
  133. packages = _read(self.path, strict=True)["packages"]
  134. # Digests walk whole tool trees: the long spelling keeps them past MAX_PATH.
  135. root = long_root(store_root.resolve())
  136. count = 0
  137. for name, fact in packages.items():
  138. if not isinstance(fact, dict):
  139. raise ValueError(f"invalid payload fact: {name}")
  140. if "entry" not in fact and "stamp" in fact:
  141. continue
  142. entry_name = fact.get("entry")
  143. artifacts = fact.get("artifacts")
  144. if (not isinstance(entry_name, str) or entry_name in ("", ".", "..")
  145. or any(c in entry_name for c in "/\\:")
  146. or not isinstance(fact.get("version"), str) or not fact["version"]
  147. or not isinstance(fact.get("target"), str) or not fact["target"]
  148. or not isinstance(artifacts, list) or not artifacts
  149. or any(not isinstance(sha, str) or not re.fullmatch(r"[a-f0-9]{64}", sha)
  150. for sha in artifacts)):
  151. raise ValueError(f"incomplete payload tool: {name}")
  152. entry = root / entry_name
  153. if not entry.is_dir() or not entry.resolve().is_relative_to(root):
  154. raise ValueError(f"incomplete payload tool: {name}")
  155. fact["digest"] = tree_digest(entry)
  156. count += 1
  157. if not count:
  158. raise ValueError(f"payload facts carry no tool entries: {self.path}")
  159. _write(self.path, {"schema": SCHEMA, "packages": packages})
  160. self._packages = packages
  161. return count
  162. def installed(
  163. self,
  164. name: str,
  165. expected_version: str | None,
  166. store_root: Path,
  167. expected_identity=None,
  168. ) -> bool:
  169. """``expected_identity`` is (target, tuple(artifact sha256s)) — the
  170. identity the lock currently pins. When given, the fact must record
  171. that exact identity: a same-version re-pin, a different target, or
  172. a fact written before identity existed all count as NOT installed
  173. and force a reinstall. ``None`` keeps the version+path check."""
  174. fact = self._packages.get(name)
  175. if not fact or "entry" not in fact:
  176. # State facts (``record_state``: venv stamp + extras) share this
  177. # file with tool facts but own no store entry.
  178. return False
  179. if expected_version is not None and fact.get("version") != expected_version:
  180. return False
  181. if expected_identity is not None:
  182. target, shas = expected_identity
  183. if "target" not in fact or "artifacts" not in fact:
  184. # Legacy fact: pre-dates digest-bound identity. Not
  185. # vouchable — force one reinstall.
  186. return False
  187. if (fact["target"], tuple(fact["artifacts"])) != (target, tuple(shas)):
  188. return False
  189. return (store_root / fact["entry"]).exists()
  190. def env_for(self, name: str, store_root: Path) -> dict:
  191. fact = self._packages.get(name) or {}
  192. return _resolve(fact.get("env", {}), store_root)
  193. def _merge_and_write(self, name: str, fact: dict) -> None:
  194. """Read-modify-write against disk so concurrent installs of
  195. different packages never clobber each other."""
  196. on_disk = _read(self.path, strict=True)["packages"]
  197. for key, value in self._packages.items():
  198. on_disk.setdefault(key, value)
  199. self._packages = on_disk
  200. self._packages[name] = fact
  201. _write(self.path, {"schema": SCHEMA, "packages": self._packages})
  202. def record(
  203. self,
  204. name: str,
  205. version: str,
  206. entry: str,
  207. env: dict,
  208. store_root: Path,
  209. target: str | None = None,
  210. artifacts: list[str] | None = None,
  211. digest: str | None = None,
  212. ) -> None:
  213. """``target``/``artifacts`` record the identity this install came
  214. from (work item 1); ``digest`` is the realized tree digest of the
  215. published entry (work item 2). All three are additive — schema
  216. stays 1 and older facts without them still read back."""
  217. fact: dict = {
  218. "entry": entry,
  219. "version": version,
  220. "env": _templatize(env, store_root),
  221. }
  222. if target is not None:
  223. fact["target"] = target
  224. if artifacts is not None:
  225. fact["artifacts"] = list(artifacts)
  226. if digest is not None:
  227. fact["digest"] = digest
  228. self._merge_and_write(name, fact)
  229. def record_state(
  230. self, name: str, stamp: str, extras: list[str], *,
  231. environment: Path | None = None, resolved_lock: Path | None = None,
  232. ) -> None:
  233. """Commit a verified state package and its selected environment atomically."""
  234. fact = {"stamp": stamp, "extras": extras}
  235. if environment is not None:
  236. fact["environment"] = str(environment.resolve())
  237. if resolved_lock is not None:
  238. fact["resolved_lock"] = str(resolved_lock.resolve())
  239. self._merge_and_write(name, fact)
  240. def retain(self, names: set[str]) -> None:
  241. """Drop unselected package facts from an exclusively owned staged store."""
  242. packages = _read(self.path, strict=True)["packages"]
  243. retained = {name: fact for name, fact in packages.items() if name in names}
  244. if retained != packages:
  245. _write(self.path, {"schema": SCHEMA, "packages": retained})
  246. self._packages = retained
  247. def forget(self, names) -> None:
  248. """Drop the facts of packages pm no longer defines, so gc reclaims their entries."""
  249. packages = _read(self.path, strict=True)["packages"]
  250. self.retain(set(packages) - set(names))
  251. def entries_in_use(self) -> set[str]:
  252. return {f["entry"] for f in self._packages.values() if "entry" in f}
  253. def _templatize(env: dict, store_root: Path) -> dict:
  254. root = str(store_root)
  255. out = {}
  256. for key, value in env.items():
  257. if isinstance(value, list):
  258. out[key] = [str(v).replace(root, STORE_TOKEN) for v in value]
  259. else:
  260. out[key] = str(value).replace(root, STORE_TOKEN)
  261. return out
  262. def _resolve(env: dict, store_root: Path) -> dict:
  263. root = str(store_root)
  264. out = {}
  265. for key, value in env.items():
  266. if isinstance(value, list):
  267. out[key] = [str(v).replace(STORE_TOKEN, root) for v in value]
  268. else:
  269. out[key] = str(value).replace(STORE_TOKEN, root)
  270. return out