source_releases.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394
  1. """Resolve promoted source releases, never infer publication from a Git tag."""
  2. from __future__ import annotations
  3. from dataclasses import dataclass
  4. from html.parser import HTMLParser
  5. import json
  6. import logging
  7. import re
  8. import subprocess
  9. import urllib.error
  10. import urllib.request
  11. from hermes_cli.update_channel import STABLE_TAG_RE, is_canary_tag
  12. logger = logging.getLogger(__name__)
  13. _PUBLIC_BASE = "https://hermes-assets.nousresearch.com"
  14. OFFICIAL_REPOSITORY = "NousResearch/hermes-agent"
  15. _GITHUB_ORIGIN = re.compile(
  16. r"^(?:https://github\.com/|git@github\.com:|ssh://git@github\.com/)"
  17. r"([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+?)(?:\.git)?/?$", re.IGNORECASE,
  18. )
  19. _SHA = re.compile(r"[0-9a-f]{40}")
  20. def source_repository(git_cmd=None, cwd=None) -> str:
  21. """GitHub forks own their releases; other origins must mirror official tags."""
  22. if git_cmd is not None:
  23. from hermes_cli.source_check import source_git_env
  24. result = subprocess.run(
  25. [*git_cmd, "config", "--get", "remote.origin.url"], cwd=cwd,
  26. capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10,
  27. stdin=subprocess.DEVNULL, env=source_git_env(),
  28. check=False,
  29. )
  30. match = _GITHUB_ORIGIN.fullmatch(result.stdout.strip())
  31. if result.returncode == 0 and match:
  32. return match[1]
  33. return OFFICIAL_REPOSITORY
  34. @dataclass(frozen=True)
  35. class SourceTarget:
  36. """A pinned source build or an explicitly declared source-branch delivery."""
  37. requested_channel: str
  38. channel: str
  39. repository: str
  40. commit: str | None = None
  41. branch: str | None = None
  42. version: str | None = None
  43. build_id: str | None = None
  44. @property
  45. def retired(self) -> bool:
  46. return self.requested_channel != self.channel
  47. ahead: bool = False
  48. @property
  49. def label(self) -> str:
  50. if self.ahead:
  51. return (f"{self.channel} v{self.version}; this checkout is newer and waits for the next release"
  52. " (`hermes update --set-channel main` follows every commit)")
  53. return f"{self.channel} v{self.version} ({self.commit[:12]})" if self.commit else self.channel
  54. def _resolve_channel(name: str, repository: str):
  55. """The only adapter to the validated requested/terminal records and manifest.
  56. ChannelReader owns HTTPS, authority and digests. The source adapter below
  57. admits its retirement constraints before any checkout operation. No legacy
  58. GitHub fallback is allowed when a record is unavailable; the one exception
  59. is an unpublished ``main`` record, which resolves to the main branch.
  60. """
  61. from hermes_cli.release_channels import ChannelReader
  62. return ChannelReader(_PUBLIC_BASE, repository=repository).resolve(name)
  63. def resolve_source_target(channel: str, git_cmd=None, cwd=None, *, repository=None,
  64. forward_only: bool = False) -> SourceTarget:
  65. """Resolve a subscription: stable from GitHub releases, main from the git branch,
  66. preview channels (e.g. ``pm-preview``) through their R2 records.
  67. ``forward_only`` (an unchosen default subscription) pins a checkout that already
  68. contains the release to its own HEAD, so the update is a no-op instead of a downgrade.
  69. """
  70. target = _resolve_source_target(channel, git_cmd, cwd, repository=repository)
  71. if forward_only and target.commit and git_cmd is not None and cwd is not None:
  72. head = _head_containing(git_cmd, cwd, target.commit, target.repository)
  73. if head is not None and head != target.commit:
  74. from dataclasses import replace
  75. return replace(target, commit=head, ahead=True)
  76. return target
  77. def _probe(git_cmd, cwd, *args):
  78. """``git <args>`` in ``cwd``, or None when git did not answer: an unknown relation, not a claim.
  79. Before the updater's fetch the release commit is usually absent, and git 2.43 spends 7-12 s
  80. of CPU on a blobless clone failing to find it even with lazy fetch off (#137646).
  81. """
  82. from hermes_cli.source_check import source_git_env
  83. try:
  84. return subprocess.run(
  85. [*git_cmd, *args], cwd=cwd, capture_output=True, text=True, encoding="utf-8",
  86. errors="replace", timeout=10, stdin=subprocess.DEVNULL, env=source_git_env(),
  87. check=False)
  88. except (OSError, subprocess.SubprocessError) as exc:
  89. logger.debug("git %s did not answer: %s", " ".join(args), exc)
  90. return None
  91. def _head_containing(git_cmd, cwd, commit: str, repository: str) -> str | None:
  92. """HEAD's sha unless HEAD is PROVEN not to contain ``commit``, then None.
  93. An unchosen default must never move a checkout backward on a guess, so an
  94. unknown relation (shallow history, GitHub unreachable) keeps HEAD.
  95. """
  96. from hermes_cli.source_check import _github_compare, source_git_env
  97. head = subprocess.run(
  98. [*git_cmd, "rev-parse", "HEAD"], cwd=cwd, capture_output=True, text=True, encoding="utf-8",
  99. errors="replace", timeout=10, stdin=subprocess.DEVNULL, env=source_git_env(), check=False,
  100. ).stdout.strip()
  101. if not _SHA.fullmatch(head):
  102. return None
  103. # An unanswered probe falls through to the comparison API, as a shallow checkout's does.
  104. ancestry = _probe(git_cmd, cwd, "merge-base", "--is-ancestor", commit, head)
  105. if ancestry is not None and ancestry.returncode == 0:
  106. return head
  107. # A shallow boundary makes exit 1 a guess; full history makes it proof.
  108. if ancestry is not None and ancestry.returncode == 1:
  109. shallow = _probe(git_cmd, cwd, "rev-parse", "--is-shallow-repository")
  110. if shallow is not None and shallow.stdout.strip() == "false":
  111. return None
  112. status = (_github_compare(commit, head, repository) or {}).get("status")
  113. return None if status in ("behind", "diverged") else head
  114. def _resolve_source_target(channel: str, git_cmd=None, cwd=None, *, repository=None) -> SourceTarget:
  115. from hermes_cli.release_channels import validate_name
  116. validate_name(channel)
  117. repository = repository or source_repository(git_cmd, cwd)
  118. if channel == "stable":
  119. return _resolve_stable(repository, git_cmd, cwd)
  120. if channel == "main":
  121. # main IS the git branch. No R2 record is read: none was ever published, and a
  122. # network that answers it with 403 instead of 404 (regional WAFs) must not stop
  123. # an install from following the branch it is on.
  124. return SourceTarget(channel, channel, repository, branch="main")
  125. resolved = _resolve_channel(channel, repository)
  126. terminal = resolved.terminal
  127. if terminal["repository"].lower() != repository.lower():
  128. raise ValueError("Channel repository does not match this source installation")
  129. destination = validate_name(terminal["name"])
  130. if terminal["policy"] == "source-branch":
  131. if resolved.requested["state"] == "retired":
  132. raise ValueError("Source retirement requires a published destination commit")
  133. delivery = terminal["delivery"]
  134. if delivery["kind"] != "source-branch":
  135. raise ValueError("Channel has no source-branch delivery")
  136. return SourceTarget(channel, destination, repository, branch=delivery["branch"])
  137. if resolved.manifest is None:
  138. raise ValueError(f"No build published for channel {destination}")
  139. request = resolved.manifest["request"]
  140. commit = request["commit"]
  141. if not isinstance(commit, str) or not _SHA.fullmatch(commit):
  142. raise ValueError("Channel build has no exact source commit")
  143. if resolved.requested["state"] == "retired":
  144. _refuse_retirement_downgrade(request, terminal, git_cmd, cwd)
  145. return SourceTarget(channel, destination, repository, commit=commit,
  146. version=request["sourceVersion"], build_id=request["buildId"])
  147. def _describe_release_failure(failure: BaseException) -> str:
  148. """A transport failure names its cause (a rate limit is not "no release")."""
  149. if isinstance(failure, (urllib.error.URLError, TimeoutError)):
  150. from hermes_cli.github_api import describe_github_failure
  151. # The request that failed, not the configuration: a rejected token retries anonymously.
  152. return describe_github_failure(failure, authenticated=getattr(failure, "hermes_authenticated", False))
  153. return f"No published stable release could be verified: {failure}"
  154. def _resolve_stable(repository: str, git_cmd=None, cwd=None) -> SourceTarget:
  155. """Stable IS the repository's latest published GitHub release: the tag a release
  156. publishes (non-draft, non-prerelease, strict vX.Y.Z), verified against origin.
  157. No R2 record or pointer is consulted; publishing the release is the only promotion.
  158. """
  159. try:
  160. tag, commit = _resolve_release("stable", git_cmd, cwd, repository=repository, pointer=False)
  161. except (OSError, ValueError, subprocess.SubprocessError) as exc:
  162. logger.warning("Could not resolve the stable source release: %s", exc)
  163. raise ValueError(_describe_release_failure(exc)) from exc
  164. return SourceTarget("stable", "stable", repository, commit=commit, version=str(tag).removeprefix("v"))
  165. def _refuse_retirement_downgrade(request: dict, terminal: dict, git_cmd, cwd) -> None:
  166. """Qualification of preview data is not permission to roll back newer source."""
  167. from pathlib import Path
  168. import tomllib
  169. if cwd is None:
  170. return
  171. version_file = Path(cwd) / "pyproject.toml"
  172. if version_file.exists():
  173. with version_file.open("rb") as file:
  174. project = tomllib.load(file).get("project")
  175. installed_version = project.get("version") if isinstance(project, dict) else None
  176. if not isinstance(installed_version, str) or not re.fullmatch(r"\d+\.\d+\.\d+", installed_version, re.ASCII):
  177. raise ValueError("Source retirement cannot verify the installed source version")
  178. if tuple(map(int, installed_version.split("."))) > tuple(map(int, request["sourceVersion"].split("."))):
  179. raise ValueError("Source retirement would downgrade a newer source version; select the destination channel explicitly")
  180. if git_cmd is not None:
  181. from hermes_cli.source_check import source_git_env
  182. result = subprocess.run(
  183. [*git_cmd, "rev-list", "--ancestry-path", f"{request['commit']}..HEAD"], cwd=cwd,
  184. capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10,
  185. stdin=subprocess.DEVNULL, env=source_git_env(),
  186. check=False,
  187. )
  188. # The target need not exist locally before the updater's fetch. When it
  189. # does, any descendants prove that this pinned build would roll us back.
  190. if result.returncode == 0 and result.stdout.strip():
  191. raise ValueError("Source retirement would downgrade a newer source commit; select the destination channel explicitly")
  192. if terminal["head"]["sequence"] > request["sequence"]:
  193. # Shallow checkouts may lack the qualified commit, even when HEAD is
  194. # today's stable build. Read it with the protocol's full digest checks.
  195. current_manifest = _resolve_channel(terminal["name"], request["repository"]).manifest
  196. if current_manifest is None:
  197. raise ValueError("Source retirement cannot verify the current destination build")
  198. current = current_manifest["request"]
  199. installed = subprocess.run(
  200. [*git_cmd, "rev-parse", "HEAD"], cwd=cwd, check=True,
  201. capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10,
  202. stdin=subprocess.DEVNULL, env=source_git_env(),
  203. ).stdout.strip()
  204. if installed == current["commit"] and installed != request["commit"]:
  205. raise ValueError("Source retirement would downgrade the newer destination build; select the destination channel explicitly")
  206. def _read(url: str, *, missing_ok: bool = False) -> str | None:
  207. request = urllib.request.Request(url, headers={
  208. "User-Agent": "hermes-update", "Cache-Control": "no-cache",
  209. "Accept": "application/json, text/html",
  210. })
  211. try:
  212. with urllib.request.urlopen(request, timeout=30) as response:
  213. return response.read(2 * 1024 * 1024).decode("utf-8-sig")
  214. except urllib.error.HTTPError as exc:
  215. if missing_ok and exc.code == 404:
  216. return None
  217. raise
  218. class _BuildMetadata(HTMLParser):
  219. def __init__(self):
  220. super().__init__()
  221. self.tags = []
  222. def handle_starttag(self, tag, attrs):
  223. fields = dict(attrs)
  224. if tag == "meta" and fields.get("name") == "hermes-build":
  225. self.tags.append(fields.get("content"))
  226. def _valid_tag(tag, channel: str) -> bool:
  227. if not isinstance(tag, str):
  228. return False
  229. return bool(STABLE_TAG_RE.fullmatch(tag)) if channel == "stable" else (
  230. tag == tag.strip() and is_canary_tag(tag)
  231. )
  232. def _published(release, channel: str) -> bool:
  233. return (isinstance(release, dict) and release.get("draft") is False
  234. and release.get("prerelease") is (channel == "canary")
  235. and _valid_tag(release.get("tag_name"), channel))
  236. def _json(url: str):
  237. """GitHub API JSON with the updater's credential ladder (token, gh, anonymous).
  238. Anonymous api.github.com is 60 requests/hour per address, so an unauthenticated
  239. stable lookup fails for everyone behind one office, CI or container exit.
  240. """
  241. from hermes_cli.source_check import _request
  242. return json.loads(_request(url))
  243. def _published_fallback(channel: str, base: str) -> dict:
  244. if channel == "stable":
  245. release = _json(f"{base}/releases/latest")
  246. if _published(release, channel):
  247. return release
  248. else:
  249. # GitHub lists newest releases first. Bound the scan; failure must
  250. # never turn into an arbitrary Git-tag update.
  251. for page in range(1, 11):
  252. entries = _json(f"{base}/releases?per_page=100&page={page}")
  253. if not isinstance(entries, list):
  254. break
  255. for release in entries:
  256. if _published(release, channel):
  257. return release
  258. if len(entries) < 100:
  259. break
  260. raise ValueError(f"No published {channel} release")
  261. def _release_pointer(channel: str) -> tuple[str | None, str | None]:
  262. # Stable's completion job writes this before publishing the GitHub draft.
  263. # Publication is checked separately, so that interval fails closed.
  264. if channel == "stable":
  265. text = _read(f"{_PUBLIC_BASE}/releases/stable/release-candidates.json", missing_ok=True)
  266. if text is not None:
  267. data = json.loads(text)
  268. if (not isinstance(data, dict) or not _valid_tag(data.get("tag"), channel)
  269. or not isinstance(data.get("commit"), str) or not _SHA.fullmatch(data["commit"])):
  270. raise ValueError("Invalid stable release pointer")
  271. return data["tag"], data["commit"]
  272. text = _read(f"{_PUBLIC_BASE}/releases/{channel}/index.html", missing_ok=True)
  273. if text is None:
  274. return None, None
  275. page = _BuildMetadata()
  276. page.feed(text)
  277. if len(page.tags) != 1 or not _valid_tag(page.tags[0], channel):
  278. raise ValueError(f"Invalid {channel} release pointer")
  279. return page.tags[0], None
  280. def _resolve_release(channel: str, git_cmd=None, cwd=None, *, repository=None,
  281. pointer: bool = True) -> tuple[str, str]:
  282. """``resolve_source_release`` that raises the failure instead of returning ``(None, None)``."""
  283. if channel not in ("stable", "canary"):
  284. raise ValueError(f"Not a release channel: {channel}")
  285. repository = repository or source_repository(git_cmd, cwd)
  286. base = f"https://api.github.com/repos/{repository}"
  287. tag, pinned_sha = (_release_pointer(channel)
  288. if pointer and repository.lower() == OFFICIAL_REPOSITORY.lower() else (None, None))
  289. if tag is None:
  290. release = _published_fallback(channel, base)
  291. tag = release["tag_name"]
  292. else:
  293. release = _json(f"{base}/releases/tags/{tag}")
  294. if not _published(release, channel) or release["tag_name"] != tag:
  295. raise ValueError(f"{tag} is not a published {channel} release")
  296. commit = _json(f"{base}/commits/{tag}")
  297. sha = commit.get("sha") if isinstance(commit, dict) else None
  298. if not isinstance(sha, str) or not _SHA.fullmatch(sha):
  299. raise ValueError(f"No published commit for release {tag}")
  300. if git_cmd is not None:
  301. from hermes_cli.source_check import source_git_env
  302. ref = f"refs/tags/{tag}"
  303. result = subprocess.run(
  304. [*git_cmd, "ls-remote", "--tags", "origin", ref, ref + "^{}"],
  305. cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace",
  306. check=True, timeout=60, stdin=subprocess.DEVNULL,
  307. env=source_git_env(),
  308. )
  309. refs = dict((parts[1], parts[0]) for line in result.stdout.splitlines()
  310. if len(parts := line.split()) == 2)
  311. if refs.get(ref + "^{}", refs.get(ref)) != sha:
  312. raise ValueError(f"Origin tag {tag} does not match the published release commit")
  313. if pinned_sha is not None and sha != pinned_sha:
  314. raise ValueError(f"Release {tag} no longer matches its published commit")
  315. return tag, sha
  316. def resolve_source_release(channel: str, git_cmd=None, cwd=None, *, repository=None,
  317. pointer: bool = True) -> tuple[str | None, str | None]:
  318. """Read published stable/canary release metadata (not channel discovery).
  319. The source updater's ``stable`` reads it with ``pointer=False``: GitHub's latest
  320. published release only. With ``pointer`` the R2 channel pointers outrank GitHub's
  321. release listing. A malformed pointer,
  322. draft, or tag/commit mismatch is not permission to select a different build.
  323. ``git_cmd`` resolves the selected tag on origin; ZIP callers omit it and
  324. resolve the same tag through GitHub's commit endpoint.
  325. """
  326. try:
  327. return _resolve_release(channel, git_cmd, cwd, repository=repository, pointer=pointer)
  328. except (OSError, ValueError, subprocess.SubprocessError) as exc:
  329. logger.warning("Could not resolve the %s source release: %s", channel, exc)
  330. return None, None