bitwarden.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400
  1. """Bitwarden Secrets Manager (`bws` CLI) integration.
  2. Pulls API keys from BSM at startup so they need not live in ``~/.hermes/.env``.
  3. ``bws`` is auto-installed into the PM tool store (one pinned version,
  4. SHA-256-verified against the published checksum). The one bootstrap secret is
  5. the access token in ``.env``; every other key can live in BSM. One
  6. ``bws secret list <project_id>`` call per fetch, cached in-process and on disk
  7. for ``cache_ttl_seconds``. Failures NEVER block startup. Subprocess-driven on
  8. purpose: one cross-platform binary beats the ``bitwarden-sdk-secrets`` Rust wheel.
  9. """
  10. from __future__ import annotations
  11. import base64
  12. import json
  13. import logging
  14. import os
  15. import re
  16. import shutil
  17. import time
  18. from pathlib import Path
  19. from typing import Dict, List, Optional, Tuple
  20. from agent.secret_sources._cache import (
  21. CachedFetch as _CachedFetch, SecretCache, atomic_write_json, entry_from_payload,
  22. current_cache_path, fingerprint as _token_fingerprint, remove_cache_file, resolve_cache_home,
  23. )
  24. from agent.secret_sources.base import (
  25. ErrorKind, FetchResult, SecretSource, classify_cli_error, coerce_float,
  26. is_valid_env_name as _is_valid_env_name, get_source_environment, run_cli, source_child_env,
  27. )
  28. logger = logging.getLogger(__name__)
  29. _BWS_RUN_TIMEOUT = 30
  30. # <hermes_home>/bws_cache.json holds only secret VALUES (never the access
  31. # token); kept out of .env so users editing .env don't commit BSM-sourced secrets.
  32. _CacheKey = tuple[str, str, str] # (access_token_fingerprint, project_id, server_url)
  33. _DISK_CACHE_BASENAME = "bws_cache.json"
  34. _ENCRYPTED_CACHE_BASENAME = "bws_cache.enc.json"
  35. _ENCRYPTED_CACHE_VERSION = 1
  36. _ENCRYPTED_CACHE_INFO = b"hermes-bws-encrypted-cache-v1"
  37. def _cache_key_str(cache_key: _CacheKey) -> str:
  38. return "|".join(cache_key)
  39. _STORE: SecretCache[_CacheKey] = SecretCache(_DISK_CACHE_BASENAME, key_serializer=_cache_key_str)
  40. # Test seams: L1 dict, L2 DiskCache, and its path.
  41. _CACHE = _STORE.memory
  42. _DISK_CACHE = _STORE.disk
  43. _disk_cache_path = _DISK_CACHE.path
  44. def _encrypted_disk_cache_path(home_path: Optional[Path] = None) -> Path:
  45. return resolve_cache_home(home_path) / _ENCRYPTED_CACHE_BASENAME
  46. def _current_encrypted_disk_cache_path(home_path: Optional[Path] = None) -> Path:
  47. return current_cache_path(resolve_cache_home(home_path), _ENCRYPTED_CACHE_BASENAME)
  48. # First matching rule wins. The BSM identity endpoint rejects a revoked /
  49. # expired machine-account token with an OAuth-style
  50. # `[400 Bad Request] {"error":"invalid_client"}`, hence those AUTH tokens.
  51. _BWS_ERROR_RULES = (
  52. (ErrorKind.TIMEOUT, ("timed out",)),
  53. (ErrorKind.BINARY_MISSING, ("binary not available", "failed to invoke")),
  54. (ErrorKind.AUTH_FAILED, ("unauthorized", "invalid token", "access token", "401", "403",
  55. "invalid_client", "invalid_grant", "400 bad request")),
  56. (ErrorKind.NETWORK, ("network", "connection", "resolve", "download", "dns")),
  57. )
  58. def _classify_bws_error(message: str) -> ErrorKind:
  59. return classify_cli_error(message, _BWS_ERROR_RULES)
  60. # --- Binary discovery + lazy install ----------------------------------------
  61. def find_bws(*, install_if_missing: bool = False) -> Optional[Path]:
  62. """External tools do not require PM platform support; acquire only on a miss."""
  63. if system := shutil.which("bws"):
  64. return Path(system)
  65. import pm
  66. selected = pm.installed_package("bws")
  67. if selected is not None:
  68. return selected.binary
  69. if install_if_missing:
  70. try:
  71. pm.ensure("bws")
  72. return pm.installed_package("bws").binary
  73. except Exception as exc:
  74. logger.warning("bws auto-install failed: %s", exc)
  75. return None
  76. def install_bws(*, force: bool = False) -> Path:
  77. """Explicit setup/repair; PM re-verifies even a warm entry (including force)."""
  78. import pm
  79. pm.ensure("bws", explicit=True)
  80. return pm.installed_package("bws").binary
  81. # --- Encrypted last-good cache (opt-in) -------------------------------------
  82. def _b64e(raw: bytes) -> str:
  83. return base64.b64encode(raw).decode("ascii")
  84. def _derive_encrypted_cache_key(access_token: str, salt: bytes) -> bytes:
  85. """HKDF the local cache key from the bootstrap BWS token. cryptography is imported
  86. lazily: eagerly mapping ``_rust.pyd`` on Windows blocks the updater replacing it."""
  87. # Keep the native cryptography extension lazy. Most CLI commands import this module while building
  88. # argparse, even though only encrypted-cache reads/writes need it. See #73381.
  89. from cryptography.hazmat.primitives import hashes
  90. from cryptography.hazmat.primitives.kdf.hkdf import HKDF
  91. return HKDF(algorithm=hashes.SHA256(), length=32, salt=salt,
  92. info=_ENCRYPTED_CACHE_INFO).derive(access_token.encode("utf-8"))
  93. def _write_encrypted_disk_cache(*, cache_key: _CacheKey, access_token: str, entry: _CachedFetch,
  94. home_path: Optional[Path] = None) -> None:
  95. """Persist an AES-GCM encrypted last-good entry atomically (best-effort). The raw
  96. token only derives the key; a successful write removes the legacy plaintext cache."""
  97. try:
  98. from cryptography.hazmat.primitives.ciphers.aead import AESGCM
  99. salt = os.urandom(16)
  100. nonce = os.urandom(12)
  101. serialized_key = _cache_key_str(cache_key)
  102. key = _derive_encrypted_cache_key(access_token, salt)
  103. plaintext = json.dumps(
  104. {"secrets": entry.secrets, "fetched_at": entry.fetched_at}, separators=(",", ":"),
  105. ).encode("utf-8")
  106. ciphertext = AESGCM(key).encrypt(nonce, plaintext, serialized_key.encode("utf-8"))
  107. payload = {"version": _ENCRYPTED_CACHE_VERSION, "key": serialized_key,
  108. "salt": _b64e(salt), "nonce": _b64e(nonce), "ciphertext": _b64e(ciphertext)}
  109. atomic_write_json(_current_encrypted_disk_cache_path(home_path), payload)
  110. _STORE.disk.clear(home_path)
  111. except Exception:
  112. return
  113. def _read_encrypted_disk_cache(*, cache_key: _CacheKey, access_token: str, max_age_seconds: float,
  114. home_path: Optional[Path] = None) -> Optional[_CachedFetch]:
  115. """Decrypted encrypted-cache entry if it matches ``cache_key`` and is in-window."""
  116. if max_age_seconds <= 0:
  117. return None
  118. try:
  119. from cryptography.hazmat.primitives.ciphers.aead import AESGCM
  120. payload = json.loads(_current_encrypted_disk_cache_path(home_path).read_text(encoding="utf-8-sig"))
  121. serialized_key = _cache_key_str(cache_key)
  122. if (not isinstance(payload, dict)
  123. or payload.get("version") != _ENCRYPTED_CACHE_VERSION
  124. or payload.get("key") != serialized_key):
  125. return None
  126. salt, nonce, ciphertext = (base64.b64decode(str(payload.get(k, "")).encode("ascii"), validate=True)
  127. for k in ("salt", "nonce", "ciphertext"))
  128. key = _derive_encrypted_cache_key(access_token, salt)
  129. entry = entry_from_payload(json.loads(
  130. AESGCM(key).decrypt(nonce, ciphertext, serialized_key.encode("utf-8")).decode("utf-8")
  131. ))
  132. if entry is None:
  133. return None
  134. entry_age = time.time() - entry.fetched_at
  135. return None if entry_age < 0 or entry_age > max_age_seconds else entry
  136. except Exception:
  137. return None
  138. # --- Secret fetch -----------------------------------------------------------
  139. def fetch_bitwarden_secrets(
  140. *, access_token: str, project_id: str, binary: Optional[Path] = None,
  141. cache_ttl_seconds: float = 300, use_cache: bool = True, server_url: str = "",
  142. home_path: Optional[Path] = None, encrypted_cache_enabled: bool = False,
  143. encrypted_cache_max_stale_seconds: float = 0,
  144. ) -> tuple[dict[str, str], list[str]]:
  145. """Pull the secrets for ``project_id`` from BSM → ``(secrets, warnings)``.
  146. ``server_url``: region / self-hosted instance (empty = US Cloud). With
  147. ``encrypted_cache_enabled`` fresh entries are written AES-GCM encrypted and a
  148. last-good entry may be served after NETWORK/TIMEOUT failures for up to
  149. ``encrypted_cache_max_stale_seconds`` — independent of the fresh TTL, so
  150. ``cache_ttl_seconds: 0`` can coexist with a break-glass offline cache.
  151. Raises ``RuntimeError`` on fatal conditions (missing binary, auth failure,
  152. unparseable output); env_loader catches, the setup wizard lets it propagate.
  153. """
  154. if not access_token:
  155. raise RuntimeError("Bitwarden access token is empty")
  156. if not project_id:
  157. raise RuntimeError("Bitwarden project_id is empty")
  158. cache_key = (_token_fingerprint(access_token), project_id, server_url or "")
  159. def _read_encrypted(max_age: float) -> Optional[_CachedFetch]:
  160. return _read_encrypted_disk_cache(cache_key=cache_key, access_token=access_token,
  161. max_age_seconds=max_age, home_path=home_path)
  162. if use_cache and cache_ttl_seconds > 0:
  163. # L2 (~5ms) vs ~380ms for `bws secret list`.
  164. cached = _STORE.lookup(
  165. cache_key, cache_ttl_seconds, home_path,
  166. read_disk=(lambda: _read_encrypted(cache_ttl_seconds)) if encrypted_cache_enabled else None,
  167. )
  168. if cached is not None:
  169. return cached.secrets, []
  170. bws = binary or find_bws(install_if_missing=True)
  171. if bws is None:
  172. raise RuntimeError("bws binary not available — auto-install failed and `bws` is "
  173. "not on PATH. Install manually from "
  174. "https://github.com/bitwarden/sdk-sm/releases or re-run "
  175. "`hermes secrets bitwarden setup`.")
  176. try:
  177. secrets, warnings = _run_bws_list(bws, access_token, project_id, server_url)
  178. except RuntimeError as exc:
  179. # Stale fallback ONLY for transport failures — never AUTH_FAILED / INTERNAL,
  180. # where old secrets would mask a real problem (without it a fleet sharing
  181. # one project all stops on a network blip). With the encrypted cache on it
  182. # is the ONLY fallback (at-rest payload must never be plaintext); else the
  183. # plain DiskCache is read with ttl=inf, but only when the real TTL > 0.
  184. if use_cache and _classify_bws_error(str(exc)) in (ErrorKind.NETWORK, ErrorKind.TIMEOUT):
  185. stale = label = None
  186. if encrypted_cache_enabled:
  187. stale = _read_encrypted(encrypted_cache_max_stale_seconds)
  188. label = "stale ENCRYPTED disk cache"
  189. elif cache_ttl_seconds > 0:
  190. stale = _STORE.disk.read(cache_key, float("inf"), home_path)
  191. label = "stale disk cache"
  192. if stale is not None:
  193. age = max(0.0, time.time() - stale.fetched_at)
  194. _STORE.memory[cache_key] = stale
  195. return stale.secrets, [
  196. f"bws live fetch failed ({exc}); falling back to {label} ({int(age)}s old)"
  197. ]
  198. raise
  199. entry = _CachedFetch(secrets=secrets, fetched_at=time.time())
  200. if use_cache:
  201. if cache_ttl_seconds > 0:
  202. _STORE.memory[cache_key] = entry
  203. if encrypted_cache_enabled: # storage policy; max_stale only gates outage reads
  204. _write_encrypted_disk_cache(cache_key=cache_key, access_token=access_token,
  205. entry=entry, home_path=home_path)
  206. else:
  207. _STORE.disk.write(cache_key, entry, cache_ttl_seconds, home_path)
  208. return secrets, warnings
  209. def _summarize_bws_stderr(raw: str) -> str:
  210. """Reduce a bws (color-eyre) error dump to its numbered cause lines joined with
  211. ``; `` (dropping ``Location:``/``Backtrace`` on); raw text if unrecognized."""
  212. text = raw.replace("\x1b", "").strip()
  213. causes: list[str] = []
  214. for line in text.splitlines():
  215. stripped = line.strip()
  216. if stripped.startswith(("Location:", "Backtrace omitted", "Run with ")):
  217. break
  218. if stripped not in ("", "Error:") and (cause := re.sub(r"^\d+:\s*", "", stripped)):
  219. causes.append(cause)
  220. return "; ".join(causes) if causes else text
  221. def _run_bws_list(bws: Path, access_token: str, project_id: str, server_url: str = "") -> tuple[dict[str, str], list[str]]:
  222. cmd = [str(bws), "secret", "list", project_id, "--output", "json"]
  223. # The bws child intentionally receives the access token; a profile-local
  224. # fetch must not inherit sibling credentials (source_child_env).
  225. env = source_child_env()
  226. env["BWS_ACCESS_TOKEN"] = access_token
  227. env.setdefault("NO_COLOR", "1")
  228. if server_url: # empty keeps whatever BWS_SERVER_URL the shell already had
  229. env["BWS_SERVER_URL"] = server_url
  230. proc = run_cli(cmd, env=env, timeout=_BWS_RUN_TIMEOUT, label="bws",
  231. timeout_message=f"bws timed out after {_BWS_RUN_TIMEOUT}s fetching secrets")
  232. if proc.returncode != 0:
  233. err = _summarize_bws_stderr(proc.stderr or proc.stdout or "")
  234. raise RuntimeError(f"bws exited {proc.returncode}: {err[:200]}")
  235. raw = proc.stdout.strip()
  236. if not raw:
  237. return {}, ["bws returned no output (empty project?)"]
  238. try:
  239. payload = json.loads(raw)
  240. except json.JSONDecodeError as exc:
  241. raise RuntimeError(f"bws returned non-JSON output: {exc}") from exc
  242. if not isinstance(payload, list):
  243. raise RuntimeError(f"bws returned unexpected shape: {type(payload).__name__}")
  244. secrets: dict[str, str] = {}
  245. warnings: list[str] = []
  246. for item in payload:
  247. key, value = (item.get("key"), item.get("value")) if isinstance(item, dict) else (None, None)
  248. if not isinstance(key, str) or not isinstance(value, str):
  249. continue
  250. if _is_valid_env_name(key):
  251. secrets[key] = value
  252. else:
  253. warnings.append(f"Skipping secret {key!r}: not a valid env-var name")
  254. return secrets, warnings
  255. class BitwardenSource(SecretSource):
  256. """Bitwarden Secrets Manager as a registered **bulk** source (injects every
  257. secret in the project, so explicit mapped bindings outrank it)."""
  258. name = "bitwarden"
  259. label = "Bitwarden Secrets Manager"
  260. shape = "bulk"
  261. scheme = "bws"
  262. token_env_key = "access_token_env"
  263. default_token_env = "BWS_ACCESS_TOKEN"
  264. # override_existing defaults True: the point of BSM is centralized rotation
  265. # — a stale .env line must not have the final say.
  266. override_existing_default = True
  267. _AUTH_HINT = (
  268. "Run `hermes secrets bitwarden token` to paste a fresh access "
  269. "token (create one in the Bitwarden web app: Secrets Manager → "
  270. "Machine accounts → Access tokens). Wrong region? Re-run "
  271. "`hermes secrets bitwarden setup` and pick EU/self-hosted."
  272. )
  273. remediation_hints = {ErrorKind.AUTH_FAILED: _AUTH_HINT, ErrorKind.AUTH_EXPIRED: _AUTH_HINT}
  274. def config_schema(self) -> dict:
  275. return {
  276. "enabled": {"description": "Master switch", "default": False},
  277. "access_token_env": {"description": "Env var holding the machine-account access token",
  278. "default": "BWS_ACCESS_TOKEN"},
  279. "project_id": {"description": "BSM project UUID", "default": ""},
  280. "cache_ttl_seconds": {"description": "Fresh disk+memory cache TTL; 0 disables fresh-cache reuse",
  281. "default": 300},
  282. "encrypted_cache": {"description": "Encrypted last-good cache for network/timeout fallback",
  283. "default": {"enabled": False, "max_stale_seconds": 0}},
  284. "override_existing": {"description": "BSM values overwrite .env/shell values", "default": True},
  285. "auto_install": {"description": "Auto-download the pinned bws binary", "default": True},
  286. "server_url": {"description": "Region / self-hosted endpoint (empty = US Cloud)", "default": ""},
  287. }
  288. def fetch(self, cfg: dict, home_path: Path) -> FetchResult:
  289. cfg = cfg if isinstance(cfg, dict) else {}
  290. result = FetchResult()
  291. access_token_env = self.token_env(cfg)
  292. access_token = get_source_environment().get(access_token_env, "").strip()
  293. if not access_token:
  294. return result.fail(f"secrets.bitwarden.enabled is true but {access_token_env} is "
  295. "not set. Run `hermes secrets bitwarden setup`.", ErrorKind.NOT_CONFIGURED)
  296. project_id = str(cfg.get("project_id") or "")
  297. if not project_id:
  298. return result.fail("secrets.bitwarden.project_id is empty. Run `hermes secrets bitwarden setup`.",
  299. ErrorKind.NOT_CONFIGURED)
  300. binary = find_bws(install_if_missing=bool(cfg.get("auto_install", True)))
  301. result.binary_path = binary
  302. if binary is None:
  303. return result.fail("bws binary not available and auto-install is disabled. "
  304. "Run `hermes secrets bitwarden setup` to install.", ErrorKind.BINARY_MISSING)
  305. encrypted_cfg = cfg.get("encrypted_cache")
  306. encrypted_cfg = encrypted_cfg if isinstance(encrypted_cfg, dict) else {}
  307. try:
  308. secrets, warnings = fetch_bitwarden_secrets(
  309. access_token=access_token, project_id=project_id, binary=binary,
  310. cache_ttl_seconds=coerce_float(cfg.get("cache_ttl_seconds", 300), 300.0),
  311. server_url=str(cfg.get("server_url", "") or "").strip(), home_path=home_path,
  312. encrypted_cache_enabled=bool(encrypted_cfg.get("enabled", False)),
  313. encrypted_cache_max_stale_seconds=coerce_float(encrypted_cfg.get("max_stale_seconds", 0), 0.0),
  314. )
  315. except RuntimeError as exc:
  316. result.fail(str(exc), _classify_bws_error(str(exc)))
  317. if result.error_kind == ErrorKind.AUTH_FAILED: # say what the raw OAuth reject means first
  318. result.error = ("Bitwarden rejected the machine-account access token "
  319. f"({access_token_env}) — it was likely revoked, expired, "
  320. f"or belongs to another region. ({result.error})")
  321. return result
  322. result.secrets = secrets
  323. result.warnings.extend(warnings)
  324. return result
  325. def clear_caches(home_path: Optional[Path] = None) -> None:
  326. """Drop in-process AND disk caches (plaintext and encrypted), e.g. after a token rotation."""
  327. _STORE.clear(home_path)
  328. remove_cache_file(resolve_cache_home(home_path), _ENCRYPTED_CACHE_BASENAME)
  329. _reset_cache_for_tests = clear_caches