_common.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237
  1. """Shared plumbing for the extracted dashboard routers — thin wrappers over the
  2. late-binding seam in :mod:`hermes_cli.web_deps` (web_server owns helpers/state;
  3. every access resolves at call time so ``monkeypatch.setattr(<owning module>, ...)`` wins)."""
  4. from __future__ import annotations
  5. import asyncio
  6. import contextlib
  7. import logging
  8. import re
  9. import sqlite3
  10. import time
  11. from typing import Any, Callable, Dict, Optional
  12. from fastapi import HTTPException
  13. from hermes_cli.config_backend import ConfigLockedError, ConfigValueError, ConfigWriteError
  14. from hermes_cli.web_deps import LateState, late
  15. from hermes_cli.web_server_profiles import _profile_cli_args
  16. # Same logger the handlers used before extraction (identical logger object).
  17. log = logging.getLogger("hermes_cli.web_server")
  18. _profile_scope = late("_profile_scope", "hermes_cli.web_server_profiles")
  19. _config_profile_scope = late("_config_profile_scope", "hermes_cli.web_server_profiles")
  20. _spawn_hermes_action = late("_spawn_hermes_action", "hermes_cli.web_server_gateway")
  21. # Config read-modify-write serialization for off-loop handlers (live lock —
  22. # LateState supports ``with``-blocks).
  23. _CONFIG_MUTATION_LOCK = LateState("_CONFIG_MUTATION_LOCK")
  24. @contextlib.contextmanager
  25. def config_write_scope(profile: Optional[str]):
  26. """Profile scope, then the config mutation lock — the write-path nesting
  27. every config-mutating handler uses."""
  28. with _profile_scope(profile):
  29. with _CONFIG_MUTATION_LOCK:
  30. yield
  31. async def scoped_to_thread(profile: Optional[str], fn: Callable[[], Any]) -> Any:
  32. """Run ``fn()`` inside ``_profile_scope(profile)`` on a worker thread."""
  33. def _run():
  34. with _profile_scope(profile):
  35. return fn()
  36. return await asyncio.to_thread(_run)
  37. async def config_scoped_to_thread(profile: Optional[str], fn: Callable[[], Any]) -> Any:
  38. """Run ``fn()`` inside ``_config_profile_scope(profile)`` on a worker thread —
  39. home + secret scope without the process-global skills-module swap."""
  40. def _run():
  41. with _config_profile_scope(profile):
  42. return fn()
  43. return await asyncio.to_thread(_run)
  44. def destructive_profile(profile: Optional[str], route: str) -> Optional[str]:
  45. """The profile a DESTRUCTIVE or PRIVILEGED route acts on, or 400 when it is ambiguous.
  46. One backend serves every profile, so an omitted ``profile`` on a route that deletes,
  47. overwrites or privileges profile-owned data is not a default — it silently meant
  48. "whichever home this process launched with". Named profile: honoured. Omitted:
  49. rejected as soon as the process hosts more than one profile
  50. (``is_multiplex_active()``, decided once at boot by
  51. ``activate_multi_profile_hosting_eagerly``). A genuinely single-profile host has
  52. nothing to confuse, so there an omitted profile keeps meaning the launch profile
  53. and `curl` against a plain ``hermes serve`` is unchanged.
  54. "Privileged" is the same class as "destructive": arming an auto-approved shell hook
  55. in the wrong profile is at least as bad as removing one from it.
  56. """
  57. if (profile or "").strip():
  58. return profile
  59. from agent.secret_scope import is_multiplex_active
  60. if is_multiplex_active():
  61. raise HTTPException(
  62. status_code=400,
  63. detail=f"{route} requires an explicit profile: this backend serves several profiles, "
  64. "so an unnamed target would act on the wrong profile's data.")
  65. return profile
  66. def config_refusal_http(exc: ConfigWriteError) -> Optional[HTTPException]:
  67. """The HTTP error for an EXPECTED config-backend refusal, or None for an unexpected failure.
  68. A lock (403), a value the backend cannot store (400, or 413 when too large; ours or the
  69. plane's ``config_value_invalid``/``config_path_*``/``config_secret_literal``) and a lost CAS
  70. race (409) are the
  71. caller's to fix, and their messages are ours or the config plane's own refusal text (a path
  72. and the level that locks it; never a value or a credential), so they pass through (design
  73. §4.3: v1 shows the server's 403 message). Anything else stays opaque."""
  74. if isinstance(exc, ConfigLockedError):
  75. return HTTPException(status_code=403, detail=str(exc))
  76. if isinstance(exc, ConfigValueError):
  77. status = 413 if exc.code in ("config_level_too_large", "config_body_too_large") else 400
  78. return HTTPException(status_code=status, detail=str(exc))
  79. if exc.code == "config_version_conflict":
  80. return HTTPException(status_code=409, detail=str(exc))
  81. return None
  82. @contextlib.contextmanager
  83. def http_failure(log_msg: str, status: int, prefix: Optional[str] = None, *, detail: Optional[str] = None):
  84. """Map unexpected exceptions to an ``HTTPException``.
  85. ``HTTPException`` passes through; an expected config-backend refusal becomes its own status
  86. and message (:func:`config_refusal_http`); anything else is logged with ``log_msg`` (traceback),
  87. then re-raised as ``HTTPException(status, f"{prefix}: {exc}")`` — or ``detail`` when given
  88. (fixed message, exception text only in the log).
  89. """
  90. try:
  91. yield
  92. except HTTPException:
  93. raise
  94. except ConfigWriteError as exc:
  95. refusal = config_refusal_http(exc)
  96. if refusal is None:
  97. log.exception(log_msg)
  98. raise HTTPException(status_code=status, detail=detail if detail is not None else f"{prefix}: {exc}")
  99. raise refusal from exc
  100. except Exception as exc:
  101. log.exception(log_msg)
  102. raise HTTPException(status_code=status, detail=detail if detail is not None else f"{prefix}: {exc}")
  103. def spawn_profile_action(
  104. profile: Optional[str], argv: list, name: str, *, log_msg: str, prefix: str
  105. ) -> dict:
  106. """Spawn a background ``hermes -p <profile> <argv>`` action; a spawn
  107. failure is logged and becomes ``500 "<prefix>: <exc>"``."""
  108. with http_failure(log_msg, 500, prefix):
  109. proc = _spawn_hermes_action(_profile_cli_args(profile) + argv, name)
  110. return {"ok": True, "pid": proc.pid, "name": name}
  111. def require(value: Optional[str], detail: str) -> str:
  112. """Strip ``value``; 400 with ``detail`` when empty."""
  113. stripped = (value or "").strip()
  114. if not stripped:
  115. raise HTTPException(status_code=400, detail=detail)
  116. return stripped
  117. REDACTED_CREDENTIAL_WRITE_DETAIL = (
  118. "Refusing to save a redacted credential preview; re-enter the full secret to replace it."
  119. )
  120. def redacted_credential_preview(value: Any) -> Optional[str]:
  121. """Return a display-only credential sentinel that can never gain write authority."""
  122. if not value:
  123. return None
  124. from hermes_cli.config import redact_key
  125. return f"«redacted:{redact_key(str(value))}»"
  126. # Legacy bare masks (pre-sentinel pages, older Desktop builds) are recognised by the
  127. # producer shape of ``agent.redact.mask_secret`` — never by equality to the current
  128. # secret, which would authorise a stale preview after a rotation (#121002).
  129. _LEGACY_MASK_RE = re.compile(r".{4}\.\.\..{4}")
  130. def is_redacted_credential_preview(submitted: Any) -> bool:
  131. """Recognize current, stale and legacy dashboard previews by shape alone."""
  132. value = str(submitted or "")
  133. # Any ``«redacted…`` value is already-masked output (the same test agent.redact uses
  134. # to skip re-masking): our ``«redacted:…»`` sentinel, ``«redacted-secret»`` and the
  135. # vault marker ``«redacted-vault-secret»``. Then the legacy bare mask shapes.
  136. if value.startswith("«redacted"):
  137. return True
  138. return value == "***" or _LEGACY_MASK_RE.fullmatch(value) is not None
  139. # Corrupt-store reporting for polled read endpoints. The dashboard polls analytics every few
  140. # seconds; a persistently malformed state.db once produced ~520K identical tracebacks in 24 h
  141. # (#96591). One WARNING per store per interval, then debug; the caller gets an explicit status
  142. # instead of a 500. The file is never quarantined or renamed from here — that is `hermes doctor`'s job.
  143. _CORRUPT_STORE_WARN_INTERVAL_S = 300.0
  144. _corrupt_store_warned_at: dict[str, float] = {} # {db path: monotonic}
  145. CORRUPT_STORE_DETAIL = {
  146. "error": "state_db_corrupt",
  147. "message": "state.db corrupt — run `hermes doctor` (then `hermes doctor --fix` or `hermes sessions repair`).",
  148. }
  149. # One payload per persistence-cause bucket (``classify_persistence_error``); ``error`` codes
  150. # all follow the ``state_db_*`` scheme of the pre-existing ``state_db_corrupt``.
  151. # Same guidance as the deleted_wal / replaced turn explainers: `doctor --fix` while a holder
  152. # lives would repair the wrong generation in place, so it is deliberately NOT suggested here.
  153. DELETED_WAL_DETAIL = {
  154. "error": "state_db_deleted_wal",
  155. "message": "another Hermes process still holds an old copy of the session database's write-ahead log — "
  156. "quit every Hermes process on this profile, run `hermes doctor` (it names the holders), "
  157. "then start Hermes again. Do not run `hermes doctor --fix` while they run.",
  158. }
  159. STATE_DB_REPLACED_DETAIL = {
  160. "error": "state_db_replaced",
  161. "message": "state.db was replaced while Hermes was running — stop Hermes, run `hermes doctor`, "
  162. "then start it again. Do not run `hermes doctor --fix`, which would repair the wrong file in place.",
  163. }
  164. # Every other bucket a malformed image can classify as ("corrupt", "fts_index") is the corrupt payload.
  165. _STORE_STATUS_DETAIL_BY_CAUSE = {"deleted_wal": DELETED_WAL_DETAIL, "replaced": STATE_DB_REPLACED_DETAIL}
  166. def corrupt_store_status(db_path, exc) -> HTTPException:
  167. """The structured ``state_db_corrupt`` 503 for a malformed store, warning once per store per
  168. :data:`_CORRUPT_STORE_WARN_INTERVAL_S` (debug afterwards). Callers ``raise ... from exc``."""
  169. from hermes_state_errors import classify_persistence_error
  170. key, now = str(db_path), time.monotonic()
  171. last = _corrupt_store_warned_at.get(key)
  172. detail = _STORE_STATUS_DETAIL_BY_CAUSE.get(classify_persistence_error(exc), CORRUPT_STORE_DETAIL)
  173. if last is None or now - last >= _CORRUPT_STORE_WARN_INTERVAL_S:
  174. _corrupt_store_warned_at[key] = now
  175. log.warning("state.db at %s is unreadable (%s); dashboard reads return a status payload until it is "
  176. "repaired — run `hermes doctor`", db_path, exc)
  177. else:
  178. log.debug("state.db at %s still has error: %s", db_path, exc)
  179. return HTTPException(status_code=503, detail={**detail, "path": key})
  180. @contextlib.contextmanager
  181. def corrupt_store_as_status(db_path):
  182. """Map a corrupt-image ``sqlite3.DatabaseError`` or ``StateDbReplacedError`` from a state.db read to a 503 status
  183. payload, warning once per store per :data:`_CORRUPT_STORE_WARN_INTERVAL_S`.
  184. Busy/locked and every other error propagate unchanged."""
  185. from hermes_state_errors import StateDbReplacedError, classify_persistence_error, is_malformed_db_error
  186. try:
  187. yield
  188. except (sqlite3.DatabaseError, StateDbReplacedError) as exc:
  189. if not isinstance(exc, StateDbReplacedError) and not is_malformed_db_error(exc):
  190. raise
  191. raise corrupt_store_status(db_path, exc) from exc