| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533 |
- """``hermes sessions repair-profiles`` — settle crossed-profile durable state across every store.
- The per-profile store model (#88734) and the identity fences that followed are forward-only: they
- put NEW rows in the right place and refuse to widen existing damage, but nothing walks the stores
- and settles what an earlier release left crossed. This does, for six kinds of crossing:
- 1. ``sessions.profile_name`` disagreeing with the profile in the row's own ``session_key``;
- 2. rows physically in the wrong store (a named profile's keys in the root ``state.db``, another
- profile's keys in ``profiles/<p>/state.db``) — moved to the owning profile's store;
- 3. ``parent_session_id`` crossing namespaces — severed (the child's own identity is untouched);
- 4. ``gateway_routing`` rows in a named profile's store (the routing index lives in the gateway's
- home; #66887 copied every profile's rows into whichever store was scoped) — moved to the routing
- store when absent there, dropped otherwise — and root rows for a namespace no profile claims;
- 5. profile-less Telegram topic bindings and ``gateway_voice_mode.json`` entries owned by a
- non-default bot — relabelled from the evidence of the rows themselves;
- 6. ``sessions.json`` mirror entries whose key namespace no served profile claims — dropped, since
- the legacy import re-injects any key the routing DB lacks on every boot.
- Legacy ``agent:main:`` rows inside a named profile's store are reported but NOT repaired unless
- ``--legacy-main`` says how: they are either a standalone (pre-multiplex) gateway's own history that
- must be rekeyed to ``agent:<p>:`` (the #113884 incident), or a default-profile chat that leaked in
- under a scoped write (#102157) and must move to the root store. The row carries no evidence that
- tells the two apart, and guessing wrong hands a conversation to the other bot.
- Report-only by default. ``--apply`` refuses while a gateway owning any touched store is live (it
- holds the routing index in memory and writes it back), takes a quick snapshot of every store it
- will mutate, and is idempotent: a second run finds nothing.
- """
- from __future__ import annotations
- import contextlib
- import json
- import logging
- from dataclasses import dataclass, field
- from pathlib import Path
- from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple
- logger = logging.getLogger(__name__)
- _LEGACY_MAIN_CHOICES = ("report", "rekey", "move")
- _VOICE_MODE_FILE = "gateway_voice_mode.json"
- @dataclass(frozen=True)
- class Store:
- """One profile's ``state.db``; ``routing`` marks the store that owns the multiplexer's index."""
- profile: str
- home: Path
- routing: bool = False
- @property
- def db_path(self) -> Path:
- return self.home / "state.db"
- @dataclass
- class Finding:
- kind: str
- store: str
- subject: str
- detail: str
- action: Optional[str] = None # what --apply does; None = report only
- reason: Optional[str] = None # why it is report only
- _fix: Optional[Callable[[_Session], dict[str, int]]] = field(default=None, repr=False, compare=False)
- @property
- def repairable(self) -> bool:
- return self._fix is not None
- def as_dict(self) -> dict[str, Any]:
- return {"kind": self.kind, "store": self.store, "subject": self.subject, "detail": self.detail,
- "action": self.action, "reason": self.reason}
- class _Session:
- """Open store handles for one run; every store opened at most once, all released at the end."""
- def __init__(self, *, read_only: bool) -> None:
- self.read_only = read_only
- self._dbs: dict[Path, Any] = {}
- def db(self, store: Store):
- """The store's SessionDB. Read-only runs never create a file: a live profile that has no
- ``state.db`` yet reads as empty; an apply run opens it for real when a move lands there."""
- path = store.db_path
- if path not in self._dbs:
- if self.read_only:
- from hermes_state import SessionDB
- self._dbs[path] = SessionDB(path, read_only=True) if path.exists() else _EMPTY_STORE
- else:
- from hermes_state_registry import acquire
- self._dbs[path] = acquire(path)
- return self._dbs[path]
- def close(self) -> None:
- from hermes_state_registry import release_or_close
- for db in self._dbs.values():
- if db is _EMPTY_STORE:
- continue
- with contextlib.suppress(Exception):
- release_or_close(db) if not self.read_only else db.close()
- self._dbs.clear()
- class _EmptyStore:
- """Stand-in for a live profile whose ``state.db`` does not exist yet: nothing to find."""
- def find_crossed_profile_sessions(self, owner):
- return {"mislabelled": [], "foreign": [], "crossed_parents": []}
- def find_profile_less_telegram_topic_rows(self):
- return []
- def list_gateway_routing_rows(self):
- return []
- def key_profiles_for_chat(self, platform, chat_id):
- return set()
- _EMPTY_STORE = _EmptyStore()
- # ── enumeration ───────────────────────────────────────────────────────────────
- def enumerate_stores() -> list[Store]:
- """Default root plus every live named profile (a live profile claims its namespace whether or not
- it has written a ``state.db`` yet). The root store owns the routing index: the multiplexer's
- ``_routing_home`` is its launch home, the root."""
- from hermes_cli.profiles import _get_default_hermes_home, _iter_named_profile_dirs
- root = _get_default_hermes_home()
- stores = [Store("default", root, routing=True)]
- stores.extend(Store(entry.name, entry) for entry in _iter_named_profile_dirs())
- return stores
- def _gateway_multiplexes(root: Path) -> bool:
- """Does the default gateway serve every profile? Same reader as every other CLI surface: the live
- record, else the explicit flag, else False — an unset flag is a verdict only the gateway reaches,
- and guessing "yes" would uproot a standalone gateway's own routing index."""
- from hermes_cli.gateway_multiplex_mode import default_gateway_multiplexes
- try:
- return default_gateway_multiplexes(root)
- except Exception:
- return False
- def live_gateway_homes(stores: Iterable[Store]) -> list[tuple[str, int]]:
- from gateway.status import live_gateway_pid_for_home
- live = []
- for store in stores:
- pid = live_gateway_pid_for_home(store.home)
- if pid is not None:
- live.append((store.profile, pid))
- return live
- # ── scan ──────────────────────────────────────────────────────────────────────
- class RepairPlan:
- def __init__(self, stores: list[Store], *, legacy_main: str = "report") -> None:
- if legacy_main not in _LEGACY_MAIN_CHOICES:
- raise ValueError(f"legacy_main must be one of {_LEGACY_MAIN_CHOICES}")
- self.stores = stores
- self.by_profile = {s.profile: s for s in stores}
- self.claimed: set[str] = set(self.by_profile)
- self.routing_store = next((s for s in stores if s.routing), None)
- self.legacy_main = legacy_main
- self.multiplexes = _gateway_multiplexes(self.routing_store.home) if self.routing_store else False
- self.findings: list[Finding] = []
- self._moves: dict[tuple[Path, Path], _MoveBatch] = {}
- # -- helpers --
- def _add(self, finding: Finding) -> None:
- self.findings.append(finding)
- @property
- def repairable(self) -> list[Finding]:
- return [f for f in self.findings if f.repairable]
- # -- scan --
- def scan(self, session: _Session) -> RepairPlan:
- for store in self.stores:
- self._scan_sessions(session, store)
- self._scan_topics(session, store)
- for store in self.stores:
- self._scan_routing(session, store)
- if self.routing_store is not None:
- self._scan_voice_modes(session, self.routing_store)
- self._scan_sessions_json(self.routing_store)
- return self
- def _scan_sessions(self, session: _Session, store: Store) -> None:
- db = session.db(store)
- crossed = db.find_crossed_profile_sessions(store.profile)
- moving: set[str] = set()
- held = _ledgered_lineages([r for r in crossed["foreign"] if self._would_move(store, r)])
- for row in crossed["foreign"]:
- if row["id"] in held:
- moving.add(row["id"]) # neither relabelled nor severed in place: it is a stray
- self._add(Finding(
- "wrong_store", store.profile, row["id"],
- f"key {row['session_key']!r} ({row['message_count']} messages) sits in the "
- f"{store.profile} store", reason=held[row["id"]]))
- continue
- self._plan_foreign_row(store, row, moving)
- for row in crossed["mislabelled"]:
- if row["id"] in moving:
- continue # the move stamps the right label
- self._add(Finding(
- "mislabelled", store.profile, row["id"],
- f"profile_name={row['profile_name']!r} but key {row['session_key']!r} belongs to "
- f"{row['key_profile']!r}", action=f"relabel to {row['key_profile']!r}",
- _fix=lambda s, st=store, sid=row["id"]: {
- "relabelled": s.db(st).relabel_sessions_to_key_profile([sid])}))
- for row in crossed["crossed_parents"]:
- if row["id"] in moving:
- continue # the import drops a parent that is not in the target store
- self._add(Finding(
- "crossed_parent", store.profile, row["id"],
- f"{row['key_profile']!r} row inherits from parent {row['parent_session_id']} keyed under "
- f"{row['parent_profile']!r}", action="sever parent_session_id",
- _fix=lambda s, st=store, sid=row["id"]: {"severed": s.db(st).sever_crossed_parents([sid])}))
- def _would_move(self, store: Store, row: dict[str, Any]) -> bool:
- """Mirror of :meth:`_plan_foreign_row`: does the plan copy this row to another store?"""
- if row["key_profile"] == "default" and store.profile != "default":
- return self.legacy_main == "move" and self.routing_store is not None
- return row["key_profile"] in self.claimed
- def _plan_foreign_row(self, store: Store, row: dict[str, Any], moving: set[str]) -> None:
- key_profile, sid = row["key_profile"], row["id"]
- detail = f"key {row['session_key']!r} ({row['message_count']} messages) sits in the {store.profile} store"
- if key_profile == "default" and store.profile != "default":
- self._plan_legacy_main_row(store, row, moving)
- return
- if key_profile not in self.claimed:
- self._add(Finding(
- "unclaimed_namespace", store.profile, sid, detail,
- reason=f"no live profile named {key_profile!r}; create it, then rerun, or "
- f"`hermes profile migrate-identity {key_profile} <target>`"))
- return
- moving.add(sid)
- self._add(Finding(
- "wrong_store", store.profile, sid, detail, action=f"move to the {key_profile} store",
- _fix=self._move_batch(store, self.by_profile[key_profile]).fix_for(sid)))
- def _plan_legacy_main_row(self, store: Store, row: dict[str, Any], moving: set[str]) -> None:
- sid = row["id"]
- detail = (f"legacy default-namespace key {row['session_key']!r} ({row['message_count']} messages) "
- f"in the {store.profile} store")
- if self.legacy_main == "rekey":
- self._add(Finding(
- "legacy_main", store.profile, sid, detail, action=f"rekey to agent:{store.profile}:",
- _fix=lambda s, st=store, i=sid: {"rekeyed": s.db(st).rekey_legacy_main_sessions([i], st.profile)}))
- elif self.legacy_main == "move" and self.routing_store is not None:
- moving.add(sid)
- self._add(Finding(
- "legacy_main", store.profile, sid, detail, action="move to the default store",
- _fix=self._move_batch(store, self.routing_store).fix_for(sid)))
- else:
- self._add(Finding(
- "legacy_main", store.profile, sid, detail,
- reason="either a standalone gateway's own history (--legacy-main rekey) or a default "
- "chat that leaked in (--legacy-main move); the row cannot say which"))
- def _move_batch(self, src: Store, dst: Store) -> _MoveBatch:
- return self._moves.setdefault((src.db_path, dst.db_path), _MoveBatch(src, dst))
- def _scan_topics(self, session: _Session, store: Store) -> None:
- rows = session.db(store).find_profile_less_telegram_topic_rows()
- for row in rows:
- self._add(Finding(
- "topic_profile_less", store.profile, f"chat {row['chat_id']} thread {row['thread_id']}",
- f"telegram topic binding labelled 'default' but keyed {row['session_key']!r}",
- action=f"relabel to {row['key_profile']!r}",
- _fix=lambda s, st=store, r=row: s.db(st).relabel_telegram_topic_rows([r])))
- def _scan_routing(self, session: _Session, store: Store) -> None:
- from hermes_state_profile_repair import session_key_profile
- rows = session.db(store).list_gateway_routing_rows()
- for row in rows:
- key_profile = session_key_profile(row["session_key"])
- if key_profile is None:
- continue
- subject = f"{row['session_key']} (scope {row['scope']!r})"
- if store.routing:
- if key_profile not in self.claimed:
- self._add(Finding(
- "routing_unclaimed", store.profile, subject,
- f"routing row for a namespace no live profile claims ({key_profile!r}); every "
- "inbound event on it logs a missing-profile error", action="delete",
- _fix=lambda s, st=store, r=row: {
- "routing_deleted": s.db(st).delete_gateway_routing_rows([(r["scope"], r["session_key"])])}))
- continue
- if key_profile == store.profile and not self.multiplexes:
- continue # a standalone gateway's own index lives in its own store
- routing_store = self.routing_store
- if routing_store is None:
- continue
- self._add(Finding(
- "routing_stray", store.profile, subject,
- f"routing row in the {store.profile} store; the index lives in the default store",
- action="move to the default store (existing row there wins)",
- _fix=lambda s, st=store, r=row, rs=routing_store: self._move_routing_row(s, st, rs, r)))
- def _move_routing_row(self, session: _Session, store: Store, routing_store: Store,
- row: dict[str, Any]) -> dict[str, int]:
- adopted = session.db(routing_store).insert_gateway_routing_rows_if_absent(
- [(row["scope"], row["session_key"], row["entry_json"], row["updated_at"])])
- deleted = session.db(store).delete_gateway_routing_rows([(row["scope"], row["session_key"])])
- return {"routing_adopted": adopted, "routing_deleted": deleted}
- def _scan_voice_modes(self, session: _Session, store: Store) -> None:
- path = store.home / _VOICE_MODE_FILE
- try:
- data = json.loads(path.read_text(encoding="utf-8-sig"))
- except (FileNotFoundError, json.JSONDecodeError, OSError):
- return
- if not isinstance(data, dict):
- return
- for key in sorted(k for k in data if isinstance(k, str) and k.count(":") == 1):
- platform, chat_id = key.split(":", 1)
- owners: set[str] = set()
- for st in self.stores:
- owners |= session.db(st).key_profiles_for_chat(platform, chat_id)
- if not owners or "default" in owners:
- continue # the default bot speaks there: the unprefixed key is its own
- if len(owners) > 1:
- self._add(Finding(
- "voice_profile_less", store.profile, key,
- f"voice mode entry without a profile; chat is held by {sorted(owners)}",
- reason="more than one non-default bot speaks in this chat"))
- continue
- owner = next(iter(owners))
- self._add(Finding(
- "voice_profile_less", store.profile, key,
- f"voice mode {data[key]!r} without a profile; chat is held only by {owner!r}",
- action=f"rekey to {owner}:{key}",
- _fix=lambda s, p=path, k=key, o=owner: _rekey_voice_mode_entry(p, k, o)))
- def _scan_sessions_json(self, store: Store) -> None:
- from hermes_state_profile_repair import session_key_profile
- path = store.home / "sessions" / "sessions.json"
- try:
- data = json.loads(path.read_text(encoding="utf-8-sig"))
- except (FileNotFoundError, json.JSONDecodeError, OSError):
- return
- if not isinstance(data, dict):
- return
- for key in sorted(k for k in data if isinstance(k, str) and not k.startswith("_")):
- key_profile = session_key_profile(key)
- if key_profile is None or key_profile in self.claimed:
- continue
- self._add(Finding(
- "sessions_json_unclaimed", store.profile, key,
- f"sessions.json mirror entry for a namespace no live profile claims ({key_profile!r}); "
- "the legacy import re-injects it into routing on every boot", action="delete entry",
- _fix=lambda s, p=path, k=key: _drop_sessions_json_entry(p, k)))
- # -- apply --
- def apply(self, session: _Session, *, snapshot: Callable[[Store], Optional[str]]) -> dict[str, Any]:
- """Run every repairable fix; snapshots every store first. Fixes are independent and each
- re-checks its precondition, so a partial run leaves a state the next run completes."""
- snapshots = {s.profile: snapshot(s) for s in self.stores if s.db_path.exists()}
- totals: dict[str, int] = {}
- failures: list[dict[str, str]] = []
- for finding in self.repairable:
- fix = finding._fix
- assert fix is not None
- try:
- for key, count in (fix(session) or {}).items():
- totals[key] = totals.get(key, 0) + int(count)
- except Exception as exc: # one bad row must not abandon the rest of the plan
- logger.warning("repair-profiles: %s on %s failed", finding.kind, finding.subject, exc_info=True)
- failures.append({"kind": finding.kind, "subject": finding.subject,
- "error": f"{type(exc).__name__}: {exc}"})
- return {"snapshots": snapshots, "totals": totals, "failures": failures}
- class _MoveBatch:
- """Every session moving from one store to another, moved together: export all, import all
- (parents before children, so a moved child keeps its moved parent), then delete from the source.
- Row by row, deleting a parent would detach the children still waiting in the source. Copy
- precedes delete, so a crash between the two leaves a duplicate the next run settles
- (``present`` on import, then the delete). A row that fails is that row's failure alone: the
- rest of the batch still moves, and only its lineage (descendants, and the parent it still
- points at in the source) waits with it for the next run."""
- def __init__(self, src: Store, dst: Store) -> None:
- self.src, self.dst = src, dst
- self.ids: list[str] = []
- self._result: Optional[dict[str, dict[str, int]]] = None
- self._errors: dict[str, Exception] = {}
- def fix_for(self, sid: str) -> Callable[[_Session], dict[str, int]]:
- self.ids.append(sid)
- return lambda session: self._outcome(session, sid)
- def _outcome(self, session: _Session, sid: str) -> dict[str, int]:
- result = self.run(session)
- if sid in self._errors:
- raise self._errors[sid]
- return result.get(sid, {"missing": 1})
- def run(self, session: _Session) -> dict[str, dict[str, int]]:
- if self._result is not None:
- return self._result
- src_db, dst_db = session.db(self.src), session.db(self.dst)
- from hermes_state_profile_repair import SessionLedgeredError
- payloads: dict[str, Optional[dict[str, Any]]] = {}
- for sid in self.ids:
- try:
- payloads[sid] = src_db.export_session_for_move(sid)
- except SessionLedgeredError as exc: # gained ledger rows since the scan: never copied
- self._errors[sid] = exc
- ordered = _parents_first([p for p in payloads.values() if p is not None])
- result: dict[str, dict[str, int]] = {sid: {"missing": 1} for sid, p in payloads.items()
- if p is None and sid not in self._errors}
- for payload in ordered:
- sid, parent = payload["session"]["id"], payload["session"].get("parent_session_id")
- if parent in self._errors:
- # Imported now, it would land without the parent and never regain the link.
- self._errors[sid] = RuntimeError(
- f"its parent {parent} did not move; kept in the {self.src.profile} store with it")
- continue
- try:
- outcome = dst_db.import_moved_session(payload, profile_name=self.dst.profile)
- except Exception as exc:
- self._errors[sid] = exc
- continue
- if dst_db.count_messages_all(sid) < len(payload["messages"]):
- logger.warning("repair-profiles: %s copied into %s with fewer messages than the source; "
- "source row kept", sid, self.dst.profile)
- result[sid] = {"copied_incomplete": 1}
- continue
- result[sid] = {outcome: 1}
- # Deleting a row detaches its children still in the source; a child that did not move
- # must find its parent there on the next run.
- awaited = {p["session"].get("parent_session_id") for p in ordered if p["session"]["id"] in self._errors}
- for payload in ordered:
- sid = payload["session"]["id"]
- if sid not in result or sid in awaited or "copied_incomplete" in result[sid]:
- continue
- try:
- if src_db.delete_moved_session(sid):
- result[sid]["moved"] = 1
- except Exception as exc:
- self._errors[sid] = exc
- self._result = result
- return result
- def _ledgered_lineages(foreign: list[dict[str, Any]]) -> dict[str, str]:
- """Stray rows that must stay put, each with the reason. A row the gateway ledger references
- (admissions/workers, ``ON DELETE RESTRICT``) cannot be deleted from its store, so copying it
- would leave the session in both; the stray rows linked to it by ``parent_session_id`` stay
- with it, since moving a parent detaches the child left behind and a moved child loses its
- parent. Whether a move should carry the ledger is a separate policy decision."""
- by_id = {row["id"]: row for row in foreign}
- groups: dict[str, set[str]] = {sid: {sid} for sid in by_id}
- for row in foreign:
- parent = row.get("parent_session_id")
- if parent in by_id and groups[parent] is not groups[row["id"]]:
- merged = groups[parent] | groups[row["id"]]
- for sid in merged:
- groups[sid] = merged
- held: dict[str, str] = {}
- for sid, row in by_id.items():
- anchors = sorted(m for m in groups[sid] if by_id[m].get("ledgered"))
- if row.get("ledgered"):
- held[sid] = "has gateway admission/worker history, which a move cannot carry"
- elif anchors:
- held[sid] = f"its lineage holds {anchors[0]}, which has gateway admission/worker history"
- return held
- def _parents_first(payloads: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Order moved sessions so every parent in the batch precedes its children (a cycle, impossible
- for a well-formed store, falls back to insertion order)."""
- pending = {p["session"]["id"]: p for p in payloads}
- ordered: list[dict[str, Any]] = []
- while pending:
- ready = [sid for sid, p in pending.items() if p["session"].get("parent_session_id") not in pending]
- if not ready:
- ready = list(pending)
- for sid in ready:
- ordered.append(pending.pop(sid))
- return ordered
- def _rekey_voice_mode_entry(path: Path, key: str, owner: str) -> dict[str, int]:
- from utils import atomic_json_write
- data = json.loads(path.read_text(encoding="utf-8-sig"))
- if key not in data:
- return {}
- data.setdefault(f"{owner}:{key}", data.pop(key))
- data.pop(key, None)
- atomic_json_write(path, data)
- return {"voice_rekeyed": 1}
- def _drop_sessions_json_entry(path: Path, key: str) -> dict[str, int]:
- from utils import atomic_json_write
- data = json.loads(path.read_text(encoding="utf-8-sig"))
- if key not in data:
- return {}
- del data[key]
- atomic_json_write(path, data, mode=0o600)
- return {"sessions_json_dropped": 1}
- # ── entry points ──────────────────────────────────────────────────────────────
- def scan_stores(stores: Optional[list[Store]] = None, *, legacy_main: str = "report",
- read_only: bool = True) -> tuple[RepairPlan, _Session]:
- """Open every store and build the plan. The caller owns the returned session (close it)."""
- stores = enumerate_stores() if stores is None else stores
- session = _Session(read_only=read_only)
- try:
- plan = RepairPlan(stores, legacy_main=legacy_main).scan(session)
- except Exception:
- session.close()
- raise
- return plan, session
- def default_snapshot(store: Store) -> Optional[str]:
- from hermes_cli.backup import create_quick_snapshot
- try:
- return create_quick_snapshot(label="repair-profiles", hermes_home=store.home)
- except Exception as exc:
- logger.warning("repair-profiles: snapshot of %s failed: %s", store.home, exc)
- return None
|