session_export_md.py 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219
  1. """Markdown/QMD export helpers for Hermes sessions.
  2. Filesystem-only: formats already-exported SessionDB dicts and writes them to user-selected export
  3. directories. Must not mutate state.db or call delete/prune/archive APIs.
  4. """
  5. from __future__ import annotations
  6. import hashlib
  7. import json
  8. import re
  9. import time
  10. from datetime import datetime, timezone, UTC
  11. from pathlib import Path
  12. from typing import Any
  13. from hermes_cli.timefmt import coerce_epoch
  14. EXPORTER_VERSION = "hermes sessions export (md/qmd) v1"
  15. _SHA_LINE_RE = re.compile(r"- SHA256 of exported body: `([0-9a-f]{64})`")
  16. _SHA_PLACEHOLDER = "__SHA256_PLACEHOLDER__"
  17. _VERIFICATION_HEADING = "## Export verification"
  18. def _iso_timestamp(value: Any) -> str:
  19. if value is None or value == "":
  20. return ""
  21. if (ts := coerce_epoch(value)) is None:
  22. return str(value) # corrupt cell: odd-looking date, not an aborted export
  23. return datetime.fromtimestamp(ts, tz=UTC).isoformat().replace("+00:00", "Z")
  24. def _frontmatter_line(key: str, value: Any) -> str:
  25. if value is None or isinstance(value, bool):
  26. shown = {None: "null", True: "true", False: "false"}[value]
  27. else:
  28. shown = json.dumps(value if isinstance(value, (int, float, list)) else str(value), ensure_ascii=False)
  29. return f"{key}: {shown}"
  30. def _json_block(value: Any) -> str:
  31. return "```json\n" + json.dumps(value, ensure_ascii=False, indent=2) + "\n```"
  32. def _message_heading(message: dict[str, Any]) -> str:
  33. role = str(message.get("role") or "message")
  34. name = message.get("name") or message.get("tool_name")
  35. label = f"Tool — {name}" if role == "tool" and name else role.capitalize()
  36. timestamp = _iso_timestamp(message.get("created_at") or message.get("timestamp"))
  37. return f"### {label}{' — ' + timestamp if timestamp else ''}"
  38. def _session_id(session: dict[str, Any]) -> str:
  39. return str(session.get("id") or session.get("session_id") or "unknown-session")
  40. def _segments(session: dict[str, Any]) -> list[dict[str, Any]]:
  41. segments = session.get("segments")
  42. return [s for s in segments if isinstance(s, dict)] if isinstance(segments, list) and segments else [session]
  43. def _message_count(session: dict[str, Any]) -> int:
  44. return sum(len(seg.get("messages") or []) for seg in _segments(session))
  45. def _render_messages(session: dict[str, Any]) -> str:
  46. segments = _segments(session)
  47. if _message_count(session) == 0:
  48. return "## Messages\n\n_No messages in this session._\n"
  49. parts: list[str] = ["## Messages\n"]
  50. for segment in segments:
  51. if len(segments) > 1:
  52. parts.append(f"## Compression segment: {_session_id(segment)}\n")
  53. for message in list(segment.get("messages") or []):
  54. parts.append(_message_heading(message) + "\n")
  55. content = message.get("content")
  56. rendered = "" if content is None else content.rstrip() if isinstance(content, str) else _json_block(content)
  57. if rendered:
  58. parts.append(rendered + "\n")
  59. if tool_calls := message.get("tool_calls"):
  60. parts.append("\n\n## Tool calls\n\n" + _json_block(tool_calls) + "\n")
  61. parts.append("")
  62. return "\n".join(parts).rstrip() + "\n"
  63. def _export_body_without_hash(session: dict[str, Any], *, fmt: str, exported_at: float) -> str:
  64. session_id = _session_id(session)
  65. exported_iso = _iso_timestamp(exported_at)
  66. message_count = _message_count(session)
  67. fields = [
  68. ("session_id", session_id),
  69. ("title", session.get("title")),
  70. ("source", session.get("source")),
  71. ("created_at", _iso_timestamp(session.get("started_at") or session.get("created_at"))),
  72. ("updated_at", _iso_timestamp(session.get("last_active") or session.get("updated_at"))),
  73. ("ended_at", _iso_timestamp(session.get("ended_at"))),
  74. ("model", session.get("model")),
  75. ("provider", session.get("billing_provider") or session.get("provider")),
  76. ("cwd", session.get("cwd")),
  77. ("archived", bool(session.get("archived"))),
  78. ("message_count", message_count),
  79. ("tool_call_count", session.get("tool_call_count") or 0),
  80. *([("lineage_session_ids", session["lineage_session_ids"])] if session.get("lineage_session_ids") else []),
  81. ("format", fmt), ("exported_at", exported_iso), ("exporter", EXPORTER_VERSION),
  82. ]
  83. parts = [
  84. "\n".join(["---", *(_frontmatter_line(k, v) for k, v in fields), "---", ""]),
  85. f"# {session.get('title') or session_id}\n", f"Session ID: `{session_id}`\n",
  86. *([f"Source: `{session.get('source')}`\n"] if session.get("source") else []),
  87. *([f"Working directory: `{session.get('cwd')}`\n"] if session.get("cwd") else []),
  88. _render_messages(session),
  89. f"{_VERIFICATION_HEADING}\n",
  90. f"- Session id: `{session_id}`",
  91. f"- Exported messages: `{message_count}`",
  92. f"- Source DB message count at export: `{session.get('message_count', message_count)}`",
  93. f"- Exported at: `{exported_iso}`",
  94. f"- SHA256 of exported body: `{_SHA_PLACEHOLDER}`",
  95. ]
  96. return "\n".join(parts).rstrip() + "\n"
  97. def _check_fmt(fmt: str) -> None:
  98. if fmt not in {"md", "qmd"}:
  99. raise ValueError("fmt must be 'md' or 'qmd'")
  100. def render_session_markdown(
  101. session: dict[str, Any], *, fmt: str = "md", include_verification: bool = True
  102. ) -> str:
  103. """Render a SessionDB export dictionary as Markdown/QMD text."""
  104. _check_fmt(fmt)
  105. body = _export_body_without_hash(session, fmt=fmt, exported_at=time.time())
  106. if not include_verification:
  107. return body.split(f"\n{_VERIFICATION_HEADING}\n", 1)[0].rstrip() + "\n"
  108. # The digest covers the body with the SHA line set to `pending`, which is what verify recomputes.
  109. digest_body = body.replace(f"`{_SHA_PLACEHOLDER}`", "`pending`")
  110. return body.replace(_SHA_PLACEHOLDER, hashlib.sha256(digest_body.encode("utf-8")).hexdigest())
  111. def safe_session_filename(session: dict[str, Any], *, fmt: str = "md") -> str:
  112. """Return a deterministic, path-safe filename for a session export."""
  113. _check_fmt(fmt)
  114. title = str(session.get("title") or "session")
  115. slug = re.sub(r"[^A-Za-z0-9._-]+", "-", title).strip(".-_").lower() or "session"
  116. return f"{_session_id(session)}-{slug[:60]}.{fmt}"
  117. def file_sha256(path: Path | str) -> str:
  118. return hashlib.sha256(Path(path).read_bytes()).hexdigest()
  119. def verify_export_file(path: Path | str, session: dict[str, Any]) -> tuple[bool, str]:
  120. if not Path(path).exists():
  121. return False, "file missing"
  122. text = Path(path).read_text(encoding="utf-8-sig")
  123. match = _SHA_LINE_RE.search(text)
  124. if not match:
  125. return False, "sha256 marker missing"
  126. digest_body = _SHA_LINE_RE.sub("- SHA256 of exported body: `pending`", text)
  127. if hashlib.sha256(digest_body.encode("utf-8")).hexdigest() != match.group(1):
  128. return False, "sha256 mismatch"
  129. if f"- Exported messages: `{_message_count(session)}`" not in text:
  130. return False, "message count mismatch"
  131. if f"- Session id: `{_session_id(session)}`" not in text:
  132. return False, "session id mismatch"
  133. return True, "ok"
  134. def redact_session_data(session: dict[str, Any]) -> dict[str, Any]:
  135. """Return a copy of a session export dict with secrets redacted.
  136. Every message's content and tool-call arguments go through the force-mode redaction pass
  137. (``agent.redact.redact_sensitive_text``) so credentials in tool output never land in exports.
  138. """
  139. from agent.redact import redact_sensitive_text
  140. def _clean(value: Any) -> Any:
  141. if isinstance(value, str):
  142. return redact_sensitive_text(value, force=True)
  143. if isinstance(value, list):
  144. return [_clean(v) for v in value]
  145. if isinstance(value, dict):
  146. return {k: _clean(v) for k, v in value.items()}
  147. return value
  148. return {**session, **{k: _clean(session[k]) for k in ("messages", "segments") if session.get(k) is not None}}
  149. def _export_dir(output_dir: Path | str) -> Path:
  150. out_dir = Path(output_dir).expanduser()
  151. out_dir.mkdir(parents=True, exist_ok=True)
  152. return out_dir
  153. def write_session_markdown(
  154. session: dict[str, Any], output_dir: Path | str, *, fmt: str = "md", force: bool = False
  155. ) -> Path:
  156. """Write a Markdown/QMD export file and return its path."""
  157. path = _export_dir(output_dir) / safe_session_filename(session, fmt=fmt)
  158. if path.exists() and not force:
  159. raise FileExistsError(str(path))
  160. path.write_text(render_session_markdown(session, fmt=fmt), encoding="utf-8")
  161. return path
  162. def append_manifest_entry(output_dir: Path | str, session: dict[str, Any], path: Path | str, *, fmt: str) -> Path:
  163. entry = {
  164. "session_id": _session_id(session),
  165. "lineage_session_ids": session.get("lineage_session_ids") or [_session_id(session)],
  166. "path": str(Path(path)),
  167. "format": fmt,
  168. "message_count": _message_count(session),
  169. "sha256": file_sha256(path),
  170. "exported_at": time.time(),
  171. }
  172. manifest = _export_dir(output_dir) / "manifest.jsonl"
  173. with manifest.open("a", encoding="utf-8") as fh:
  174. fh.write(json.dumps(entry, ensure_ascii=False, sort_keys=True) + "\n")
  175. return manifest