read_extract.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623
  1. """Document-to-text extraction for ``read_file``: stdlib Jupyter/DOCX/XLSX (always
  2. authoritative for those three), plus legacy Office/OpenDocument/RTF/EPUB/PDF when the
  3. optional ``firecrawl-anydoc`` package (imports as ``anydoc``) is installed. Malformed
  4. documents raise :class:`ExtractionError`; callers fall back to text/binary handling."""
  5. from __future__ import annotations
  6. import contextlib
  7. import functools
  8. import importlib
  9. import itertools
  10. import json
  11. import os
  12. import posixpath
  13. import re
  14. import shlex
  15. import shutil
  16. import subprocess
  17. import tempfile
  18. import threading
  19. import time
  20. import zipfile
  21. from pathlib import Path
  22. from typing import Any, Callable, Iterator, Optional
  23. from xml.etree import ElementTree as ET
  24. __all__ = ["EXTRACTABLE_EXTENSIONS", "ExtractionError", "extract_document_bytes",
  25. "extract_document_text", "is_extractable_document"]
  26. EXTRACTABLE_EXTENSIONS = frozenset({".ipynb", ".docx", ".xlsx", ".db", ".sqlite", ".sqlite3"})
  27. ANYDOC_EXTENSIONS = frozenset({
  28. ".doc", ".docm", ".ppt", ".pps", ".pot", ".pptx", ".pptm", ".ppsx", ".ppsm",
  29. ".xls", ".xlsm", ".xlsb", ".odt", ".ods", ".odp", ".rtf", ".epub", ".pdf"})
  30. # anydoc loads whole files (no streaming); read_file's char budget applies only post-conversion.
  31. MAX_ANYDOC_BYTES = 50 * 1024 * 1024
  32. MAX_DOCUMENT_BYTES = 50 * 1024 * 1024
  33. _MAX_XLSX_ROWS_PER_SHEET = 5000
  34. _MAX_XLSX_COLS = 256
  35. _NS_W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
  36. _NS_S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
  37. _NS_REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
  38. _NS_PKG_REL = "http://schemas.openxmlformats.org/package/2006/relationships"
  39. class ExtractionError(Exception):
  40. """Raised when a supported-looking document cannot be rendered as text."""
  41. def _extension(path: str) -> str:
  42. ext = Path(path).suffix.lower()
  43. known = ext in EXTRACTABLE_EXTENSIONS or (ext in ANYDOC_EXTENSIONS and _anydoc() is not None)
  44. return ext if known else ""
  45. _ANYDOC_UNSET = object()
  46. _anydoc_module: Any = _ANYDOC_UNSET
  47. _anydoc_lock = threading.Lock()
  48. # Cooldown after a failed load: the attempt can shell out to pip, so retrying every call would
  49. # hammer the network where install can't succeed.
  50. ANYDOC_RETRY_SECONDS = 300.0
  51. _anydoc_failed_at: Optional[float] = None
  52. def _anydoc() -> Optional[Any]:
  53. """Lazily import the optional anydoc converter (None when unavailable; failures retried after
  54. ANYDOC_RETRY_SECONDS so one transient pip/network blip does not stick)."""
  55. global _anydoc_module, _anydoc_failed_at
  56. if _anydoc_module is not _ANYDOC_UNSET:
  57. return _anydoc_module
  58. with _anydoc_lock:
  59. if _anydoc_module is not _ANYDOC_UNSET:
  60. return _anydoc_module
  61. if (_anydoc_failed_at is not None
  62. and time.monotonic() - _anydoc_failed_at < ANYDOC_RETRY_SECONDS):
  63. return None
  64. try:
  65. from pm import ensure_import
  66. # read_file must never block on an install prompt.
  67. ensure_import("doc-extract")
  68. except Exception:
  69. _anydoc_failed_at = time.monotonic()
  70. return None
  71. try: _anydoc_module = importlib.import_module("anydoc")
  72. except Exception: # install failure, ImportError or a broken native binding
  73. _anydoc_failed_at = time.monotonic()
  74. return None
  75. _anydoc_failed_at = None
  76. return _anydoc_module # type: ignore[return-value]
  77. def is_extractable_document(path: str) -> bool:
  78. return bool(_extension(path))
  79. def _check_size(size: int, limit: int) -> None:
  80. if size > limit:
  81. raise ExtractionError(f"Document too large to convert ({size:,} bytes, limit is {limit:,})")
  82. @contextlib.contextmanager
  83. def _temp_copy(data: bytes, suffix: str) -> Iterator[str]:
  84. """Materialize backend bytes in a private host temp file; removed even when parsing fails."""
  85. with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh:
  86. fh.write(data)
  87. try:
  88. yield fh.name
  89. finally:
  90. with contextlib.suppress(OSError):
  91. os.unlink(fh.name)
  92. def extract_document_text(path: str) -> str:
  93. ext = _extension(path)
  94. if ext in _STDLIB_EXTRACTORS:
  95. return _STDLIB_EXTRACTORS[ext](path)
  96. if ext in ANYDOC_EXTENSIONS:
  97. return _extract_anydoc(path)
  98. raise ExtractionError(f"Unsupported document type: {path!r}")
  99. def extract_document_bytes(data: bytes, path: str) -> str:
  100. """Extract a document already fetched across a file backend boundary."""
  101. _check_size(len(data), MAX_DOCUMENT_BYTES)
  102. ext = _extension(path)
  103. if ext in ANYDOC_EXTENSIONS:
  104. return _extract_anydoc_bytes(data, path)
  105. if ext not in EXTRACTABLE_EXTENSIONS:
  106. raise ExtractionError(f"Unsupported document type: {path!r}")
  107. with _temp_copy(data, ext) as temp_path: # the stdlib extractors are path-oriented
  108. if ext == ".ipynb":
  109. return _extract_notebook(temp_path, display_path=path)
  110. return _STDLIB_EXTRACTORS[ext](temp_path)
  111. def _anydoc_missing_error(path: str) -> str:
  112. """Teaching text for anydoc-gated formats (not in the schema: only sessions hitting one pay).
  113. Response-time hint (#95681 pattern): the schema no longer lists the anydoc-gated formats or the
  114. availability caveat — a session that never touches a .doc/.odt/.epub never pays for the explanation, and
  115. one that does gets the full story here, with the fix.
  116. """
  117. return (
  118. f"Cannot convert {path!r}: this format needs the optional anydoc "
  119. "converter, which is not installed (install blocked or first "
  120. "attempt failed; retried every 5 minutes). Run `hermes pm repair` "
  121. "to restore firecrawl-anydoc, or convert the file "
  122. "yourself via terminal (e.g. libreoffice --headless --convert-to "
  123. "txt).")
  124. def _hosted_ocr_config() -> tuple:
  125. """(enabled, api_key, api_url); never raises, no network. Maintainer decision: the ONLY route
  126. is a direct ``FIRECRAWL_API_KEY`` (anydoc defaults api_url); the Nous gateway's Parse proxy
  127. live-probed broken, so it is NOT used. ``file_tools.hosted_ocr: false`` disables even with a
  128. key. The key is a profile credential: read through the secret scope so a multiplexed
  129. secondary never spends (or reveals its documents to) the default profile's Firecrawl key."""
  130. from agent.secret_scope import get_secret
  131. api_key = get_secret("FIRECRAWL_API_KEY") or None
  132. enabled = api_key is not None
  133. with contextlib.suppress(Exception):
  134. from hermes_cli.config import load_config_readonly
  135. section = load_config_readonly().get("file_tools")
  136. if isinstance(section, dict) and section.get("hosted_ocr") is False:
  137. enabled = False
  138. return enabled, api_key, None
  139. def hosted_ocr_available() -> bool:
  140. """Probe for read_file's schema line; a key failing at conversion time surfaces in NEEDS-OCR."""
  141. return _hosted_ocr_config()[0]
  142. def _needs_ocr_warning(path: str, pages, hosted_error: str = "") -> str:
  143. """NeedsOcrError result when hosted OCR is off/failed; hints at CHECKING for an OCR skill
  144. (never names one) and never advertises the hosted_ocr knob."""
  145. page_list = ", ".join(str(p) for p in pages) if pages else "unknown"
  146. hosted = f"Hosted OCR was attempted and failed ({hosted_error}). " if hosted_error else ""
  147. return (
  148. f"[NEEDS OCR: pages {page_list} of this PDF are scanned images "
  149. f"with no text layer — their content is MISSING below. {hosted}"
  150. "If the missing pages matter: render just those pages with "
  151. f"`pdftoppm -jpeg -r 150 -f <first> -l <last> {shlex.quote(path)} $TMPDIR/page` "
  152. "and inspect via vision_analyze, or check whether an OCR skill is "
  153. "available (skills_list).]\n")
  154. def _finalize_anydoc_text(text: Any, path: str, pdf_note: Callable[[], str]) -> str:
  155. """Normalize converter output; PDFs get the coverage note PREPENDED (read_file paginates, so a
  156. footer may never be fetched) — this covers PARTIAL scan gaps that raise no NeedsOcrError."""
  157. if not isinstance(text, str) or not text.strip():
  158. raise ExtractionError("Document contains no extractable text")
  159. return (pdf_note() if Path(path).suffix.lower() == ".pdf" else "") + text.rstrip("\n") + "\n"
  160. def _ocr_scanned_pdf(mod: Any, path: str, exc: BaseException) -> str:
  161. """anydoc >= 0.2 scanned-pages signal: hosted OCR when a route exists, else teach recovery."""
  162. pages = list(getattr(exc, "pages", []) or [])
  163. enabled, api_key, api_url = _hosted_ocr_config()
  164. hosted_error = ""
  165. if enabled:
  166. try:
  167. extra = {k: v for k, v in (("api_key", api_key), ("api_url", api_url)) if v}
  168. return mod.to_markdown(path, ocr="hosted", **extra).rstrip("\n") + "\n"
  169. except Exception as hosted_exc:
  170. hosted_error = f"{type(hosted_exc).__name__}: {hosted_exc}"
  171. return _needs_ocr_warning(path, pages, hosted_error) # whole doc is scans: the warning IS it
  172. def _extract_anydoc(path: str) -> str:
  173. mod = _anydoc()
  174. if mod is None:
  175. raise ExtractionError(_anydoc_missing_error(path))
  176. try:
  177. _check_size(os.path.getsize(path), MAX_ANYDOC_BYTES)
  178. text = mod.to_markdown(path)
  179. except ExtractionError:
  180. raise
  181. except OSError as exc:
  182. raise ExtractionError(str(exc)) from exc
  183. except Exception as exc:
  184. needs_ocr = getattr(mod, "NeedsOcrError", None)
  185. if needs_ocr is not None and isinstance(exc, needs_ocr):
  186. return _ocr_scanned_pdf(mod, path, exc)
  187. # Any ConvertError subclass (Unsupported/Malformed/Encrypted/...) = "no meaningful text".
  188. raise ExtractionError(f"{type(exc).__name__}: {exc}") from exc
  189. return _finalize_anydoc_text(text, path, lambda: _pdf_coverage_note(path))
  190. def _extract_anydoc_bytes(data: bytes, path: str) -> str:
  191. mod = _anydoc()
  192. if mod is None:
  193. raise ExtractionError(_anydoc_missing_error(path))
  194. _check_size(len(data), MAX_ANYDOC_BYTES)
  195. try:
  196. text = mod.to_markdown_bytes(data)
  197. except Exception as exc:
  198. raise ExtractionError(f"{type(exc).__name__}: {exc}") from exc
  199. return _finalize_anydoc_text(text, path, lambda: _pdf_coverage_note_from_bytes(data, path))
  200. # ── Scanned-PDF coverage: text-layer extractors return nothing for scanned pages, so a mostly
  201. # scanned PDF converts "successfully" into silent data loss. Count per-page text via pdftotext.
  202. PDF_EMPTY_PAGE_CHARS = 20 # fewer extracted chars than this = empty page
  203. # Warn when empty pages reach both MIN_EMPTY and MIN_RATIO, or ABSOLUTE_EMPTY alone.
  204. PDF_COVERAGE_MIN_EMPTY, PDF_COVERAGE_MIN_RATIO, PDF_COVERAGE_ABSOLUTE_EMPTY = 2, 0.2, 10
  205. PDF_PAGE_SCAN_TIMEOUT = 20.0
  206. PDF_GAP_MAP_MAX_ENTRIES = 20 # cap so alternating text/scan pages can't balloon the warning
  207. _GAP_CONTEXT_CHARS = 60
  208. def _pdf_page_texts(path: str) -> Optional[list[str]]:
  209. """Per-page extracted text, or None when undeterminable."""
  210. if shutil.which("pdftotext") is None:
  211. return None
  212. try:
  213. proc = subprocess.run(
  214. ["pdftotext", path, "-"], stdin=subprocess.DEVNULL, capture_output=True, timeout=PDF_PAGE_SCAN_TIMEOUT, check=False)
  215. except (OSError, subprocess.SubprocessError):
  216. return None
  217. out = proc.stdout.decode("utf-8", errors="replace") if proc.returncode == 0 else ""
  218. pages = out.split("\f") if out else []
  219. if pages and not pages[-1].strip():
  220. pages.pop() # trailing form-feed artifact
  221. return pages or None
  222. def _gap_map(counts: list[int], texts: list[str], empty: list[int]) -> str:
  223. """Per-gap breakdown labeled with the text before each gap, so the agent picks which to OCR."""
  224. # Sorted 1-based page numbers -> (start, end) runs; consecutive pages share ``page - index``.
  225. runs = [list(g) for _k, g in itertools.groupby(enumerate(empty), lambda e: e[1] - e[0])]
  226. ranges = [(run[0][1], run[-1][1]) for run in runs]
  227. lines: list[str] = []
  228. for a, b in ranges[:PDF_GAP_MAP_MAX_ENTRIES]:
  229. label = ""
  230. for prev in range(a - 2, -1, -1): # nearest preceding page with text
  231. if counts[prev] >= PDF_EMPTY_PAGE_CHARS:
  232. snippet = " ".join(texts[prev].split())[:_GAP_CONTEXT_CHARS]
  233. label = f' — after "{snippet}" (p{prev + 1})'
  234. break
  235. span = f"page {a}" if a == b else f"pages {a}-{b}"
  236. n = b - a + 1
  237. lines.append(f" {span} ({n} page{'s' if n != 1 else ''}){label}")
  238. if len(ranges) > PDF_GAP_MAP_MAX_ENTRIES:
  239. rest = ranges[PDF_GAP_MAP_MAX_ENTRIES:]
  240. lines.append(f" … {len(rest)} more gaps ({sum(b - a + 1 for a, b in rest)} pages)")
  241. return "\n".join(lines)
  242. def _pdf_coverage_note(path: str, display_path: Optional[str] = None) -> str:
  243. """Warning header when many pages yielded no text, else ''. ``display_path`` (default ``path``,
  244. which may be a host temp file) is what the recovery command shows."""
  245. texts = _pdf_page_texts(path)
  246. if not texts or len(texts) < 2:
  247. return ""
  248. counts = [len(page.strip()) for page in texts]
  249. empty = [i + 1 for i, n in enumerate(counts) if n < PDF_EMPTY_PAGE_CHARS]
  250. total = len(counts)
  251. n_empty = len(empty)
  252. enough = n_empty / total >= PDF_COVERAGE_MIN_RATIO or n_empty >= PDF_COVERAGE_ABSOLUTE_EMPTY
  253. if n_empty < PDF_COVERAGE_MIN_EMPTY or not enough:
  254. return ""
  255. shown = display_path or path
  256. return (
  257. "[EXTRACTION COVERAGE WARNING: "
  258. f"{len(empty)} of {total} pages in this PDF yielded no text. "
  259. "Those pages are likely scanned images (or blank) — their content "
  260. "is MISSING from the extracted text below, even where section "
  261. "headers appear with empty bodies. Unreadable gaps, each labeled "
  262. "with the last text extracted before it:\n"
  263. f"{_gap_map(counts, texts, empty)}\n"
  264. "Decide which gaps you actually need — do NOT OCR or render "
  265. "everything. For the gaps that matter, render just that range with "
  266. f"`pdftoppm -jpeg -r 150 -f <first> -l <last> {shlex.quote(shown)} $TMPDIR/page` "
  267. "and inspect each image with the vision_analyze tool, or use the "
  268. "ocr-and-documents skill (marker-pdf) for bulk OCR of large "
  269. "ranges.]\n")
  270. def _pdf_coverage_note_from_bytes(data: bytes, display_path: str) -> str:
  271. """Coverage note for backend PDF bytes via a host temp copy (pdftotext needs a path)."""
  272. with contextlib.suppress(OSError), _temp_copy(data, ".pdf") as temp_path:
  273. return _pdf_coverage_note(temp_path, display_path=display_path)
  274. return ""
  275. def _joined(lines: list[str], empty_error: str) -> str:
  276. """Join extracted lines with a single trailing newline; raise when nothing non-blank."""
  277. if not any(line.strip() for line in lines):
  278. raise ExtractionError(empty_error)
  279. return "\n".join(lines).rstrip("\n") + "\n"
  280. def _source_text(source) -> str:
  281. """Notebook source/text fields are a str or a list of str fragments."""
  282. if isinstance(source, list):
  283. source = "".join(item for item in source if isinstance(item, str))
  284. return source if isinstance(source, str) else ""
  285. def _human_size(n_bytes: int) -> str:
  286. return f"{round(n_bytes / 1024)} KB" if n_bytes >= 1024 else f"{n_bytes} B"
  287. def _base64_bytes(payload: str) -> int:
  288. """Approximate decoded size of a base64 payload (whitespace ignored)."""
  289. clean = re.sub(r"[^0-9+/=A-Za-z]", "", payload)
  290. return max(0, (len(clean) * 3) // 4 - min(2, len(clean) - len(clean.rstrip("="))))
  291. def _clean_stream_text(text: str) -> str:
  292. """Strip ANSI escapes; keep only the final ``\\r`` frame of each line (tqdm redraws)."""
  293. from tools.ansi_strip import strip_ansi
  294. return "\n".join(([f for f in line.split("\r") if f] or [""])[-1]
  295. for line in strip_ansi(text).replace("\r\n", "\n").split("\n"))
  296. _MAX_OUTPUT_CHARS = 20_000 # per code cell, so one runaway training log cannot flood the extraction
  297. # nbformat v3 stores mime data flat on the output dict under these keys.
  298. _V3_MIME_KEYS = (("png", "image/png"), ("jpeg", "image/jpeg"), ("svg", "image/svg+xml"), ("html", "text/html"))
  299. def _notebook_output_text(output: Any) -> str:
  300. """One notebook output as compact text: stream/traceback/textual results kept; token-heavy
  301. payloads (images, HTML, widgets) become sized placeholders. Handles v4 and legacy v3 shapes."""
  302. if not isinstance(output, dict):
  303. return ""
  304. otype = output.get("output_type")
  305. if otype == "stream":
  306. body = _clean_stream_text(_source_text(output.get("text", "")))
  307. return body if body.strip() else ""
  308. if otype in {"error", "pyerr"}:
  309. tb = output.get("traceback")
  310. tb_text = _clean_stream_text("\n".join(filter(lambda l: isinstance(l, str), tb))
  311. if isinstance(tb, list) else "")
  312. header = f"Error: {output.get('ename', '')}: {output.get('evalue', '')}".rstrip(": ")
  313. return f"{header}\n{tb_text}".rstrip()
  314. if otype not in {"execute_result", "display_data", "pyout"}:
  315. return ""
  316. data = output.get("data")
  317. if not isinstance(data, dict): # legacy v3: mime payloads sit flat on the output dict
  318. data = {"text/plain": output["text"]} if isinstance(output.get("text"), (str, list)) else {}
  319. data.update((mime, output[k]) for k, mime in _V3_MIME_KEYS if k in output)
  320. if "application/vnd.jupyter.widget-view+json" in data:
  321. return "[interactive widget — omitted]"
  322. for mime in ("text/plain", "text/markdown"): # models consume text far better than markup
  323. body = _clean_stream_text(_source_text(data[mime])) if mime in data else ""
  324. if body.strip():
  325. return body
  326. for mime, value in data.items():
  327. if isinstance(mime, str) and mime.startswith("image/"):
  328. return f"[{mime} output — {_human_size(_base64_bytes(_source_text(value)))}, omitted]"
  329. if "text/html" in data:
  330. return f"[text/html output — {len(_source_text(data['text/html'])):,} chars, omitted]"
  331. return f"[{', '.join(str(m) for m in data) or 'unknown'} output — omitted]"
  332. def _notebook_outputs(cell: dict, jq_pointer: str = "", filename: str = "") -> str:
  333. outputs = cell.get("outputs")
  334. if not isinstance(outputs, list):
  335. return ""
  336. joined = "\n".join(filter(None, map(_notebook_output_text, outputs)))
  337. if len(joined) <= _MAX_OUTPUT_CHARS:
  338. return joined
  339. hint = f" — full output: jq -r '{jq_pointer}' {shlex.quote(filename)}" if jq_pointer and filename else ""
  340. omitted = len(joined) - _MAX_OUTPUT_CHARS
  341. return joined[:_MAX_OUTPUT_CHARS] + f"\n… [{omitted:,} output chars truncated{hint}]"
  342. _CELL_LABELS = {"markdown": "Markdown", "code": "Code", "raw": "Raw"}
  343. def _extract_notebook(path: str, *, display_path: Optional[str] = None) -> str:
  344. try:
  345. with open(path, encoding="utf-8-sig", errors="replace") as fh:
  346. nb = json.load(fh)
  347. except (OSError, ValueError, json.JSONDecodeError) as exc:
  348. raise ExtractionError(f"Not a valid notebook: {exc}") from exc
  349. if not isinstance(nb, dict):
  350. raise ExtractionError("Notebook root is not an object")
  351. raw_cells = nb.get("cells")
  352. if isinstance(raw_cells, list):
  353. cells = [(f".cells[{i}].outputs", cell) for i, cell in enumerate(raw_cells)]
  354. else: # nbformat v3: cells live under worksheets
  355. cells = [
  356. (f".worksheets[{wi}].cells[{ci}].outputs", cell)
  357. for wi, ws in enumerate(nb.get("worksheets", [])) if isinstance(ws, dict)
  358. for ci, cell in enumerate(ws.get("cells", []))]
  359. if not cells:
  360. raise ExtractionError("Notebook contains no cells")
  361. # Backend bytes are parsed through a disposable host copy; recovery commands
  362. # must instead name the original notebook in the user's filesystem.
  363. nb_name = display_path if display_path is not None else path
  364. counts = dict.fromkeys(_CELL_LABELS, 0)
  365. out: list[str] = []
  366. for jq_pointer, cell in cells:
  367. typ = cell.get("cell_type") if isinstance(cell, dict) else None
  368. if typ not in _CELL_LABELS:
  369. continue
  370. counts[typ] += 1
  371. suffix = f" {counts[typ]}" if typ != "raw" else ""
  372. source = _source_text(cell.get("source", "")).rstrip("\n")
  373. out += [f"# ── {_CELL_LABELS[typ]} cell{suffix} ──", source, ""]
  374. rendered = _notebook_outputs(cell, jq_pointer, nb_name) if typ == "code" else ""
  375. if rendered:
  376. out += [f"# ── Output (cell {counts[typ]}) ──", rendered.rstrip("\n"), ""]
  377. return _joined(out, "Notebook contains no readable cells")
  378. @contextlib.contextmanager
  379. def _open_zip(path: str, kind: str) -> Iterator[zipfile.ZipFile]:
  380. """Open an OOXML package; bad-zip/OS failures (body included) become ExtractionError."""
  381. try:
  382. with zipfile.ZipFile(path) as zf:
  383. yield zf
  384. except (zipfile.BadZipFile, OSError) as exc:
  385. bad_zip = isinstance(exc, zipfile.BadZipFile)
  386. raise ExtractionError(f"Not a valid {kind}: {exc}" if bad_zip else str(exc)) from exc
  387. def _zip_xml(zf: zipfile.ZipFile, name: str, optional: bool = False) -> Any:
  388. """Parse a package part; ``optional`` parts yield an empty element when absent or malformed."""
  389. try:
  390. return ET.fromstring(zf.read(name))
  391. except (KeyError, ET.ParseError) as exc:
  392. if optional:
  393. return ET.Element("missing")
  394. raise ExtractionError(
  395. f"Missing {name}" if isinstance(exc, KeyError) else f"Malformed XML in {name}: {exc}"
  396. ) from exc
  397. def _extract_docx(path: str) -> str:
  398. with _open_zip(path, "DOCX") as zf:
  399. root = _zip_xml(zf, "word/document.xml")
  400. w = f"{{{_NS_W}}}"
  401. breaks = {f"{w}tab": "\t", f"{w}br": "\n", f"{w}cr": "\n"}
  402. lines: list[str] = []
  403. for para in root.iter(f"{w}p"):
  404. # w:rt is the ruby (phonetic) guide over w:rubyBase; it annotates the text, it is not text.
  405. guide = {n for rt in para.iter(f"{w}rt") for n in rt.iter()}
  406. text = "".join(
  407. (n.text or "") if n.tag == f"{w}t" else breaks.get(n.tag, "")
  408. for n in para.iter() if n not in guide)
  409. lines.extend(text.split("\n"))
  410. return _joined(lines, "DOCX contains no extractable text")
  411. def _xlsx_string_text(item: ET.Element, s: str) -> str:
  412. """Base text of an si/is, excluding rPh phonetic annotations."""
  413. return "".join(
  414. (child.text or "") if child.tag == f"{s}t" else child.findtext(f"{s}t") or ""
  415. for child in item if child.tag in {f"{s}t", f"{s}r"})
  416. def _extract_xlsx(path: str) -> str:
  417. s, r, pr = f"{{{_NS_S}}}", f"{{{_NS_REL}}}", f"{{{_NS_PKG_REL}}}"
  418. with _open_zip(path, "XLSX") as zf:
  419. names = set(zf.namelist())
  420. sst = _zip_xml(zf, "xl/sharedStrings.xml", optional=True)
  421. shared = [_xlsx_string_text(item, s) for item in sst.iter(f"{s}si")]
  422. rels_root = _zip_xml(zf, "xl/_rels/workbook.xml.rels", optional=True)
  423. rels = {rel.get("Id", ""): rel.get("Target", "")
  424. for rel in rels_root.iter(f"{pr}Relationship") if rel.get("Id")}
  425. out: list[str] = []
  426. for sheet in _zip_xml(zf, "xl/workbook.xml").iter(f"{s}sheet"):
  427. target = rels.get(sheet.get(f"{r}id", ""), "").lstrip("/")
  428. part = posixpath.normpath(target if target.startswith("xl/") else f"xl/{target}")
  429. if sheet.get("state", "visible") in {"hidden", "veryHidden"} or part not in names:
  430. continue
  431. with contextlib.suppress(ET.ParseError):
  432. rows = _sheet_rows(zf.read(part), shared)
  433. out += [f"# ── Sheet: {sheet.get('name', 'Sheet')} ──",
  434. *(["\t".join(row) for row in rows] or ["(empty)"]), ""]
  435. return _joined(out, "XLSX has no visible sheets with content")
  436. def _col_index(ref: str) -> int:
  437. """0-based column of a cell ref: ``A1`` -> 0, ``AB7`` -> 27 (bijective base-26 letters)."""
  438. idx = functools.reduce(lambda acc, ch: acc * 26 + ord(ch.upper()) - ord("A") + 1,
  439. itertools.takewhile(str.isalpha, ref), 0)
  440. return max(idx - 1, 0)
  441. def _sheet_rows(xml_bytes: bytes, shared: list[str]) -> list[list[str]]:
  442. root = ET.fromstring(xml_bytes)
  443. s = f"{{{_NS_S}}}"
  444. rows: list[list[str]] = []
  445. for row in itertools.islice(root.iter(f"{s}row"), _MAX_XLSX_ROWS_PER_SHEET):
  446. cells: dict[int, str] = {}
  447. max_col = -1
  448. for cell in row.iter(f"{s}c"):
  449. col = _col_index(cell.get("r", "")) if cell.get("r") else max_col + 1
  450. if col < _MAX_XLSX_COLS:
  451. cells[col] = _cell_value(cell, shared, s)
  452. max_col = max(max_col, col)
  453. rows.append([cells.get(i, "") for i in range(max_col + 1)])
  454. while rows and not any(value.strip() for value in rows[-1]):
  455. rows.pop()
  456. return rows
  457. def _cell_value(cell: ET.Element, shared: list[str], s: str) -> str:
  458. value = cell.findtext(f"{s}v") or ""
  459. typ = cell.get("t", "")
  460. if typ == "s":
  461. try:
  462. return shared[int(value)]
  463. except (ValueError, IndexError):
  464. return ""
  465. if typ == "inlineStr":
  466. inline = cell.find(f"{s}is")
  467. return "" if inline is None else _xlsx_string_text(inline, s)
  468. if typ == "b":
  469. return "TRUE" if value.strip() in {"1", "true", "TRUE"} else "FALSE"
  470. return (value or "#ERROR") if typ == "e" else value
  471. _SQLITE_MAGIC = b"SQLite format 3\x00"
  472. _SQLITE_PREVIEW_ROWS = 5
  473. _SQLITE_MAX_TABLES = 200
  474. _SQLITE_CELL_CHARS = 80
  475. def _extract_sqlite(path: str) -> str:
  476. """Render a SQLite file as a schema overview: per table the CREATE statement, row count and the
  477. first rows. A ``.db`` that is not SQLite raises ExtractionError so read_file reports the real type."""
  478. import sqlite3
  479. with open(path, "rb") as fh:
  480. if fh.read(len(_SQLITE_MAGIC)) != _SQLITE_MAGIC:
  481. raise ExtractionError("not a SQLite database (magic bytes do not match)")
  482. # Read-only URI: never create or mutate; immutable=1 also skips WAL/journal sidecars so a live
  483. # database that another process has open is still readable without taking locks.
  484. uri = Path(path).resolve().as_uri() + "?mode=ro&immutable=1"
  485. con = sqlite3.connect(uri, uri=True)
  486. try:
  487. objs = con.execute(
  488. "SELECT type, name, sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY type DESC, name"
  489. ).fetchall()
  490. out = ["# SQLite database", ""]
  491. tables = [o for o in objs if o[0] == "table"]
  492. others = [o for o in objs if o[0] != "table"]
  493. out.append(f"{len(tables)} table(s), {len(others)} index/view/trigger(s)")
  494. for _, name, sql in tables[:_SQLITE_MAX_TABLES]:
  495. quoted = '"' + name.replace('"', '""') + '"'
  496. count = con.execute(f"SELECT COUNT(*) FROM {quoted}").fetchone()[0]
  497. out += ["", f"## {name} ({count:,} rows)", sql.strip()]
  498. cur = con.execute(f"SELECT * FROM {quoted} LIMIT {_SQLITE_PREVIEW_ROWS}")
  499. cols = [d[0] for d in cur.description]
  500. rows = cur.fetchall()
  501. if rows:
  502. out.append("| " + " | ".join(cols) + " |")
  503. out.append("|" + "---|" * len(cols))
  504. for row in rows:
  505. cells = [_sqlite_cell(v) for v in row]
  506. out.append("| " + " | ".join(cells) + " |")
  507. if len(tables) > _SQLITE_MAX_TABLES:
  508. out.append(f"\n... {len(tables) - _SQLITE_MAX_TABLES} more tables omitted")
  509. if others:
  510. out += ["", "## Indexes / views / triggers"] + [f"- {t} {n}" for t, n, _ in others]
  511. out += ["", "Query it with the terminal: sqlite3 <path> 'SELECT ...' (or Python's sqlite3 module)."]
  512. return "\n".join(out)
  513. except sqlite3.DatabaseError as exc:
  514. raise ExtractionError(f"SQLite read failed: {exc}") from exc
  515. finally:
  516. con.close()
  517. def _sqlite_cell(value: Any) -> str:
  518. if value is None:
  519. return "NULL"
  520. if isinstance(value, (bytes, bytearray)):
  521. return f"<blob {len(value)} bytes>"
  522. text = str(value).replace("|", "\\|").replace("\n", " ")
  523. return text if len(text) <= _SQLITE_CELL_CHARS else text[:_SQLITE_CELL_CHARS - 1] + "…"
  524. # Extension -> stdlib extractor; anydoc formats fall through in extract_document_text.
  525. _STDLIB_EXTRACTORS: dict[str, Callable[[str], str]] = {
  526. ".ipynb": _extract_notebook, ".docx": _extract_docx, ".xlsx": _extract_xlsx,
  527. ".db": _extract_sqlite, ".sqlite": _extract_sqlite, ".sqlite3": _extract_sqlite}