| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623 |
- """Document-to-text extraction for ``read_file``: stdlib Jupyter/DOCX/XLSX (always
- authoritative for those three), plus legacy Office/OpenDocument/RTF/EPUB/PDF when the
- optional ``firecrawl-anydoc`` package (imports as ``anydoc``) is installed. Malformed
- documents raise :class:`ExtractionError`; callers fall back to text/binary handling."""
- from __future__ import annotations
- import contextlib
- import functools
- import importlib
- import itertools
- import json
- import os
- import posixpath
- import re
- import shlex
- import shutil
- import subprocess
- import tempfile
- import threading
- import time
- import zipfile
- from pathlib import Path
- from typing import Any, Callable, Iterator, Optional
- from xml.etree import ElementTree as ET
- __all__ = ["EXTRACTABLE_EXTENSIONS", "ExtractionError", "extract_document_bytes",
- "extract_document_text", "is_extractable_document"]
- EXTRACTABLE_EXTENSIONS = frozenset({".ipynb", ".docx", ".xlsx", ".db", ".sqlite", ".sqlite3"})
- ANYDOC_EXTENSIONS = frozenset({
- ".doc", ".docm", ".ppt", ".pps", ".pot", ".pptx", ".pptm", ".ppsx", ".ppsm",
- ".xls", ".xlsm", ".xlsb", ".odt", ".ods", ".odp", ".rtf", ".epub", ".pdf"})
- # anydoc loads whole files (no streaming); read_file's char budget applies only post-conversion.
- MAX_ANYDOC_BYTES = 50 * 1024 * 1024
- MAX_DOCUMENT_BYTES = 50 * 1024 * 1024
- _MAX_XLSX_ROWS_PER_SHEET = 5000
- _MAX_XLSX_COLS = 256
- _NS_W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
- _NS_S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
- _NS_REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
- _NS_PKG_REL = "http://schemas.openxmlformats.org/package/2006/relationships"
- class ExtractionError(Exception):
- """Raised when a supported-looking document cannot be rendered as text."""
- def _extension(path: str) -> str:
- ext = Path(path).suffix.lower()
- known = ext in EXTRACTABLE_EXTENSIONS or (ext in ANYDOC_EXTENSIONS and _anydoc() is not None)
- return ext if known else ""
- _ANYDOC_UNSET = object()
- _anydoc_module: Any = _ANYDOC_UNSET
- _anydoc_lock = threading.Lock()
- # Cooldown after a failed load: the attempt can shell out to pip, so retrying every call would
- # hammer the network where install can't succeed.
- ANYDOC_RETRY_SECONDS = 300.0
- _anydoc_failed_at: Optional[float] = None
- def _anydoc() -> Optional[Any]:
- """Lazily import the optional anydoc converter (None when unavailable; failures retried after
- ANYDOC_RETRY_SECONDS so one transient pip/network blip does not stick)."""
- global _anydoc_module, _anydoc_failed_at
- if _anydoc_module is not _ANYDOC_UNSET:
- return _anydoc_module
- with _anydoc_lock:
- if _anydoc_module is not _ANYDOC_UNSET:
- return _anydoc_module
- if (_anydoc_failed_at is not None
- and time.monotonic() - _anydoc_failed_at < ANYDOC_RETRY_SECONDS):
- return None
- try:
- from pm import ensure_import
- # read_file must never block on an install prompt.
- ensure_import("doc-extract")
- except Exception:
- _anydoc_failed_at = time.monotonic()
- return None
- try: _anydoc_module = importlib.import_module("anydoc")
- except Exception: # install failure, ImportError or a broken native binding
- _anydoc_failed_at = time.monotonic()
- return None
- _anydoc_failed_at = None
- return _anydoc_module # type: ignore[return-value]
- def is_extractable_document(path: str) -> bool:
- return bool(_extension(path))
- def _check_size(size: int, limit: int) -> None:
- if size > limit:
- raise ExtractionError(f"Document too large to convert ({size:,} bytes, limit is {limit:,})")
- @contextlib.contextmanager
- def _temp_copy(data: bytes, suffix: str) -> Iterator[str]:
- """Materialize backend bytes in a private host temp file; removed even when parsing fails."""
- with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh:
- fh.write(data)
- try:
- yield fh.name
- finally:
- with contextlib.suppress(OSError):
- os.unlink(fh.name)
- def extract_document_text(path: str) -> str:
- ext = _extension(path)
- if ext in _STDLIB_EXTRACTORS:
- return _STDLIB_EXTRACTORS[ext](path)
- if ext in ANYDOC_EXTENSIONS:
- return _extract_anydoc(path)
- raise ExtractionError(f"Unsupported document type: {path!r}")
- def extract_document_bytes(data: bytes, path: str) -> str:
- """Extract a document already fetched across a file backend boundary."""
- _check_size(len(data), MAX_DOCUMENT_BYTES)
- ext = _extension(path)
- if ext in ANYDOC_EXTENSIONS:
- return _extract_anydoc_bytes(data, path)
- if ext not in EXTRACTABLE_EXTENSIONS:
- raise ExtractionError(f"Unsupported document type: {path!r}")
- with _temp_copy(data, ext) as temp_path: # the stdlib extractors are path-oriented
- if ext == ".ipynb":
- return _extract_notebook(temp_path, display_path=path)
- return _STDLIB_EXTRACTORS[ext](temp_path)
- def _anydoc_missing_error(path: str) -> str:
- """Teaching text for anydoc-gated formats (not in the schema: only sessions hitting one pay).
- Response-time hint (#95681 pattern): the schema no longer lists the anydoc-gated formats or the
- availability caveat — a session that never touches a .doc/.odt/.epub never pays for the explanation, and
- one that does gets the full story here, with the fix.
- """
- return (
- f"Cannot convert {path!r}: this format needs the optional anydoc "
- "converter, which is not installed (install blocked or first "
- "attempt failed; retried every 5 minutes). Run `hermes pm repair` "
- "to restore firecrawl-anydoc, or convert the file "
- "yourself via terminal (e.g. libreoffice --headless --convert-to "
- "txt).")
- def _hosted_ocr_config() -> tuple:
- """(enabled, api_key, api_url); never raises, no network. Maintainer decision: the ONLY route
- is a direct ``FIRECRAWL_API_KEY`` (anydoc defaults api_url); the Nous gateway's Parse proxy
- live-probed broken, so it is NOT used. ``file_tools.hosted_ocr: false`` disables even with a
- key. The key is a profile credential: read through the secret scope so a multiplexed
- secondary never spends (or reveals its documents to) the default profile's Firecrawl key."""
- from agent.secret_scope import get_secret
- api_key = get_secret("FIRECRAWL_API_KEY") or None
- enabled = api_key is not None
- with contextlib.suppress(Exception):
- from hermes_cli.config import load_config_readonly
- section = load_config_readonly().get("file_tools")
- if isinstance(section, dict) and section.get("hosted_ocr") is False:
- enabled = False
- return enabled, api_key, None
- def hosted_ocr_available() -> bool:
- """Probe for read_file's schema line; a key failing at conversion time surfaces in NEEDS-OCR."""
- return _hosted_ocr_config()[0]
- def _needs_ocr_warning(path: str, pages, hosted_error: str = "") -> str:
- """NeedsOcrError result when hosted OCR is off/failed; hints at CHECKING for an OCR skill
- (never names one) and never advertises the hosted_ocr knob."""
- page_list = ", ".join(str(p) for p in pages) if pages else "unknown"
- hosted = f"Hosted OCR was attempted and failed ({hosted_error}). " if hosted_error else ""
- return (
- f"[NEEDS OCR: pages {page_list} of this PDF are scanned images "
- f"with no text layer — their content is MISSING below. {hosted}"
- "If the missing pages matter: render just those pages with "
- f"`pdftoppm -jpeg -r 150 -f <first> -l <last> {shlex.quote(path)} $TMPDIR/page` "
- "and inspect via vision_analyze, or check whether an OCR skill is "
- "available (skills_list).]\n")
- def _finalize_anydoc_text(text: Any, path: str, pdf_note: Callable[[], str]) -> str:
- """Normalize converter output; PDFs get the coverage note PREPENDED (read_file paginates, so a
- footer may never be fetched) — this covers PARTIAL scan gaps that raise no NeedsOcrError."""
- if not isinstance(text, str) or not text.strip():
- raise ExtractionError("Document contains no extractable text")
- return (pdf_note() if Path(path).suffix.lower() == ".pdf" else "") + text.rstrip("\n") + "\n"
- def _ocr_scanned_pdf(mod: Any, path: str, exc: BaseException) -> str:
- """anydoc >= 0.2 scanned-pages signal: hosted OCR when a route exists, else teach recovery."""
- pages = list(getattr(exc, "pages", []) or [])
- enabled, api_key, api_url = _hosted_ocr_config()
- hosted_error = ""
- if enabled:
- try:
- extra = {k: v for k, v in (("api_key", api_key), ("api_url", api_url)) if v}
- return mod.to_markdown(path, ocr="hosted", **extra).rstrip("\n") + "\n"
- except Exception as hosted_exc:
- hosted_error = f"{type(hosted_exc).__name__}: {hosted_exc}"
- return _needs_ocr_warning(path, pages, hosted_error) # whole doc is scans: the warning IS it
- def _extract_anydoc(path: str) -> str:
- mod = _anydoc()
- if mod is None:
- raise ExtractionError(_anydoc_missing_error(path))
- try:
- _check_size(os.path.getsize(path), MAX_ANYDOC_BYTES)
- text = mod.to_markdown(path)
- except ExtractionError:
- raise
- except OSError as exc:
- raise ExtractionError(str(exc)) from exc
- except Exception as exc:
- needs_ocr = getattr(mod, "NeedsOcrError", None)
- if needs_ocr is not None and isinstance(exc, needs_ocr):
- return _ocr_scanned_pdf(mod, path, exc)
- # Any ConvertError subclass (Unsupported/Malformed/Encrypted/...) = "no meaningful text".
- raise ExtractionError(f"{type(exc).__name__}: {exc}") from exc
- return _finalize_anydoc_text(text, path, lambda: _pdf_coverage_note(path))
- def _extract_anydoc_bytes(data: bytes, path: str) -> str:
- mod = _anydoc()
- if mod is None:
- raise ExtractionError(_anydoc_missing_error(path))
- _check_size(len(data), MAX_ANYDOC_BYTES)
- try:
- text = mod.to_markdown_bytes(data)
- except Exception as exc:
- raise ExtractionError(f"{type(exc).__name__}: {exc}") from exc
- return _finalize_anydoc_text(text, path, lambda: _pdf_coverage_note_from_bytes(data, path))
- # ── Scanned-PDF coverage: text-layer extractors return nothing for scanned pages, so a mostly
- # scanned PDF converts "successfully" into silent data loss. Count per-page text via pdftotext.
- PDF_EMPTY_PAGE_CHARS = 20 # fewer extracted chars than this = empty page
- # Warn when empty pages reach both MIN_EMPTY and MIN_RATIO, or ABSOLUTE_EMPTY alone.
- PDF_COVERAGE_MIN_EMPTY, PDF_COVERAGE_MIN_RATIO, PDF_COVERAGE_ABSOLUTE_EMPTY = 2, 0.2, 10
- PDF_PAGE_SCAN_TIMEOUT = 20.0
- PDF_GAP_MAP_MAX_ENTRIES = 20 # cap so alternating text/scan pages can't balloon the warning
- _GAP_CONTEXT_CHARS = 60
- def _pdf_page_texts(path: str) -> Optional[list[str]]:
- """Per-page extracted text, or None when undeterminable."""
- if shutil.which("pdftotext") is None:
- return None
- try:
- proc = subprocess.run(
- ["pdftotext", path, "-"], stdin=subprocess.DEVNULL, capture_output=True, timeout=PDF_PAGE_SCAN_TIMEOUT, check=False)
- except (OSError, subprocess.SubprocessError):
- return None
- out = proc.stdout.decode("utf-8", errors="replace") if proc.returncode == 0 else ""
- pages = out.split("\f") if out else []
- if pages and not pages[-1].strip():
- pages.pop() # trailing form-feed artifact
- return pages or None
- def _gap_map(counts: list[int], texts: list[str], empty: list[int]) -> str:
- """Per-gap breakdown labeled with the text before each gap, so the agent picks which to OCR."""
- # Sorted 1-based page numbers -> (start, end) runs; consecutive pages share ``page - index``.
- runs = [list(g) for _k, g in itertools.groupby(enumerate(empty), lambda e: e[1] - e[0])]
- ranges = [(run[0][1], run[-1][1]) for run in runs]
- lines: list[str] = []
- for a, b in ranges[:PDF_GAP_MAP_MAX_ENTRIES]:
- label = ""
- for prev in range(a - 2, -1, -1): # nearest preceding page with text
- if counts[prev] >= PDF_EMPTY_PAGE_CHARS:
- snippet = " ".join(texts[prev].split())[:_GAP_CONTEXT_CHARS]
- label = f' — after "{snippet}" (p{prev + 1})'
- break
- span = f"page {a}" if a == b else f"pages {a}-{b}"
- n = b - a + 1
- lines.append(f" {span} ({n} page{'s' if n != 1 else ''}){label}")
- if len(ranges) > PDF_GAP_MAP_MAX_ENTRIES:
- rest = ranges[PDF_GAP_MAP_MAX_ENTRIES:]
- lines.append(f" … {len(rest)} more gaps ({sum(b - a + 1 for a, b in rest)} pages)")
- return "\n".join(lines)
- def _pdf_coverage_note(path: str, display_path: Optional[str] = None) -> str:
- """Warning header when many pages yielded no text, else ''. ``display_path`` (default ``path``,
- which may be a host temp file) is what the recovery command shows."""
- texts = _pdf_page_texts(path)
- if not texts or len(texts) < 2:
- return ""
- counts = [len(page.strip()) for page in texts]
- empty = [i + 1 for i, n in enumerate(counts) if n < PDF_EMPTY_PAGE_CHARS]
- total = len(counts)
- n_empty = len(empty)
- enough = n_empty / total >= PDF_COVERAGE_MIN_RATIO or n_empty >= PDF_COVERAGE_ABSOLUTE_EMPTY
- if n_empty < PDF_COVERAGE_MIN_EMPTY or not enough:
- return ""
- shown = display_path or path
- return (
- "[EXTRACTION COVERAGE WARNING: "
- f"{len(empty)} of {total} pages in this PDF yielded no text. "
- "Those pages are likely scanned images (or blank) — their content "
- "is MISSING from the extracted text below, even where section "
- "headers appear with empty bodies. Unreadable gaps, each labeled "
- "with the last text extracted before it:\n"
- f"{_gap_map(counts, texts, empty)}\n"
- "Decide which gaps you actually need — do NOT OCR or render "
- "everything. For the gaps that matter, render just that range with "
- f"`pdftoppm -jpeg -r 150 -f <first> -l <last> {shlex.quote(shown)} $TMPDIR/page` "
- "and inspect each image with the vision_analyze tool, or use the "
- "ocr-and-documents skill (marker-pdf) for bulk OCR of large "
- "ranges.]\n")
- def _pdf_coverage_note_from_bytes(data: bytes, display_path: str) -> str:
- """Coverage note for backend PDF bytes via a host temp copy (pdftotext needs a path)."""
- with contextlib.suppress(OSError), _temp_copy(data, ".pdf") as temp_path:
- return _pdf_coverage_note(temp_path, display_path=display_path)
- return ""
- def _joined(lines: list[str], empty_error: str) -> str:
- """Join extracted lines with a single trailing newline; raise when nothing non-blank."""
- if not any(line.strip() for line in lines):
- raise ExtractionError(empty_error)
- return "\n".join(lines).rstrip("\n") + "\n"
- def _source_text(source) -> str:
- """Notebook source/text fields are a str or a list of str fragments."""
- if isinstance(source, list):
- source = "".join(item for item in source if isinstance(item, str))
- return source if isinstance(source, str) else ""
- def _human_size(n_bytes: int) -> str:
- return f"{round(n_bytes / 1024)} KB" if n_bytes >= 1024 else f"{n_bytes} B"
- def _base64_bytes(payload: str) -> int:
- """Approximate decoded size of a base64 payload (whitespace ignored)."""
- clean = re.sub(r"[^0-9+/=A-Za-z]", "", payload)
- return max(0, (len(clean) * 3) // 4 - min(2, len(clean) - len(clean.rstrip("="))))
- def _clean_stream_text(text: str) -> str:
- """Strip ANSI escapes; keep only the final ``\\r`` frame of each line (tqdm redraws)."""
- from tools.ansi_strip import strip_ansi
- return "\n".join(([f for f in line.split("\r") if f] or [""])[-1]
- for line in strip_ansi(text).replace("\r\n", "\n").split("\n"))
- _MAX_OUTPUT_CHARS = 20_000 # per code cell, so one runaway training log cannot flood the extraction
- # nbformat v3 stores mime data flat on the output dict under these keys.
- _V3_MIME_KEYS = (("png", "image/png"), ("jpeg", "image/jpeg"), ("svg", "image/svg+xml"), ("html", "text/html"))
- def _notebook_output_text(output: Any) -> str:
- """One notebook output as compact text: stream/traceback/textual results kept; token-heavy
- payloads (images, HTML, widgets) become sized placeholders. Handles v4 and legacy v3 shapes."""
- if not isinstance(output, dict):
- return ""
- otype = output.get("output_type")
- if otype == "stream":
- body = _clean_stream_text(_source_text(output.get("text", "")))
- return body if body.strip() else ""
- if otype in {"error", "pyerr"}:
- tb = output.get("traceback")
- tb_text = _clean_stream_text("\n".join(filter(lambda l: isinstance(l, str), tb))
- if isinstance(tb, list) else "")
- header = f"Error: {output.get('ename', '')}: {output.get('evalue', '')}".rstrip(": ")
- return f"{header}\n{tb_text}".rstrip()
- if otype not in {"execute_result", "display_data", "pyout"}:
- return ""
- data = output.get("data")
- if not isinstance(data, dict): # legacy v3: mime payloads sit flat on the output dict
- data = {"text/plain": output["text"]} if isinstance(output.get("text"), (str, list)) else {}
- data.update((mime, output[k]) for k, mime in _V3_MIME_KEYS if k in output)
- if "application/vnd.jupyter.widget-view+json" in data:
- return "[interactive widget — omitted]"
- for mime in ("text/plain", "text/markdown"): # models consume text far better than markup
- body = _clean_stream_text(_source_text(data[mime])) if mime in data else ""
- if body.strip():
- return body
- for mime, value in data.items():
- if isinstance(mime, str) and mime.startswith("image/"):
- return f"[{mime} output — {_human_size(_base64_bytes(_source_text(value)))}, omitted]"
- if "text/html" in data:
- return f"[text/html output — {len(_source_text(data['text/html'])):,} chars, omitted]"
- return f"[{', '.join(str(m) for m in data) or 'unknown'} output — omitted]"
- def _notebook_outputs(cell: dict, jq_pointer: str = "", filename: str = "") -> str:
- outputs = cell.get("outputs")
- if not isinstance(outputs, list):
- return ""
- joined = "\n".join(filter(None, map(_notebook_output_text, outputs)))
- if len(joined) <= _MAX_OUTPUT_CHARS:
- return joined
- hint = f" — full output: jq -r '{jq_pointer}' {shlex.quote(filename)}" if jq_pointer and filename else ""
- omitted = len(joined) - _MAX_OUTPUT_CHARS
- return joined[:_MAX_OUTPUT_CHARS] + f"\n… [{omitted:,} output chars truncated{hint}]"
- _CELL_LABELS = {"markdown": "Markdown", "code": "Code", "raw": "Raw"}
- def _extract_notebook(path: str, *, display_path: Optional[str] = None) -> str:
- try:
- with open(path, encoding="utf-8-sig", errors="replace") as fh:
- nb = json.load(fh)
- except (OSError, ValueError, json.JSONDecodeError) as exc:
- raise ExtractionError(f"Not a valid notebook: {exc}") from exc
- if not isinstance(nb, dict):
- raise ExtractionError("Notebook root is not an object")
- raw_cells = nb.get("cells")
- if isinstance(raw_cells, list):
- cells = [(f".cells[{i}].outputs", cell) for i, cell in enumerate(raw_cells)]
- else: # nbformat v3: cells live under worksheets
- cells = [
- (f".worksheets[{wi}].cells[{ci}].outputs", cell)
- for wi, ws in enumerate(nb.get("worksheets", [])) if isinstance(ws, dict)
- for ci, cell in enumerate(ws.get("cells", []))]
- if not cells:
- raise ExtractionError("Notebook contains no cells")
- # Backend bytes are parsed through a disposable host copy; recovery commands
- # must instead name the original notebook in the user's filesystem.
- nb_name = display_path if display_path is not None else path
- counts = dict.fromkeys(_CELL_LABELS, 0)
- out: list[str] = []
- for jq_pointer, cell in cells:
- typ = cell.get("cell_type") if isinstance(cell, dict) else None
- if typ not in _CELL_LABELS:
- continue
- counts[typ] += 1
- suffix = f" {counts[typ]}" if typ != "raw" else ""
- source = _source_text(cell.get("source", "")).rstrip("\n")
- out += [f"# ── {_CELL_LABELS[typ]} cell{suffix} ──", source, ""]
- rendered = _notebook_outputs(cell, jq_pointer, nb_name) if typ == "code" else ""
- if rendered:
- out += [f"# ── Output (cell {counts[typ]}) ──", rendered.rstrip("\n"), ""]
- return _joined(out, "Notebook contains no readable cells")
- @contextlib.contextmanager
- def _open_zip(path: str, kind: str) -> Iterator[zipfile.ZipFile]:
- """Open an OOXML package; bad-zip/OS failures (body included) become ExtractionError."""
- try:
- with zipfile.ZipFile(path) as zf:
- yield zf
- except (zipfile.BadZipFile, OSError) as exc:
- bad_zip = isinstance(exc, zipfile.BadZipFile)
- raise ExtractionError(f"Not a valid {kind}: {exc}" if bad_zip else str(exc)) from exc
- def _zip_xml(zf: zipfile.ZipFile, name: str, optional: bool = False) -> Any:
- """Parse a package part; ``optional`` parts yield an empty element when absent or malformed."""
- try:
- return ET.fromstring(zf.read(name))
- except (KeyError, ET.ParseError) as exc:
- if optional:
- return ET.Element("missing")
- raise ExtractionError(
- f"Missing {name}" if isinstance(exc, KeyError) else f"Malformed XML in {name}: {exc}"
- ) from exc
- def _extract_docx(path: str) -> str:
- with _open_zip(path, "DOCX") as zf:
- root = _zip_xml(zf, "word/document.xml")
- w = f"{{{_NS_W}}}"
- breaks = {f"{w}tab": "\t", f"{w}br": "\n", f"{w}cr": "\n"}
- lines: list[str] = []
- for para in root.iter(f"{w}p"):
- # w:rt is the ruby (phonetic) guide over w:rubyBase; it annotates the text, it is not text.
- guide = {n for rt in para.iter(f"{w}rt") for n in rt.iter()}
- text = "".join(
- (n.text or "") if n.tag == f"{w}t" else breaks.get(n.tag, "")
- for n in para.iter() if n not in guide)
- lines.extend(text.split("\n"))
- return _joined(lines, "DOCX contains no extractable text")
- def _xlsx_string_text(item: ET.Element, s: str) -> str:
- """Base text of an si/is, excluding rPh phonetic annotations."""
- return "".join(
- (child.text or "") if child.tag == f"{s}t" else child.findtext(f"{s}t") or ""
- for child in item if child.tag in {f"{s}t", f"{s}r"})
- def _extract_xlsx(path: str) -> str:
- s, r, pr = f"{{{_NS_S}}}", f"{{{_NS_REL}}}", f"{{{_NS_PKG_REL}}}"
- with _open_zip(path, "XLSX") as zf:
- names = set(zf.namelist())
- sst = _zip_xml(zf, "xl/sharedStrings.xml", optional=True)
- shared = [_xlsx_string_text(item, s) for item in sst.iter(f"{s}si")]
- rels_root = _zip_xml(zf, "xl/_rels/workbook.xml.rels", optional=True)
- rels = {rel.get("Id", ""): rel.get("Target", "")
- for rel in rels_root.iter(f"{pr}Relationship") if rel.get("Id")}
- out: list[str] = []
- for sheet in _zip_xml(zf, "xl/workbook.xml").iter(f"{s}sheet"):
- target = rels.get(sheet.get(f"{r}id", ""), "").lstrip("/")
- part = posixpath.normpath(target if target.startswith("xl/") else f"xl/{target}")
- if sheet.get("state", "visible") in {"hidden", "veryHidden"} or part not in names:
- continue
- with contextlib.suppress(ET.ParseError):
- rows = _sheet_rows(zf.read(part), shared)
- out += [f"# ── Sheet: {sheet.get('name', 'Sheet')} ──",
- *(["\t".join(row) for row in rows] or ["(empty)"]), ""]
- return _joined(out, "XLSX has no visible sheets with content")
- def _col_index(ref: str) -> int:
- """0-based column of a cell ref: ``A1`` -> 0, ``AB7`` -> 27 (bijective base-26 letters)."""
- idx = functools.reduce(lambda acc, ch: acc * 26 + ord(ch.upper()) - ord("A") + 1,
- itertools.takewhile(str.isalpha, ref), 0)
- return max(idx - 1, 0)
- def _sheet_rows(xml_bytes: bytes, shared: list[str]) -> list[list[str]]:
- root = ET.fromstring(xml_bytes)
- s = f"{{{_NS_S}}}"
- rows: list[list[str]] = []
- for row in itertools.islice(root.iter(f"{s}row"), _MAX_XLSX_ROWS_PER_SHEET):
- cells: dict[int, str] = {}
- max_col = -1
- for cell in row.iter(f"{s}c"):
- col = _col_index(cell.get("r", "")) if cell.get("r") else max_col + 1
- if col < _MAX_XLSX_COLS:
- cells[col] = _cell_value(cell, shared, s)
- max_col = max(max_col, col)
- rows.append([cells.get(i, "") for i in range(max_col + 1)])
- while rows and not any(value.strip() for value in rows[-1]):
- rows.pop()
- return rows
- def _cell_value(cell: ET.Element, shared: list[str], s: str) -> str:
- value = cell.findtext(f"{s}v") or ""
- typ = cell.get("t", "")
- if typ == "s":
- try:
- return shared[int(value)]
- except (ValueError, IndexError):
- return ""
- if typ == "inlineStr":
- inline = cell.find(f"{s}is")
- return "" if inline is None else _xlsx_string_text(inline, s)
- if typ == "b":
- return "TRUE" if value.strip() in {"1", "true", "TRUE"} else "FALSE"
- return (value or "#ERROR") if typ == "e" else value
- _SQLITE_MAGIC = b"SQLite format 3\x00"
- _SQLITE_PREVIEW_ROWS = 5
- _SQLITE_MAX_TABLES = 200
- _SQLITE_CELL_CHARS = 80
- def _extract_sqlite(path: str) -> str:
- """Render a SQLite file as a schema overview: per table the CREATE statement, row count and the
- first rows. A ``.db`` that is not SQLite raises ExtractionError so read_file reports the real type."""
- import sqlite3
- with open(path, "rb") as fh:
- if fh.read(len(_SQLITE_MAGIC)) != _SQLITE_MAGIC:
- raise ExtractionError("not a SQLite database (magic bytes do not match)")
- # Read-only URI: never create or mutate; immutable=1 also skips WAL/journal sidecars so a live
- # database that another process has open is still readable without taking locks.
- uri = Path(path).resolve().as_uri() + "?mode=ro&immutable=1"
- con = sqlite3.connect(uri, uri=True)
- try:
- objs = con.execute(
- "SELECT type, name, sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY type DESC, name"
- ).fetchall()
- out = ["# SQLite database", ""]
- tables = [o for o in objs if o[0] == "table"]
- others = [o for o in objs if o[0] != "table"]
- out.append(f"{len(tables)} table(s), {len(others)} index/view/trigger(s)")
- for _, name, sql in tables[:_SQLITE_MAX_TABLES]:
- quoted = '"' + name.replace('"', '""') + '"'
- count = con.execute(f"SELECT COUNT(*) FROM {quoted}").fetchone()[0]
- out += ["", f"## {name} ({count:,} rows)", sql.strip()]
- cur = con.execute(f"SELECT * FROM {quoted} LIMIT {_SQLITE_PREVIEW_ROWS}")
- cols = [d[0] for d in cur.description]
- rows = cur.fetchall()
- if rows:
- out.append("| " + " | ".join(cols) + " |")
- out.append("|" + "---|" * len(cols))
- for row in rows:
- cells = [_sqlite_cell(v) for v in row]
- out.append("| " + " | ".join(cells) + " |")
- if len(tables) > _SQLITE_MAX_TABLES:
- out.append(f"\n... {len(tables) - _SQLITE_MAX_TABLES} more tables omitted")
- if others:
- out += ["", "## Indexes / views / triggers"] + [f"- {t} {n}" for t, n, _ in others]
- out += ["", "Query it with the terminal: sqlite3 <path> 'SELECT ...' (or Python's sqlite3 module)."]
- return "\n".join(out)
- except sqlite3.DatabaseError as exc:
- raise ExtractionError(f"SQLite read failed: {exc}") from exc
- finally:
- con.close()
- def _sqlite_cell(value: Any) -> str:
- if value is None:
- return "NULL"
- if isinstance(value, (bytes, bytearray)):
- return f"<blob {len(value)} bytes>"
- text = str(value).replace("|", "\\|").replace("\n", " ")
- return text if len(text) <= _SQLITE_CELL_CHARS else text[:_SQLITE_CELL_CHARS - 1] + "…"
- # Extension -> stdlib extractor; anydoc formats fall through in extract_document_text.
- _STDLIB_EXTRACTORS: dict[str, Callable[[str], str]] = {
- ".ipynb": _extract_notebook, ".docx": _extract_docx, ".xlsx": _extract_xlsx,
- ".db": _extract_sqlite, ".sqlite": _extract_sqlite, ".sqlite3": _extract_sqlite}
|