diora-web/tui/diora_tui/blocks.py

"""Whole-book flat block extraction, matching static/js/app.js's EPUB_BLOCK_SELECTOR
and getPositionAnchor()/restoreFromAnchor() exactly in *structure* (blockIndex is
purely DOM order, no rendering needed) so position anchors are comparable across
the web reader and this TUI. See CLAUDE.md's `tui/` section for the full picture.

innerFraction on the web is defined via getBoundingClientRect() pixel geometry,
which has no TUI equivalent — a terminal has no font metrics/reflow the way a
browser does. We approximate it with row-based geometry from Rich's own text
wrapping (see reader.py), which is close enough because the server's "furthest
wins" comparison (_progress_is_further) only falls back to comparing fractions
when two anchors share the exact same block index — block index is the primary,
exactly-reproducible signal.

Footnote references are detected with the same heuristic as app.js's
_looksLikeFootnoteLink: wrapped in/wrapping a <sup>, a class name containing
note/footnote/fn, or an epub:type="noteref" attribute.
"""

from __future__ import annotations

import re
import warnings
from dataclasses import dataclass, field
from pathlib import Path

import ebooklib
from bs4 import BeautifulSoup, NavigableString, Tag, XMLParsedAsHTMLWarning
from bs4.element import PreformattedString
from ebooklib import epub

# EPUB content documents are XHTML; treating them as HTML (matching app.js's
# `new DOMParser().parseFromString(html, 'text/html')`, static/js/app.js:2638)
# is intentional, not a mistake — silence bs4's XML-vs-HTML nudge for it.
# html5lib builds the tree with the HTML5 algorithm, as the browser's DOMParser the web reader
# uses does: same nesting for broken markup (`<h1/>` opens a block) and the same text nodes
# (whitespace included), which the block indexes and the character offsets depend on. lxml's
# HTML parser (libxml2) trims indentation and closes elements differently.
_PARSER = "html5lib"

# Matches app.js's EPUB_BLOCK_SELECTOR = 'p, h1, h2, h3, h4, h5, h6, li,
# blockquote, dt, dd, figcaption, div:not(:has(*))'
_BLOCK_TAGS = {"p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "blockquote", "dt", "dd", "figcaption"}
# Matches app.js's sanitizeEpubHtml() strip list (script/style are also stripped
# via regex before DOMParser even runs there; decomposing here is equivalent).
_STRIP_TAGS = ["script", "style", "iframe", "object", "embed", "head", "meta", "link"]
# Matches app.js's _looksLikeFootnoteLink's class-name check.
_FOOTNOTE_CLASS_RE = re.compile(r"\bnote|\bfootnote|\bfn\b")
# JavaScript's `\s`, not Python's (which also matches \x1c-\x1f and \x85 but not ): a
# block's text is cut and measured the way the web reader does.
_JS_SPACE_RE = re.compile("[ \t\n\v\f\r   - 

  　]+")

# Anchor format the server accepts (books/views.py:save_progress); anything else
# is silently discarded back to ''.
_ANCHOR_RE = re.compile(r"\d{1,7}:\d(\.\d{1,6})?")


@dataclass
class FootnoteRef:
    marker: str  # visible text of the reference link, e.g. "1"
    offset: int  # character offset into the block's text where the marker sits
    target_id: str  # fragment id to resolve against block ids


@dataclass
class Block:
    text: str
    tag: str
    chapter_index: int
    id: str | None = None
    footnotes: list[FootnoteRef] = field(default_factory=list)
    # Every id found anywhere in this block's subtree, not just on the block
    # element itself — footnote *targets* are frequently an <a id="..."> or
    # similar nested a level or two inside the actual containing paragraph
    # (see showFootnotePopover's `.closest('.footnote') || .parentElement`
    # walk-up in app.js), so a target id resolves to "the block containing
    # it" rather than requiring the id to sit on the block tag itself.
    ids: list[str] = field(default_factory=list)
    # Where the block sits in the book's text, counted in characters the way the web reader
    # counts them (44-books.js _bookCharIndex): char_start = offset of the block's first text
    # node among all text nodes of the book, char_len = its textContent length (nested blocks
    # included). Progress as a share of the text (position -> fraction) is computed from these.
    char_start: int = 0
    char_len: int = 0


@dataclass
class FlatBook:
    id: str
    title: str
    author: str
    path: Path
    blocks: list[Block]
    chapter_titles: list[str]
    chapter_start_block: list[int]  # blocks[chapter_start_block[i]] is chapter i's first block
    footnote_targets: dict[str, int]  # fragment id -> block index of its content
    total_chars: int = 1  # all text nodes of the book (never 0: it divides)
    # chapter i's file as the web reader names it (data-epub-src: the OPF's folder + the manifest
    # href): what highlights and bookmarks store as chapterSrc / anchor
    chapter_srcs: list[str] = field(default_factory=list)


def _is_leaf_div(tag: Tag) -> bool:
    return tag.name == "div" and tag.find(True) is None


def _looks_like_footnote_link(el: Tag, href: str) -> bool:
    if "#" not in href:
        return False
    cls = " ".join(el.get("class") or []).lower()
    if _FOOTNOTE_CLASS_RE.search(cls):
        return True
    epub_type = (el.get("epub:type") or "").lower()
    if "noteref" in epub_type:
        return True
    return el.find_parent("sup") is not None or el.find("sup") is not None


def _walk_text(el: Tag, footnotes: list[FootnoteRef], out: list[str]) -> None:
    for child in el.children:
        if isinstance(child, PreformattedString):
            continue  # comments, CDATA, declarations: not text (textContent skips them)
        if isinstance(child, NavigableString):
            out.append(str(child))
        elif isinstance(child, Tag):
            if child.name == "a":
                href = child.get("href") or ""
                if _looks_like_footnote_link(child, href):
                    marker = _JS_SPACE_RE.sub(" ", child.get_text("")).strip(" ")
                    offset = len("".join(out))
                    target_id = href.split("#", 1)[1] if "#" in href else ""
                    footnotes.append(FootnoteRef(marker=marker, offset=offset, target_id=target_id))
                    out.append(marker)
                    continue
            _walk_text(child, footnotes, out)


def _text_nodes(el: Tag):
    for node in el.descendants:
        if isinstance(node, NavigableString) and not isinstance(node, PreformattedString):
            yield node


def _extract_blocks(html: bytes, chapter_index: int) -> tuple[list[Block], int]:
    """The chapter's blocks (char_start relative to the chapter) and its text length in chars."""
    # Decoded as UTF-8 first, as the web reader reads the file (JSZip 'text'); handing html5lib
    # bytes would have it guess an encoding.
    soup = BeautifulSoup(html.decode("utf-8", "replace"), _PARSER)
    for name in _STRIP_TAGS:
        for el in soup.find_all(name):
            el.decompose()

    root = soup.body or soup
    blocks: list[Block] = []
    offset = 0
    for node in root.descendants:
        if isinstance(node, Tag):
            if node.name in _BLOCK_TAGS or _is_leaf_div(node):
                blocks.append(_make_block(node, chapter_index, offset))
        elif isinstance(node, NavigableString) and not isinstance(node, PreformattedString):
            offset += len(node)
    return blocks, offset


def _make_block(el: Tag, chapter_index: int, char_start: int) -> Block:
    footnotes: list[FootnoteRef] = []
    parts: list[str] = []
    _walk_text(el, footnotes, parts)
    text = _JS_SPACE_RE.sub(" ", "".join(parts)).strip(" ")
    ids = [tag_id for tag_id in (el.get("id"), *(d.get("id") for d in el.find_all(True))) if tag_id]
    return Block(
        text=text,
        tag=el.name,
        chapter_index=chapter_index,
        id=el.get("id"),
        footnotes=footnotes,
        ids=ids,
        char_start=char_start,
        char_len=sum(len(n) for n in _text_nodes(el)),
    )


def _opf_dir(path: Path) -> str:
    """The folder of the book's OPF inside the zip, with a trailing slash ('' at the root)."""
    import re
    import zipfile

    try:
        with zipfile.ZipFile(path) as z:
            m = re.search(r'full-path="([^"]+)"', z.read("META-INF/container.xml").decode("utf-8", "replace"))
    except (OSError, KeyError, zipfile.BadZipFile):
        return ""
    return m.group(1).rsplit("/", 1)[0] + "/" if m and "/" in m.group(1) else ""


def load_flat_book(path: Path) -> FlatBook:
    from .epub import _book_id  # reuse the same path-hash id scheme

    raw = epub.read_epub(str(path), options={"ignore_ncx": True})

    title_meta = raw.get_metadata("DC", "title")
    title = title_meta[0][0] if title_meta else path.stem
    author_meta = raw.get_metadata("DC", "creator")
    author = author_meta[0][0] if author_meta else "Unbekannt"

    blocks: list[Block] = []
    chapter_titles: list[str] = []
    chapter_start_block: list[int] = []
    chapter_srcs: list[str] = []
    opf_dir = _opf_dir(path)
    total_chars = 0

    # app.js's parseEpub() includes every spine itemref unconditionally — no
    # `linear` filtering (static/js/app.js:2623-2625) — since footnote/endnote
    # targets are commonly parked in a linear="no" document. Skipping it here
    # would both break footnote-target resolution and shift blockIndex
    # numbering out of sync with the web reader for every block after it.
    for idref, _linear in raw.spine:
        item = raw.get_item_with_id(idref)
        if item is None or item.get_type() != ebooklib.ITEM_DOCUMENT:
            continue
        chapter_index = len(chapter_titles)
        chapter_blocks, chapter_chars = _extract_blocks(item.content, chapter_index)  # raw bytes: EpubHtml.get_content() pretty-prints, adding whitespace
        for b in chapter_blocks:
            b.char_start += total_chars
        total_chars += chapter_chars
        chapter_start_block.append(len(blocks))
        first_text = next((b.text for b in chapter_blocks if b.text), None)
        chapter_titles.append((first_text or item.get_name())[:60])
        chapter_srcs.append(opf_dir + item.file_name)
        blocks.extend(chapter_blocks)

    footnote_targets: dict[str, int] = {}
    for idx, b in enumerate(blocks):
        for tag_id in b.ids:
            footnote_targets.setdefault(tag_id, idx)
        if not b.char_len:  # no text of its own: where the previous block ends (0 for the first), as the web reader
            b.char_start = blocks[idx - 1].char_start + blocks[idx - 1].char_len if idx else 0

    return FlatBook(
        id=_book_id(path),
        title=title,
        author=author,
        path=path,
        blocks=blocks,
        chapter_titles=chapter_titles,
        chapter_start_block=chapter_start_block,
        footnote_targets=footnote_targets,
        total_chars=total_chars or 1,
        chapter_srcs=chapter_srcs,
    )


def fraction_for_anchor(book: FlatBook, block_index: int, inner_fraction: float) -> float:
    """The share of the book's text before the anchored place (0..1), as the web reader's
    _fractionFromAnchor: what other clients show as progress."""
    if not 0 <= block_index < len(book.blocks):
        return 0.0
    b = book.blocks[block_index]
    return max(0.0, min(1.0, (b.char_start + inner_fraction * b.char_len) / book.total_chars))


def format_anchor(block_index: int, inner_fraction: float) -> str:
    inner_fraction = max(0.0, min(1.0, inner_fraction))
    return f"{block_index}:{inner_fraction:.6f}"


def parse_anchor(anchor: str) -> tuple[int, float] | None:
    if not anchor or not _ANCHOR_RE.fullmatch(anchor):
        return None
    block_str, _, frac_str = anchor.partition(":")
    return int(block_str), float(frac_str)


def anchor_is_further(new_anchor: str, old_anchor: str) -> bool:
    """Mirrors _progress_is_further / _cmpProgress (books/views.py, app.js)."""
    new_parts = parse_anchor(new_anchor)
    old_parts = parse_anchor(old_anchor)
    if new_parts is None or old_parts is None:
        return bool(new_anchor) and not old_anchor
    nb, ni = new_parts
    ob, oi = old_parts
    return ni >= oi if nb == ob else nb >= ob


def book_pages(path: Path) -> int:
    """The book's length in printed pages from its text alone (bookPagesOfFile, 40-epub.js): every
    spine document's body text by 1800 characters a page. Cheaper than load_flat_book: no blocks."""
    from .library import pages_of_chars

    raw = epub.read_epub(str(path), options={"ignore_ncx": True})
    chars = 0
    for idref, _linear in raw.spine:
        item = raw.get_item_with_id(idref)
        if item is None or item.get_type() != ebooklib.ITEM_DOCUMENT:
            continue
        soup = BeautifulSoup(item.content.decode("utf-8", "replace"), _PARSER)
        for name in ("script", "style"):
            for el in soup.find_all(name):
                el.decompose()
        chars += sum(len(n) for n in _text_nodes(soup.body or soup))
    return pages_of_chars(chars)


# ---- raw text of blocks (for highlights: the web stores character offsets into a block's raw text)

def raw_block_texts(path: Path, chapter_index: int) -> list[str]:
    """The raw text (all text nodes, whitespace as in the file) of every block of one spine
    document, in block order: what the web's _charOffsetInBlock counts in. `chapter_index` is the
    index load_flat_book gave the document."""
    raw = epub.read_epub(str(path), options={"ignore_ncx": True})
    seen = -1
    for idref, _linear in raw.spine:
        item = raw.get_item_with_id(idref)
        if item is None or item.get_type() != ebooklib.ITEM_DOCUMENT:
            continue
        seen += 1
        if seen != chapter_index:
            continue
        soup = BeautifulSoup(item.content.decode("utf-8", "replace"), _PARSER)
        for name in _STRIP_TAGS:
            for el in soup.find_all(name):
                el.decompose()
        root = soup.body or soup
        return ["".join(str(n) for n in _text_nodes(el)) for el in root.descendants
                if isinstance(el, Tag) and (el.name in _BLOCK_TAGS or _is_leaf_div(el))]
    return []


def collapse_map(raw: str) -> tuple[list[int], list[int], int]:
    """How a block's raw text becomes its displayed text (whitespace runs to one space, the ends
    cut). Returns (norm_of_raw, raw_of_norm, length):
    norm_of_raw[i] = offset in the displayed text of raw offset i (0..len(raw));
    raw_of_norm[k] = raw offset of the character that is displayed character k."""
    norm_of_raw: list[int] = []
    raw_of_norm: list[int] = []
    n = 0
    started = False
    prev_space = False
    for i, ch in enumerate(raw):
        norm_of_raw.append(n)
        if _JS_SPACE_RE.fullmatch(ch):
            if started and not prev_space:
                raw_of_norm.append(i)
                n += 1
            prev_space = True
        else:
            started, prev_space = True, False
            raw_of_norm.append(i)
            n += 1
    norm_of_raw.append(n)
    length = n - 1 if prev_space and started else (n if started else 0)
    if prev_space and started:
        raw_of_norm.pop()
    return [min(x, length) for x in norm_of_raw], raw_of_norm[:length], length