TUI: EPUB wie der Browser lesen (HTML5, Rohdaten) und Zeichenpositionen je Block

Blocktext, Block-Index und Zeichenoffsets stimmen nun bei sechs echten Büchern exakt mit
parseEpub des Web-Readers überein. Grundlage für einen echten Fortschrittsanteil
(fraction_for_anchor) statt 0.0. Parser html5lib statt lxml, Rohinhalt statt
ebooklib-get_content (formatiert um), UTF-8 wie JSZip.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

tui/README.md

@@ -131,10 +131,11 @@ NODE_PATH=<diora-web>/node_modules PLAYWRIGHT_BROWSERS_PATH=...   node tools/par
 python tools/parity/compare_blocks.py web.json
 ```
 
-Stand 2026-10-26 (6 Bücher, bis 25.798 Blöcke): Anzahl und Tags aller Blöcke gleich. Bekannte
-Textabweichungen ohne Wirkung auf Anker: lxml liest `<h1/>` (selbstschließend) als leer, der
-HTML5-Parser des Browsers als offenen Block; vereinzelt ein Leerzeichen mehr zwischen
-verschachtelten Blöcken.
+Stand 2026-10-26 (6 Bücher, bis 25.798 Blöcke): Anzahl, Tags, Text, Zeichenposition und
+-länge aller Blöcke sowie die Gesamtlänge sind identisch. Das setzt voraus, dass die TUI den
+EPUB-Inhalt wie der Browser liest: UTF-8-dekodiert, mit dem HTML5-Parser (`html5lib`, nicht
+libxml2) und aus den Rohdaten (`item.content`; `get_content()` von ebooklib formatiert um und
+fügt Leerraum ein).
 
 ## Grenzen der aktuellen Version
 

tui/diora_tui/blocks.py

@@ -31,10 +31,11 @@ from ebooklib import epub
 # EPUB content documents are XHTML; treating them as HTML (matching app.js's
 # `new DOMParser().parseFromString(html, 'text/html')`, static/js/app.js:2638)
 # is intentional, not a mistake — silence bs4's XML-vs-HTML nudge for it.
-# lxml's HTML parser is ~3x faster than bs4's built-in html.parser, which
-# matters here: some real-world EPUBs run to tens of thousands of blocks.
-warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
-_PARSER = "lxml"
+# html5lib builds the tree with the HTML5 algorithm, as the browser's DOMParser the web reader
+# uses does: same nesting for broken markup (`<h1/>` opens a block) and the same text nodes
+# (whitespace included), which the block indexes and the character offsets depend on. lxml's
+# HTML parser (libxml2) trims indentation and closes elements differently.
+_PARSER = "html5lib"
 
 # Matches app.js's EPUB_BLOCK_SELECTOR = 'p, h1, h2, h3, h4, h5, h6, li,
 # blockquote, dt, dd, figcaption, div:not(:has(*))'
@@ -74,6 +75,12 @@ class Block:
     # walk-up in app.js), so a target id resolves to "the block containing
     # it" rather than requiring the id to sit on the block tag itself.
     ids: list[str] = field(default_factory=list)
+    # Where the block sits in the book's text, counted in characters the way the web reader
+    # counts them (44-books.js _bookCharIndex): char_start = offset of the block's first text
+    # node among all text nodes of the book, char_len = its textContent length (nested blocks
+    # included). Progress as a share of the text (position -> fraction) is computed from these.
+    char_start: int = 0
+    char_len: int = 0
 
 
 @dataclass
@@ -86,6 +93,7 @@ class FlatBook:
     chapter_titles: list[str]
     chapter_start_block: list[int]  # blocks[chapter_start_block[i]] is chapter i's first block
     footnote_targets: dict[str, int]  # fragment id -> block index of its content
+    total_chars: int = 1  # all text nodes of the book (never 0: it divides)
 
 
 def _is_leaf_div(tag: Tag) -> bool:
@@ -123,31 +131,49 @@ def _walk_text(el: Tag, footnotes: list[FootnoteRef], out: list[str]) -> None:
             _walk_text(child, footnotes, out)
 
 
-def _extract_blocks(html: bytes, chapter_index: int) -> list[Block]:
-    soup = BeautifulSoup(html, _PARSER)
+def _text_nodes(el: Tag):
+    for node in el.descendants:
+        if isinstance(node, NavigableString) and not isinstance(node, PreformattedString):
+            yield node
+
+
+def _extract_blocks(html: bytes, chapter_index: int) -> tuple[list[Block], int]:
+    """The chapter's blocks (char_start relative to the chapter) and its text length in chars."""
+    # Decoded as UTF-8 first, as the web reader reads the file (JSZip 'text'); handing html5lib
+    # bytes would have it guess an encoding.
+    soup = BeautifulSoup(html.decode("utf-8", "replace"), _PARSER)
     for name in _STRIP_TAGS:
         for el in soup.find_all(name):
             el.decompose()
 
+    root = soup.body or soup
     blocks: list[Block] = []
-    for el in soup.find_all(True):
-        if el.name in _BLOCK_TAGS or _is_leaf_div(el):
-            footnotes: list[FootnoteRef] = []
-            parts: list[str] = []
-            _walk_text(el, footnotes, parts)
-            text = _JS_SPACE_RE.sub(" ", "".join(parts)).strip(" ")
-            ids = [tag_id for tag_id in (el.get("id"), *(d.get("id") for d in el.find_all(True))) if tag_id]
-            blocks.append(
-                Block(
-                    text=text,
-                    tag=el.name,
-                    chapter_index=chapter_index,
-                    id=el.get("id"),
-                    footnotes=footnotes,
-                    ids=ids,
-                )
-            )
-    return blocks
+    offset = 0
+    for node in root.descendants:
+        if isinstance(node, Tag):
+            if node.name in _BLOCK_TAGS or _is_leaf_div(node):
+                blocks.append(_make_block(node, chapter_index, offset))
+        elif isinstance(node, NavigableString) and not isinstance(node, PreformattedString):
+            offset += len(node)
+    return blocks, offset
+
+
+def _make_block(el: Tag, chapter_index: int, char_start: int) -> Block:
+    footnotes: list[FootnoteRef] = []
+    parts: list[str] = []
+    _walk_text(el, footnotes, parts)
+    text = _JS_SPACE_RE.sub(" ", "".join(parts)).strip(" ")
+    ids = [tag_id for tag_id in (el.get("id"), *(d.get("id") for d in el.find_all(True))) if tag_id]
+    return Block(
+        text=text,
+        tag=el.name,
+        chapter_index=chapter_index,
+        id=el.get("id"),
+        footnotes=footnotes,
+        ids=ids,
+        char_start=char_start,
+        char_len=sum(len(n) for n in _text_nodes(el)),
+    )
 
 
 def load_flat_book(path: Path) -> FlatBook:
@@ -163,6 +189,7 @@ def load_flat_book(path: Path) -> FlatBook:
     blocks: list[Block] = []
     chapter_titles: list[str] = []
     chapter_start_block: list[int] = []
+    total_chars = 0
 
     # app.js's parseEpub() includes every spine itemref unconditionally — no
     # `linear` filtering (static/js/app.js:2623-2625) — since footnote/endnote
@@ -174,7 +201,10 @@ def load_flat_book(path: Path) -> FlatBook:
         if item is None or item.get_type() != ebooklib.ITEM_DOCUMENT:
             continue
         chapter_index = len(chapter_titles)
-        chapter_blocks = _extract_blocks(item.get_content(), chapter_index)
+        chapter_blocks, chapter_chars = _extract_blocks(item.content, chapter_index)  # raw bytes: EpubHtml.get_content() pretty-prints, adding whitespace
+        for b in chapter_blocks:
+            b.char_start += total_chars
+        total_chars += chapter_chars
         chapter_start_block.append(len(blocks))
         first_text = next((b.text for b in chapter_blocks if b.text), None)
         chapter_titles.append((first_text or item.get_name())[:60])
@@ -184,6 +214,8 @@ def load_flat_book(path: Path) -> FlatBook:
     for idx, b in enumerate(blocks):
         for tag_id in b.ids:
             footnote_targets.setdefault(tag_id, idx)
+        if not b.char_len:  # no text of its own: where the previous block ends (0 for the first), as the web reader
+            b.char_start = blocks[idx - 1].char_start + blocks[idx - 1].char_len if idx else 0
 
     return FlatBook(
         id=_book_id(path),
@@ -194,9 +226,19 @@ def load_flat_book(path: Path) -> FlatBook:
         chapter_titles=chapter_titles,
         chapter_start_block=chapter_start_block,
         footnote_targets=footnote_targets,
+        total_chars=total_chars or 1,
     )
 
 
+def fraction_for_anchor(book: FlatBook, block_index: int, inner_fraction: float) -> float:
+    """The share of the book's text before the anchored place (0..1), as the web reader's
+    _fractionFromAnchor: what other clients show as progress."""
+    if not 0 <= block_index < len(book.blocks):
+        return 0.0
+    b = book.blocks[block_index]
+    return max(0.0, min(1.0, (b.char_start + inner_fraction * b.char_len) / book.total_chars))
+
+
 def format_anchor(block_index: int, inner_fraction: float) -> str:
     inner_fraction = max(0.0, min(1.0, inner_fraction))
     return f"{block_index}:{inner_fraction:.6f}"

tui/diora_tui/cache.py

@@ -19,12 +19,13 @@ from platformdirs import user_cache_dir
 from .blocks import FlatBook
 from .layout import BookLayout
 
+_CACHE_VERSION = 2  # 2: blocks carry char_start/char_len
 _CACHE_DIR = Path(user_cache_dir("diora-tui", "diora")) / "layout_cache"
 
 
 def _cache_key(path: Path, width: int) -> str:
     stat = path.stat()
-    raw = f"{path.resolve()}|{stat.st_size}|{stat.st_mtime_ns}|{width}"
+    raw = f"v{_CACHE_VERSION}|{path.resolve()}|{stat.st_size}|{stat.st_mtime_ns}|{width}"
     return hashlib.sha256(raw.encode()).hexdigest()[:32]
 
 

tui/pyproject.toml

@@ -8,6 +8,7 @@ dependencies = [
     "ebooklib>=0.18",
     "beautifulsoup4>=4.12",
     "lxml>=5.0",
+    "html5lib>=1.1",
     "platformdirs>=4.0",
     "requests>=2.31",
     "cryptography>=42.0",

tui/tools/parity/compare_blocks.py

@@ -25,9 +25,12 @@ def main(ref_file, only):
         wb, tb = web["blocks"], book.blocks
         structure = len(wb) == len(tb) and all(w["tag"] == t.tag for w, t in zip(wb, tb))
         texts = [i for i, (w, t) in enumerate(zip(wb, tb)) if w["text"] != t.text]
+        chars = [i for i, (w, t) in enumerate(zip(wb, tb)) if (w["start"], w["len"]) != (t.char_start, t.char_len)]
+        total_ok = web["total"] == book.total_chars
         failed |= not structure
         print(f"{'OK  ' if structure else 'FAIL'} {os.path.basename(file)[:50]}: web {len(wb)} / tui {len(tb)} blocks, "
-              f"{len(texts)} with other text" + (f" (first: {texts[0]})" if texts else ""))
+              f"{len(texts)} with other text" + (f" (first: {texts[0]})" if texts else "") +
+              f"; chars {'equal' if not chars and total_ok else f'{len(chars)} blocks differ, total web {web['total']} / tui {book.total_chars}'}")
     return 1 if failed else 0
 
 

tui/tools/parity/web_blocks.cjs

@@ -28,14 +28,25 @@ const { chromium } = require('playwright-core');
       const SEL = 'p, h1, h2, h3, h4, h5, h6, li, blockquote, dt, dd, figcaption, div:not(:has(*))'; // EPUB_BLOCK_SELECTOR, 44-books.js
       const blocks = Array.from(el.querySelectorAll(SEL));
       const chapters = Array.from(el.querySelectorAll('[data-epub-src]'));
-      const res = blocks.map((b) => ({
+      // _bookCharIndex (44-books.js): offset of each block's first text node among all text nodes
+      const offsets = new Map();
+      let pos = 0;
+      const walker = document.createTreeWalker(el, NodeFilter.SHOW_TEXT);
+      for (let n; (n = walker.nextNode());) { offsets.set(n, pos); pos += n.nodeValue.length; }
+      const starts = [];
+      blocks.forEach((b, i) => {
+        const first = document.createTreeWalker(b, NodeFilter.SHOW_TEXT).nextNode();
+        starts[i] = first ? offsets.get(first) : (i ? starts[i - 1] + blocks[i - 1].textContent.length : 0);
+      });
+      const res = blocks.map((b, i) => ({
+        start: starts[i],
         tag: b.tagName.toLowerCase(),
         text: b.textContent.replace(/\s+/g, ' ').trim(),
         len: b.textContent.length,
         ch: chapters.findIndex((c) => c.contains(b)),
       }));
       el.remove();
-      return { title: book.title, chapters: chapters.length, blocks: res };
+      return { title: book.title, chapters: chapters.length, total: pos || 1, blocks: res };
     }, b64);
     console.error(path.basename(file), result[file].blocks.length, 'blocks');
   }