TUI: Blocktext wie der Web-Reader, Tests und Anker-Parität

HTML-Kommentare zählten als Text (1435 Blöcke in einem Testbuch), Leerraum wurde mit
Pythons statt JavaScripts \s zusammengezogen, Fußnotenmarker bekamen Leerzeichen.
Neu: tests/ (Blockliste, Anker, App-Start mit Textual-Pilot) und tools/parity/ – vergleicht
Anzahl und Tags der Blöcke mit parseEpub aus dem Web-Reader in headless Chromium
(6 Bücher, bis 25.798 Blöcke: gleich).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

tui/README.md

@@ -115,6 +115,27 @@ deutlich schneller. Das Rendering selbst skaliert nicht mit der Buchgröße —
 tatsächlich sichtbaren Zeilen werden gezeichnet (Textual Line API), nicht das ganze Buch
 auf einmal.
 
+## Tests und Anker-Parität mit dem Web-Reader
+
+```bash
+pip install -e . pytest
+pytest tests                       # Blockliste, Anker, App-Start (Textual-Pilot)
+```
+
+Die Position ist ein Block-Index, den alle Clients gleich zählen müssen. Ob die TUI das tut,
+prüft `tools/parity/` gegen den echten Web-Code (`parseEpub` aus `static/js/app/40-epub.js`
+in headless Chromium):
+
+```bash
+NODE_PATH=<diora-web>/node_modules PLAYWRIGHT_BROWSERS_PATH=...   node tools/parity/web_blocks.cjs <diora-web> web.json buch1.epub buch2.epub
+python tools/parity/compare_blocks.py web.json
+```
+
+Stand 2026-10-26 (6 Bücher, bis 25.798 Blöcke): Anzahl und Tags aller Blöcke gleich. Bekannte
+Textabweichungen ohne Wirkung auf Anker: lxml liest `<h1/>` (selbstschließend) als leer, der
+HTML5-Parser des Browsers als offenen Block; vereinzelt ein Leerzeichen mehr zwischen
+verschachtelten Blöcken.
+
 ## Grenzen der aktuellen Version
 
 - Nur EPUB, kein PDF — `sync` lädt PDFs im Account gar nicht erst herunter (übersprungen,

tui/diora_tui/blocks.py

@@ -25,6 +25,7 @@ from pathlib import Path
 
 import ebooklib
 from bs4 import BeautifulSoup, NavigableString, Tag, XMLParsedAsHTMLWarning
+from bs4.element import PreformattedString
 from ebooklib import epub
 
 # EPUB content documents are XHTML; treating them as HTML (matching app.js's
@@ -43,6 +44,9 @@ _BLOCK_TAGS = {"p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "blockquote", "dt"
 _STRIP_TAGS = ["script", "style", "iframe", "object", "embed", "head", "meta", "link"]
 # Matches app.js's _looksLikeFootnoteLink's class-name check.
 _FOOTNOTE_CLASS_RE = re.compile(r"\bnote|\bfootnote|\bfn\b")
+# JavaScript's `\s`, not Python's (which also matches \x1c-\x1f and \x85 but not ): a
+# block's text is cut and measured the way the web reader does.
+_JS_SPACE_RE = re.compile("[ \t\n\v\f\r   - 

  　]+")
 
 # Anchor format the server accepts (books/views.py:save_progress); anything else
 # is silently discarded back to ''.
@@ -102,13 +106,15 @@ def _looks_like_footnote_link(el: Tag, href: str) -> bool:
 
 def _walk_text(el: Tag, footnotes: list[FootnoteRef], out: list[str]) -> None:
     for child in el.children:
+        if isinstance(child, PreformattedString):
+            continue  # comments, CDATA, declarations: not text (textContent skips them)
         if isinstance(child, NavigableString):
             out.append(str(child))
         elif isinstance(child, Tag):
             if child.name == "a":
                 href = child.get("href") or ""
                 if _looks_like_footnote_link(child, href):
-                    marker = child.get_text(" ", strip=True)
+                    marker = _JS_SPACE_RE.sub(" ", child.get_text("")).strip(" ")
                     offset = len("".join(out))
                     target_id = href.split("#", 1)[1] if "#" in href else ""
                     footnotes.append(FootnoteRef(marker=marker, offset=offset, target_id=target_id))
@@ -129,7 +135,7 @@ def _extract_blocks(html: bytes, chapter_index: int) -> list[Block]:
             footnotes: list[FootnoteRef] = []
             parts: list[str] = []
             _walk_text(el, footnotes, parts)
-            text = re.sub(r"\s+", " ", "".join(parts)).strip()
+            text = _JS_SPACE_RE.sub(" ", "".join(parts)).strip(" ")
             ids = [tag_id for tag_id in (el.get("id"), *(d.get("id") for d in el.find_all(True))) if tag_id]
             blocks.append(
                 Block(

tui/tests/conftest.py

@@ -0,0 +1,44 @@
+"""Tests run against a throwaway home: progress, config and caches are module-level
+paths from platformdirs, read at import time, so the environment is set before anything
+from diora_tui is imported."""
+import os
+import tempfile
+import zipfile
+from pathlib import Path
+
+_HOME = tempfile.mkdtemp(prefix="diora-tui-test-")
+for var in ("LOCALAPPDATA", "APPDATA", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME"):
+    os.environ[var] = _HOME
+os.environ["HOME"] = _HOME
+
+import pytest  # noqa: E402
+
+CONTAINER = """<?xml version="1.0"?>
+<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
+<rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles>
+</container>"""
+
+
+def make_epub(path: Path, chapters: list[str], title: str = "Testbuch") -> Path:
+    """A minimal EPUB whose spine is `chapters` (each one the full XHTML of a document)."""
+    items = "".join(f'<item id="c{i}" href="c{i}.xhtml" media-type="application/xhtml+xml"/>' for i in range(len(chapters)))
+    spine = "".join(f'<itemref idref="c{i}"/>' for i in range(len(chapters)))
+    opf = f"""<?xml version="1.0"?>
+<package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="id">
+<metadata xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:title>{title}</dc:title><dc:creator>Autor</dc:creator>
+<dc:identifier id="id">test-{title}</dc:identifier><dc:language>de</dc:language></metadata>
+<manifest>{items}</manifest><spine>{spine}</spine></package>"""
+    with zipfile.ZipFile(path, "w") as z:
+        z.writestr("mimetype", "application/epub+zip")
+        z.writestr("META-INF/container.xml", CONTAINER)
+        z.writestr("OEBPS/content.opf", opf)
+        for i, body in enumerate(chapters):
+            z.writestr(f"OEBPS/c{i}.xhtml", f'<?xml version="1.0"?><html xmlns="http://www.w3.org/1999/xhtml"><head><title>x</title></head><body>{body}</body></html>')
+    return path
+
+
+@pytest.fixture
+def epub(tmp_path):
+    def build(chapters, title="Testbuch"):
+        return make_epub(tmp_path / f"{title}.epub", chapters, title)
+    return build

tui/tests/test_app_smoke.py

@@ -0,0 +1,30 @@
+"""The Textual app starts, lists a library, opens a book and leaves it again."""
+import asyncio
+
+from diora_tui.app import DioraTuiApp
+from diora_tui.reader_screen import ReaderScreen
+
+
+def test_open_read_and_leave(epub, tmp_path):
+    lib = tmp_path / "lib"
+    lib.mkdir()
+    path = epub(["<h1>Anfang</h1>" + "".join(f"<p>Absatz {i} " + "wort " * 40 + "</p>" for i in range(60))])
+    path.rename(lib / path.name)
+
+    async def run():
+        app = DioraTuiApp(lib)
+        async with app.run_test(size=(100, 30)) as pilot:
+            await pilot.pause()
+            await pilot.press("enter")
+            for _ in range(50):
+                await pilot.pause(0.1)
+                if isinstance(app.screen, ReaderScreen):
+                    break
+            assert isinstance(app.screen, ReaderScreen)
+            await pilot.pause(0.5)
+            await pilot.press("j", "j", "j", "n")
+            await pilot.press("escape")
+            await pilot.pause()
+            assert not isinstance(app.screen, ReaderScreen)
+
+    asyncio.run(run())

tui/tests/test_blocks.py

@@ -0,0 +1,53 @@
+"""The block list must count and read exactly like the web reader (static/js/app/44-books.js,
+40-epub.js sanitizeEpubHtml): anchors are block indexes shared between all clients. The
+real-book comparison against the web reader's own code is tools/parity/."""
+from diora_tui.blocks import anchor_is_further, format_anchor, load_flat_book, parse_anchor
+
+
+def texts(book):
+    return [(b.tag, b.text) for b in book.blocks]
+
+
+def test_block_selector_and_nesting(epub):
+    book = load_flat_book(epub([
+        "<h1>Titel</h1><p>Eins</p><blockquote><p>Zitat</p></blockquote><ul><li>a</li></ul>"
+        "<div>Blatt</div><div><span>kein Blatt</span></div><div><p>Behälter</p></div>"
+    ]))
+    # a div counts only when it has no element children; blockquote and its p both count
+    assert texts(book) == [
+        ("h1", "Titel"), ("p", "Eins"), ("blockquote", "Zitat"), ("p", "Zitat"),
+        ("li", "a"), ("div", "Blatt"), ("p", "Behälter"),
+    ]
+
+
+def test_all_spine_documents_count_even_without_text(epub):
+    book = load_flat_book(epub(["<p>a</p>", "<p>b</p>"]))
+    assert [b.chapter_index for b in book.blocks] == [0, 1]
+
+
+def test_comments_are_not_text(epub):
+    book = load_flat_book(epub(['<h1><!-- d="kap1" -->Titel</h1><p>x<!-- weg -->y</p>']))
+    assert texts(book) == [("h1", "Titel"), ("p", "xy")]
+
+
+def test_whitespace_is_javascripts(epub):
+    # \s in JS: no-break space and BOM collapse like spaces; \x85 does not
+    book = load_flat_book(epub(["<p>a  bc</p><p>\x85</p>"]))
+    assert texts(book)[0] == ("p", "a b c")
+    assert texts(book)[1][1] == "\x85"
+
+
+def test_footnote_marker_adds_no_space(epub):
+    book = load_flat_book(epub(['<p>Satz.<sup><a href="#n1">1</a></sup>Weiter</p><p id="n1">Fußnote</p>']))
+    assert book.blocks[0].text == "Satz.1Weiter"
+    assert book.blocks[0].footnotes[0].marker == "1"
+    assert book.footnote_targets["n1"] == 1
+
+
+def test_anchor_format_and_order():
+    assert format_anchor(12, 0.5) == "12:0.500000"
+    assert parse_anchor("12:0.500000") == (12, 0.5)
+    assert parse_anchor("kaputt") is None
+    assert anchor_is_further("5:0.1", "4:0.9")
+    assert anchor_is_further("5:0.5", "5:0.4")
+    assert not anchor_is_further("4:0.9", "5:0.1")

tui/tools/parity/compare_blocks.py

@@ -0,0 +1,35 @@
+"""Compare the TUI's block list with the web reader's (web_blocks.cjs output).
+
+  python compare_blocks.py web.json [name-part ...]
+
+What must match is the number and tag of the blocks (they make the position anchors
+shared by all clients). Text differences are reported too: they are cosmetic, but they
+show where the TUI's HTML parser (lxml) reads a book differently from Chrome's.
+Exit status 1 when a book's block list differs in length or tags.
+"""
+import json
+import os
+import sys
+from pathlib import Path
+
+from diora_tui.blocks import load_flat_book
+
+
+def main(ref_file, only):
+    ref = json.load(open(ref_file, encoding="utf-8"))
+    failed = False
+    for file, web in ref.items():
+        if only and not any(o in file for o in only):
+            continue
+        book = load_flat_book(Path(file))
+        wb, tb = web["blocks"], book.blocks
+        structure = len(wb) == len(tb) and all(w["tag"] == t.tag for w, t in zip(wb, tb))
+        texts = [i for i, (w, t) in enumerate(zip(wb, tb)) if w["text"] != t.text]
+        failed |= not structure
+        print(f"{'OK  ' if structure else 'FAIL'} {os.path.basename(file)[:50]}: web {len(wb)} / tui {len(tb)} blocks, "
+              f"{len(texts)} with other text" + (f" (first: {texts[0]})" if texts else ""))
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1], sys.argv[2:]))

tui/tools/parity/web_blocks.cjs

@@ -0,0 +1,44 @@
+// The web reader's own block list for EPUBs, from the real app code (static/js/app/40-epub.js
+// parseEpub + the reader's block selector) in headless Chromium. Reference for compare_blocks.py.
+//
+//   node web_blocks.cjs <diora-web checkout> <out.json> book.epub [more.epub ...]
+//
+// Needs playwright-core (e.g. NODE_PATH=<diora-web>/node_modules) and a Chromium
+// (PLAYWRIGHT_BROWSERS_PATH as for the e2e tests).
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+
+(async () => {
+  const [web, out, ...books] = process.argv.slice(2);
+  const browser = await chromium.launch();
+  const page = await browser.newPage();
+  await page.goto('about:blank');
+  await page.addScriptTag({ path: path.join(web, 'static/js/jszip.min.js') });
+  await page.addScriptTag({ path: path.join(web, 'static/js/app/40-epub.js') });
+  const result = {};
+  for (const file of books) {
+    const b64 = fs.readFileSync(file).toString('base64');
+    result[file] = await page.evaluate(async (b64) => {
+      const bin = Uint8Array.from(atob(b64), (c) => c.charCodeAt(0));
+      const book = await parseEpub(bin.buffer);
+      const el = document.createElement('div');
+      el.innerHTML = book.chapters.join('');
+      document.body.appendChild(el);
+      const SEL = 'p, h1, h2, h3, h4, h5, h6, li, blockquote, dt, dd, figcaption, div:not(:has(*))'; // EPUB_BLOCK_SELECTOR, 44-books.js
+      const blocks = Array.from(el.querySelectorAll(SEL));
+      const chapters = Array.from(el.querySelectorAll('[data-epub-src]'));
+      const res = blocks.map((b) => ({
+        tag: b.tagName.toLowerCase(),
+        text: b.textContent.replace(/\s+/g, ' ').trim(),
+        len: b.textContent.length,
+        ch: chapters.findIndex((c) => c.contains(b)),
+      }));
+      el.remove();
+      return { title: book.title, chapters: chapters.length, blocks: res };
+    }, b64);
+    console.error(path.basename(file), result[file].blocks.length, 'blocks');
+  }
+  fs.writeFileSync(out, JSON.stringify(result));
+  await browser.close();
+})().catch((e) => { console.error(e); process.exit(1); });