From edc9c6d4c358527f15e889631cf0f0059dc32159 Mon Sep 17 00:00:00 2001 From: mrwnslz Date: Mon, 26 Oct 2026 12:59:03 +0100 Subject: [PATCH] TUI: EPUB wie der Browser lesen (HTML5, Rohdaten) und Zeichenpositionen je Block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blocktext, Block-Index und Zeichenoffsets stimmen nun bei sechs echten Büchern exakt mit parseEpub des Web-Readers überein. Grundlage für einen echten Fortschrittsanteil (fraction_for_anchor) statt 0.0. Parser html5lib statt lxml, Rohinhalt statt ebooklib-get_content (formatiert um), UTF-8 wie JSZip. Co-Authored-By: Claude Sonnet 5.5 --- tui/README.md | 9 +-- tui/diora_tui/blocks.py | 92 ++++++++++++++++++++++-------- tui/diora_tui/cache.py | 3 +- tui/pyproject.toml | 1 + tui/tools/parity/compare_blocks.py | 5 +- tui/tools/parity/web_blocks.cjs | 15 ++++- 6 files changed, 92 insertions(+), 33 deletions(-) diff --git a/tui/README.md b/tui/README.md index c70167d..40fcd74 100644 --- a/tui/README.md +++ b/tui/README.md @@ -131,10 +131,11 @@ NODE_PATH=/node_modules PLAYWRIGHT_BROWSERS_PATH=... node tools/par python tools/parity/compare_blocks.py web.json ``` -Stand 2026-10-26 (6 Bücher, bis 25.798 Blöcke): Anzahl und Tags aller Blöcke gleich. Bekannte -Textabweichungen ohne Wirkung auf Anker: lxml liest `

` (selbstschließend) als leer, der -HTML5-Parser des Browsers als offenen Block; vereinzelt ein Leerzeichen mehr zwischen -verschachtelten Blöcken. +Stand 2026-10-26 (6 Bücher, bis 25.798 Blöcke): Anzahl, Tags, Text, Zeichenposition und +-länge aller Blöcke sowie die Gesamtlänge sind identisch. Das setzt voraus, dass die TUI den +EPUB-Inhalt wie der Browser liest: UTF-8-dekodiert, mit dem HTML5-Parser (`html5lib`, nicht +libxml2) und aus den Rohdaten (`item.content`; `get_content()` von ebooklib formatiert um und +fügt Leerraum ein). ## Grenzen der aktuellen Version diff --git a/tui/diora_tui/blocks.py b/tui/diora_tui/blocks.py index 9f985ba..fc3be10 100644 --- a/tui/diora_tui/blocks.py +++ b/tui/diora_tui/blocks.py @@ -31,10 +31,11 @@ from ebooklib import epub # EPUB content documents are XHTML; treating them as HTML (matching app.js's # `new DOMParser().parseFromString(html, 'text/html')`, static/js/app.js:2638) # is intentional, not a mistake — silence bs4's XML-vs-HTML nudge for it. -# lxml's HTML parser is ~3x faster than bs4's built-in html.parser, which -# matters here: some real-world EPUBs run to tens of thousands of blocks. -warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) -_PARSER = "lxml" +# html5lib builds the tree with the HTML5 algorithm, as the browser's DOMParser the web reader +# uses does: same nesting for broken markup (`

` opens a block) and the same text nodes +# (whitespace included), which the block indexes and the character offsets depend on. lxml's +# HTML parser (libxml2) trims indentation and closes elements differently. +_PARSER = "html5lib" # Matches app.js's EPUB_BLOCK_SELECTOR = 'p, h1, h2, h3, h4, h5, h6, li, # blockquote, dt, dd, figcaption, div:not(:has(*))' @@ -74,6 +75,12 @@ class Block: # walk-up in app.js), so a target id resolves to "the block containing # it" rather than requiring the id to sit on the block tag itself. ids: list[str] = field(default_factory=list) + # Where the block sits in the book's text, counted in characters the way the web reader + # counts them (44-books.js _bookCharIndex): char_start = offset of the block's first text + # node among all text nodes of the book, char_len = its textContent length (nested blocks + # included). Progress as a share of the text (position -> fraction) is computed from these. + char_start: int = 0 + char_len: int = 0 @dataclass @@ -86,6 +93,7 @@ class FlatBook: chapter_titles: list[str] chapter_start_block: list[int] # blocks[chapter_start_block[i]] is chapter i's first block footnote_targets: dict[str, int] # fragment id -> block index of its content + total_chars: int = 1 # all text nodes of the book (never 0: it divides) def _is_leaf_div(tag: Tag) -> bool: @@ -123,31 +131,49 @@ def _walk_text(el: Tag, footnotes: list[FootnoteRef], out: list[str]) -> None: _walk_text(child, footnotes, out) -def _extract_blocks(html: bytes, chapter_index: int) -> list[Block]: - soup = BeautifulSoup(html, _PARSER) +def _text_nodes(el: Tag): + for node in el.descendants: + if isinstance(node, NavigableString) and not isinstance(node, PreformattedString): + yield node + + +def _extract_blocks(html: bytes, chapter_index: int) -> tuple[list[Block], int]: + """The chapter's blocks (char_start relative to the chapter) and its text length in chars.""" + # Decoded as UTF-8 first, as the web reader reads the file (JSZip 'text'); handing html5lib + # bytes would have it guess an encoding. + soup = BeautifulSoup(html.decode("utf-8", "replace"), _PARSER) for name in _STRIP_TAGS: for el in soup.find_all(name): el.decompose() + root = soup.body or soup blocks: list[Block] = [] - for el in soup.find_all(True): - if el.name in _BLOCK_TAGS or _is_leaf_div(el): - footnotes: list[FootnoteRef] = [] - parts: list[str] = [] - _walk_text(el, footnotes, parts) - text = _JS_SPACE_RE.sub(" ", "".join(parts)).strip(" ") - ids = [tag_id for tag_id in (el.get("id"), *(d.get("id") for d in el.find_all(True))) if tag_id] - blocks.append( - Block( - text=text, - tag=el.name, - chapter_index=chapter_index, - id=el.get("id"), - footnotes=footnotes, - ids=ids, - ) - ) - return blocks + offset = 0 + for node in root.descendants: + if isinstance(node, Tag): + if node.name in _BLOCK_TAGS or _is_leaf_div(node): + blocks.append(_make_block(node, chapter_index, offset)) + elif isinstance(node, NavigableString) and not isinstance(node, PreformattedString): + offset += len(node) + return blocks, offset + + +def _make_block(el: Tag, chapter_index: int, char_start: int) -> Block: + footnotes: list[FootnoteRef] = [] + parts: list[str] = [] + _walk_text(el, footnotes, parts) + text = _JS_SPACE_RE.sub(" ", "".join(parts)).strip(" ") + ids = [tag_id for tag_id in (el.get("id"), *(d.get("id") for d in el.find_all(True))) if tag_id] + return Block( + text=text, + tag=el.name, + chapter_index=chapter_index, + id=el.get("id"), + footnotes=footnotes, + ids=ids, + char_start=char_start, + char_len=sum(len(n) for n in _text_nodes(el)), + ) def load_flat_book(path: Path) -> FlatBook: @@ -163,6 +189,7 @@ def load_flat_book(path: Path) -> FlatBook: blocks: list[Block] = [] chapter_titles: list[str] = [] chapter_start_block: list[int] = [] + total_chars = 0 # app.js's parseEpub() includes every spine itemref unconditionally — no # `linear` filtering (static/js/app.js:2623-2625) — since footnote/endnote @@ -174,7 +201,10 @@ def load_flat_book(path: Path) -> FlatBook: if item is None or item.get_type() != ebooklib.ITEM_DOCUMENT: continue chapter_index = len(chapter_titles) - chapter_blocks = _extract_blocks(item.get_content(), chapter_index) + chapter_blocks, chapter_chars = _extract_blocks(item.content, chapter_index) # raw bytes: EpubHtml.get_content() pretty-prints, adding whitespace + for b in chapter_blocks: + b.char_start += total_chars + total_chars += chapter_chars chapter_start_block.append(len(blocks)) first_text = next((b.text for b in chapter_blocks if b.text), None) chapter_titles.append((first_text or item.get_name())[:60]) @@ -184,6 +214,8 @@ def load_flat_book(path: Path) -> FlatBook: for idx, b in enumerate(blocks): for tag_id in b.ids: footnote_targets.setdefault(tag_id, idx) + if not b.char_len: # no text of its own: where the previous block ends (0 for the first), as the web reader + b.char_start = blocks[idx - 1].char_start + blocks[idx - 1].char_len if idx else 0 return FlatBook( id=_book_id(path), @@ -194,9 +226,19 @@ def load_flat_book(path: Path) -> FlatBook: chapter_titles=chapter_titles, chapter_start_block=chapter_start_block, footnote_targets=footnote_targets, + total_chars=total_chars or 1, ) +def fraction_for_anchor(book: FlatBook, block_index: int, inner_fraction: float) -> float: + """The share of the book's text before the anchored place (0..1), as the web reader's + _fractionFromAnchor: what other clients show as progress.""" + if not 0 <= block_index < len(book.blocks): + return 0.0 + b = book.blocks[block_index] + return max(0.0, min(1.0, (b.char_start + inner_fraction * b.char_len) / book.total_chars)) + + def format_anchor(block_index: int, inner_fraction: float) -> str: inner_fraction = max(0.0, min(1.0, inner_fraction)) return f"{block_index}:{inner_fraction:.6f}" diff --git a/tui/diora_tui/cache.py b/tui/diora_tui/cache.py index 21340b8..e3abc49 100644 --- a/tui/diora_tui/cache.py +++ b/tui/diora_tui/cache.py @@ -19,12 +19,13 @@ from platformdirs import user_cache_dir from .blocks import FlatBook from .layout import BookLayout +_CACHE_VERSION = 2 # 2: blocks carry char_start/char_len _CACHE_DIR = Path(user_cache_dir("diora-tui", "diora")) / "layout_cache" def _cache_key(path: Path, width: int) -> str: stat = path.stat() - raw = f"{path.resolve()}|{stat.st_size}|{stat.st_mtime_ns}|{width}" + raw = f"v{_CACHE_VERSION}|{path.resolve()}|{stat.st_size}|{stat.st_mtime_ns}|{width}" return hashlib.sha256(raw.encode()).hexdigest()[:32] diff --git a/tui/pyproject.toml b/tui/pyproject.toml index 969d473..5a26af8 100644 --- a/tui/pyproject.toml +++ b/tui/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "ebooklib>=0.18", "beautifulsoup4>=4.12", "lxml>=5.0", + "html5lib>=1.1", "platformdirs>=4.0", "requests>=2.31", "cryptography>=42.0", diff --git a/tui/tools/parity/compare_blocks.py b/tui/tools/parity/compare_blocks.py index e26061f..4e38e85 100644 --- a/tui/tools/parity/compare_blocks.py +++ b/tui/tools/parity/compare_blocks.py @@ -25,9 +25,12 @@ def main(ref_file, only): wb, tb = web["blocks"], book.blocks structure = len(wb) == len(tb) and all(w["tag"] == t.tag for w, t in zip(wb, tb)) texts = [i for i, (w, t) in enumerate(zip(wb, tb)) if w["text"] != t.text] + chars = [i for i, (w, t) in enumerate(zip(wb, tb)) if (w["start"], w["len"]) != (t.char_start, t.char_len)] + total_ok = web["total"] == book.total_chars failed |= not structure print(f"{'OK ' if structure else 'FAIL'} {os.path.basename(file)[:50]}: web {len(wb)} / tui {len(tb)} blocks, " - f"{len(texts)} with other text" + (f" (first: {texts[0]})" if texts else "")) + f"{len(texts)} with other text" + (f" (first: {texts[0]})" if texts else "") + + f"; chars {'equal' if not chars and total_ok else f'{len(chars)} blocks differ, total web {web['total']} / tui {book.total_chars}'}") return 1 if failed else 0 diff --git a/tui/tools/parity/web_blocks.cjs b/tui/tools/parity/web_blocks.cjs index 3453404..edc13d8 100644 --- a/tui/tools/parity/web_blocks.cjs +++ b/tui/tools/parity/web_blocks.cjs @@ -28,14 +28,25 @@ const { chromium } = require('playwright-core'); const SEL = 'p, h1, h2, h3, h4, h5, h6, li, blockquote, dt, dd, figcaption, div:not(:has(*))'; // EPUB_BLOCK_SELECTOR, 44-books.js const blocks = Array.from(el.querySelectorAll(SEL)); const chapters = Array.from(el.querySelectorAll('[data-epub-src]')); - const res = blocks.map((b) => ({ + // _bookCharIndex (44-books.js): offset of each block's first text node among all text nodes + const offsets = new Map(); + let pos = 0; + const walker = document.createTreeWalker(el, NodeFilter.SHOW_TEXT); + for (let n; (n = walker.nextNode());) { offsets.set(n, pos); pos += n.nodeValue.length; } + const starts = []; + blocks.forEach((b, i) => { + const first = document.createTreeWalker(b, NodeFilter.SHOW_TEXT).nextNode(); + starts[i] = first ? offsets.get(first) : (i ? starts[i - 1] + blocks[i - 1].textContent.length : 0); + }); + const res = blocks.map((b, i) => ({ + start: starts[i], tag: b.tagName.toLowerCase(), text: b.textContent.replace(/\s+/g, ' ').trim(), len: b.textContent.length, ch: chapters.findIndex((c) => c.contains(b)), })); el.remove(); - return { title: book.title, chapters: chapters.length, blocks: res }; + return { title: book.title, chapters: chapters.length, total: pos || 1, blocks: res }; }, b64); console.error(path.basename(file), result[file].blocks.length, 'blocks'); }