mrwnslz
edc9c6d4c3
Blocktext, Block-Index und Zeichenoffsets stimmen nun bei sechs echten Büchern exakt mit parseEpub des Web-Readers überein. Grundlage für einen echten Fortschrittsanteil (fraction_for_anchor) statt 0.0. Parser html5lib statt lxml, Rohinhalt statt ebooklib-get_content (formatiert um), UTF-8 wie JSZip. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
262 lines
11 KiB
Python
262 lines
11 KiB
Python
"""Whole-book flat block extraction, matching static/js/app.js's EPUB_BLOCK_SELECTOR
|
||
and getPositionAnchor()/restoreFromAnchor() exactly in *structure* (blockIndex is
|
||
purely DOM order, no rendering needed) so position anchors are comparable across
|
||
the web reader and this TUI. See CLAUDE.md's `tui/` section for the full picture.
|
||
|
||
innerFraction on the web is defined via getBoundingClientRect() pixel geometry,
|
||
which has no TUI equivalent — a terminal has no font metrics/reflow the way a
|
||
browser does. We approximate it with row-based geometry from Rich's own text
|
||
wrapping (see reader.py), which is close enough because the server's "furthest
|
||
wins" comparison (_progress_is_further) only falls back to comparing fractions
|
||
when two anchors share the exact same block index — block index is the primary,
|
||
exactly-reproducible signal.
|
||
|
||
Footnote references are detected with the same heuristic as app.js's
|
||
_looksLikeFootnoteLink: wrapped in/wrapping a <sup>, a class name containing
|
||
note/footnote/fn, or an epub:type="noteref" attribute.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
import warnings
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
|
||
import ebooklib
|
||
from bs4 import BeautifulSoup, NavigableString, Tag, XMLParsedAsHTMLWarning
|
||
from bs4.element import PreformattedString
|
||
from ebooklib import epub
|
||
|
||
# EPUB content documents are XHTML; treating them as HTML (matching app.js's
|
||
# `new DOMParser().parseFromString(html, 'text/html')`, static/js/app.js:2638)
|
||
# is intentional, not a mistake — silence bs4's XML-vs-HTML nudge for it.
|
||
# html5lib builds the tree with the HTML5 algorithm, as the browser's DOMParser the web reader
|
||
# uses does: same nesting for broken markup (`<h1/>` opens a block) and the same text nodes
|
||
# (whitespace included), which the block indexes and the character offsets depend on. lxml's
|
||
# HTML parser (libxml2) trims indentation and closes elements differently.
|
||
_PARSER = "html5lib"
|
||
|
||
# Matches app.js's EPUB_BLOCK_SELECTOR = 'p, h1, h2, h3, h4, h5, h6, li,
|
||
# blockquote, dt, dd, figcaption, div:not(:has(*))'
|
||
_BLOCK_TAGS = {"p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "blockquote", "dt", "dd", "figcaption"}
|
||
# Matches app.js's sanitizeEpubHtml() strip list (script/style are also stripped
|
||
# via regex before DOMParser even runs there; decomposing here is equivalent).
|
||
_STRIP_TAGS = ["script", "style", "iframe", "object", "embed", "head", "meta", "link"]
|
||
# Matches app.js's _looksLikeFootnoteLink's class-name check.
|
||
_FOOTNOTE_CLASS_RE = re.compile(r"\bnote|\bfootnote|\bfn\b")
|
||
# JavaScript's `\s`, not Python's (which also matches \x1c-\x1f and \x85 but not ): a
|
||
# block's text is cut and measured the way the web reader does.
|
||
_JS_SPACE_RE = re.compile("[ \t\n\v\f\r -
]+")
|
||
|
||
# Anchor format the server accepts (books/views.py:save_progress); anything else
|
||
# is silently discarded back to ''.
|
||
_ANCHOR_RE = re.compile(r"\d{1,7}:\d(\.\d{1,6})?")
|
||
|
||
|
||
@dataclass
|
||
class FootnoteRef:
|
||
marker: str # visible text of the reference link, e.g. "1"
|
||
offset: int # character offset into the block's text where the marker sits
|
||
target_id: str # fragment id to resolve against block ids
|
||
|
||
|
||
@dataclass
|
||
class Block:
|
||
text: str
|
||
tag: str
|
||
chapter_index: int
|
||
id: str | None = None
|
||
footnotes: list[FootnoteRef] = field(default_factory=list)
|
||
# Every id found anywhere in this block's subtree, not just on the block
|
||
# element itself — footnote *targets* are frequently an <a id="..."> or
|
||
# similar nested a level or two inside the actual containing paragraph
|
||
# (see showFootnotePopover's `.closest('.footnote') || .parentElement`
|
||
# walk-up in app.js), so a target id resolves to "the block containing
|
||
# it" rather than requiring the id to sit on the block tag itself.
|
||
ids: list[str] = field(default_factory=list)
|
||
# Where the block sits in the book's text, counted in characters the way the web reader
|
||
# counts them (44-books.js _bookCharIndex): char_start = offset of the block's first text
|
||
# node among all text nodes of the book, char_len = its textContent length (nested blocks
|
||
# included). Progress as a share of the text (position -> fraction) is computed from these.
|
||
char_start: int = 0
|
||
char_len: int = 0
|
||
|
||
|
||
@dataclass
|
||
class FlatBook:
|
||
id: str
|
||
title: str
|
||
author: str
|
||
path: Path
|
||
blocks: list[Block]
|
||
chapter_titles: list[str]
|
||
chapter_start_block: list[int] # blocks[chapter_start_block[i]] is chapter i's first block
|
||
footnote_targets: dict[str, int] # fragment id -> block index of its content
|
||
total_chars: int = 1 # all text nodes of the book (never 0: it divides)
|
||
|
||
|
||
def _is_leaf_div(tag: Tag) -> bool:
|
||
return tag.name == "div" and tag.find(True) is None
|
||
|
||
|
||
def _looks_like_footnote_link(el: Tag, href: str) -> bool:
|
||
if "#" not in href:
|
||
return False
|
||
cls = " ".join(el.get("class") or []).lower()
|
||
if _FOOTNOTE_CLASS_RE.search(cls):
|
||
return True
|
||
epub_type = (el.get("epub:type") or "").lower()
|
||
if "noteref" in epub_type:
|
||
return True
|
||
return el.find_parent("sup") is not None or el.find("sup") is not None
|
||
|
||
|
||
def _walk_text(el: Tag, footnotes: list[FootnoteRef], out: list[str]) -> None:
|
||
for child in el.children:
|
||
if isinstance(child, PreformattedString):
|
||
continue # comments, CDATA, declarations: not text (textContent skips them)
|
||
if isinstance(child, NavigableString):
|
||
out.append(str(child))
|
||
elif isinstance(child, Tag):
|
||
if child.name == "a":
|
||
href = child.get("href") or ""
|
||
if _looks_like_footnote_link(child, href):
|
||
marker = _JS_SPACE_RE.sub(" ", child.get_text("")).strip(" ")
|
||
offset = len("".join(out))
|
||
target_id = href.split("#", 1)[1] if "#" in href else ""
|
||
footnotes.append(FootnoteRef(marker=marker, offset=offset, target_id=target_id))
|
||
out.append(marker)
|
||
continue
|
||
_walk_text(child, footnotes, out)
|
||
|
||
|
||
def _text_nodes(el: Tag):
|
||
for node in el.descendants:
|
||
if isinstance(node, NavigableString) and not isinstance(node, PreformattedString):
|
||
yield node
|
||
|
||
|
||
def _extract_blocks(html: bytes, chapter_index: int) -> tuple[list[Block], int]:
|
||
"""The chapter's blocks (char_start relative to the chapter) and its text length in chars."""
|
||
# Decoded as UTF-8 first, as the web reader reads the file (JSZip 'text'); handing html5lib
|
||
# bytes would have it guess an encoding.
|
||
soup = BeautifulSoup(html.decode("utf-8", "replace"), _PARSER)
|
||
for name in _STRIP_TAGS:
|
||
for el in soup.find_all(name):
|
||
el.decompose()
|
||
|
||
root = soup.body or soup
|
||
blocks: list[Block] = []
|
||
offset = 0
|
||
for node in root.descendants:
|
||
if isinstance(node, Tag):
|
||
if node.name in _BLOCK_TAGS or _is_leaf_div(node):
|
||
blocks.append(_make_block(node, chapter_index, offset))
|
||
elif isinstance(node, NavigableString) and not isinstance(node, PreformattedString):
|
||
offset += len(node)
|
||
return blocks, offset
|
||
|
||
|
||
def _make_block(el: Tag, chapter_index: int, char_start: int) -> Block:
|
||
footnotes: list[FootnoteRef] = []
|
||
parts: list[str] = []
|
||
_walk_text(el, footnotes, parts)
|
||
text = _JS_SPACE_RE.sub(" ", "".join(parts)).strip(" ")
|
||
ids = [tag_id for tag_id in (el.get("id"), *(d.get("id") for d in el.find_all(True))) if tag_id]
|
||
return Block(
|
||
text=text,
|
||
tag=el.name,
|
||
chapter_index=chapter_index,
|
||
id=el.get("id"),
|
||
footnotes=footnotes,
|
||
ids=ids,
|
||
char_start=char_start,
|
||
char_len=sum(len(n) for n in _text_nodes(el)),
|
||
)
|
||
|
||
|
||
def load_flat_book(path: Path) -> FlatBook:
|
||
from .epub import _book_id # reuse the same path-hash id scheme
|
||
|
||
raw = epub.read_epub(str(path), options={"ignore_ncx": True})
|
||
|
||
title_meta = raw.get_metadata("DC", "title")
|
||
title = title_meta[0][0] if title_meta else path.stem
|
||
author_meta = raw.get_metadata("DC", "creator")
|
||
author = author_meta[0][0] if author_meta else "Unbekannt"
|
||
|
||
blocks: list[Block] = []
|
||
chapter_titles: list[str] = []
|
||
chapter_start_block: list[int] = []
|
||
total_chars = 0
|
||
|
||
# app.js's parseEpub() includes every spine itemref unconditionally — no
|
||
# `linear` filtering (static/js/app.js:2623-2625) — since footnote/endnote
|
||
# targets are commonly parked in a linear="no" document. Skipping it here
|
||
# would both break footnote-target resolution and shift blockIndex
|
||
# numbering out of sync with the web reader for every block after it.
|
||
for idref, _linear in raw.spine:
|
||
item = raw.get_item_with_id(idref)
|
||
if item is None or item.get_type() != ebooklib.ITEM_DOCUMENT:
|
||
continue
|
||
chapter_index = len(chapter_titles)
|
||
chapter_blocks, chapter_chars = _extract_blocks(item.content, chapter_index) # raw bytes: EpubHtml.get_content() pretty-prints, adding whitespace
|
||
for b in chapter_blocks:
|
||
b.char_start += total_chars
|
||
total_chars += chapter_chars
|
||
chapter_start_block.append(len(blocks))
|
||
first_text = next((b.text for b in chapter_blocks if b.text), None)
|
||
chapter_titles.append((first_text or item.get_name())[:60])
|
||
blocks.extend(chapter_blocks)
|
||
|
||
footnote_targets: dict[str, int] = {}
|
||
for idx, b in enumerate(blocks):
|
||
for tag_id in b.ids:
|
||
footnote_targets.setdefault(tag_id, idx)
|
||
if not b.char_len: # no text of its own: where the previous block ends (0 for the first), as the web reader
|
||
b.char_start = blocks[idx - 1].char_start + blocks[idx - 1].char_len if idx else 0
|
||
|
||
return FlatBook(
|
||
id=_book_id(path),
|
||
title=title,
|
||
author=author,
|
||
path=path,
|
||
blocks=blocks,
|
||
chapter_titles=chapter_titles,
|
||
chapter_start_block=chapter_start_block,
|
||
footnote_targets=footnote_targets,
|
||
total_chars=total_chars or 1,
|
||
)
|
||
|
||
|
||
def fraction_for_anchor(book: FlatBook, block_index: int, inner_fraction: float) -> float:
|
||
"""The share of the book's text before the anchored place (0..1), as the web reader's
|
||
_fractionFromAnchor: what other clients show as progress."""
|
||
if not 0 <= block_index < len(book.blocks):
|
||
return 0.0
|
||
b = book.blocks[block_index]
|
||
return max(0.0, min(1.0, (b.char_start + inner_fraction * b.char_len) / book.total_chars))
|
||
|
||
|
||
def format_anchor(block_index: int, inner_fraction: float) -> str:
|
||
inner_fraction = max(0.0, min(1.0, inner_fraction))
|
||
return f"{block_index}:{inner_fraction:.6f}"
|
||
|
||
|
||
def parse_anchor(anchor: str) -> tuple[int, float] | None:
|
||
if not anchor or not _ANCHOR_RE.fullmatch(anchor):
|
||
return None
|
||
block_str, _, frac_str = anchor.partition(":")
|
||
return int(block_str), float(frac_str)
|
||
|
||
|
||
def anchor_is_further(new_anchor: str, old_anchor: str) -> bool:
|
||
"""Mirrors _progress_is_further / _cmpProgress (books/views.py, app.js)."""
|
||
new_parts = parse_anchor(new_anchor)
|
||
old_parts = parse_anchor(old_anchor)
|
||
if new_parts is None or old_parts is None:
|
||
return bool(new_anchor) and not old_anchor
|
||
nb, ni = new_parts
|
||
ob, oi = old_parts
|
||
return ni >= oi if nb == ob else nb >= ob
|