From 3c7be76a93f2fd79fab580a4e5fe0862f780c262 Mon Sep 17 00:00:00 2001 From: mrwnslz Date: Sat, 17 Oct 2026 11:48:44 +0200 Subject: [PATCH] Zeitschriften: Artikel richtig erkennen (GegenStandpunkt, PDF-Hefte) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - EPUB: ein Artikel ist eine Datei; die Einträge danach in dieselbe Datei sind seine Abschnitte, gleich auf welcher Ebene (beim GegenStandpunkt stehen Zwischenüberschriften teils ganz oben). Unterzeile = die Schlagzeile unter dem Titel, nicht nummerierte Abschnitte. - PDF: die meisten Hefte haben keine Lesezeichen – die Artikel kommen jetzt aus dem gedruckten Inhaltsverzeichnis (Jacobin, NLR, LuXemburg, manuskripte): Seitenzahl vorn oder hinten, Autor in eigener Spalte oder darunter („Von …“), Titel in Versalien über mehrere Zeilen, Rubriken; die gedruckten Seitenzahlen werden über die Titel auf die PDF-Seiten abgebildet. - Index-Version 2: schon gebaute Indizes werden neu gebaut. sw.js: diora-v128. Co-Authored-By: Claude Opus 5.5 --- docs/library.md | 37 +++- static/js/app/54-issues.js | 419 +++++++++++++++++++++++++++++++++---- static/js/sw.js | 2 +- 3 files changed, 414 insertions(+), 44 deletions(-) diff --git a/docs/library.md b/docs/library.md index b167270..d72fcea 100644 --- a/docs/library.md +++ b/docs/library.md @@ -106,13 +106,36 @@ so is each article in it. Neither is reduced to the other articles with their state, a teaser and the reading time; the ⋮ / mark sets read / unread / skipped by hand. -**The index** comes from the file, on the device: an EPUB's table of contents -(its leaves are the articles, their parents the Rubriken; without one, each -spine file with text), a PDF's outline (page ranges). Teasers and reading times -(200 words a minute) come from the text. It is kept in the encrypted meta as -`issue: {v: 1, a: [{h, t, r, x, m, p?, e?}]}` (href, title, Rubrik, teaser, -minutes, a PDF's first and last page), or `{v: 1, a: [], none: true}` for an -issue without a usable table of contents (read as a whole). Built in the +**The index** comes from the file, on the device: + +- **EPUB**: the table of contents, by file — an article is a file (the first + entry pointing into it); the entries after it into the same file are its + sections, whatever their depth (GegenStandpunkt puts some at the top level); + an entry whose children point into other files is a Rubrik. A one-file EPUB: + the leaves. No table of contents: each spine file with text. The teaser is + the headline under the title (the headings before the first paragraph that + the title doesn't say, not numbered sections), else the first paragraph. +- **PDF**: a real outline (word processors' `_Hlk…`/`_Toc…` leftovers don't + count); most magazines' PDFs have none, so mostly **the contents page + printed in the issue**: among the first 16 pages the one with most numbered + rows (numbers that merely count 1, 2, 3 … are a list, not a contents), plus + its neighbours with entries. A row: the page number at the start + ("14 Der Krieg gegen Palästina", Jacobin), or at the end, maybe with the + author in a column before the title ("Nathan Sperber | Beyond Neoliberalism? + | 23", NLR); lines under it in its type size continue the title, an author + line ("Von …", a name) gives the author, lines in other type the teaser; a + title in capitals may begin on the lines above its number, a lone word on top + is its kicker (LuXemburg: GESPRÄCH, INFOGRAFIK); "Name: Titel" + (manuskripte). Capitals in the contents' type are Rubriken. The printed + numbers are mapped to the file's pages by the offset most titles agree on + (found in the page's text, a headline in capitals preferred); running heads + and the contents page's own head are dropped. + +Reading times count 200 words a minute. It is kept in the encrypted meta as +`issue: {v: 2, a: [{h, t, r, x, m, a?, p?, e?}]}` (href, title, Rubrik, teaser, +minutes, author, a PDF's first and last page), or `{v: 2, a: [], none: true}` +for an issue without a usable table of contents (read as a whole). An index of +another version is built anew. Built in the background for issues without one (not on e-ink: there when the issue is opened, or it comes from another device). diff --git a/static/js/app/54-issues.js b/static/js/app/54-issues.js index dea3e59..6b34c29 100644 --- a/static/js/app/54-issues.js +++ b/static/js/app/54-issues.js @@ -16,6 +16,7 @@ // --------------------------------------------------------------------------- const ISSUE_WPM = 200; // words per minute, for the reading times +const ISSUE_INDEX_V = 2; // an index of another version is built anew const _issuePendingKey = 'diora_issue_pending'; function _issueEsc(s) { @@ -26,7 +27,12 @@ const _issueMeta = b => (b && (bookMetaCache[b.id] || {})) || {}; function _issueIndexOf(b) { const m = _issueMeta(b); const ix = m.issue || (b && b.issue); - return ix && ix.a && ix.a.length ? ix.a : null; + return ix && ix.v === ISSUE_INDEX_V && ix.a && ix.a.length ? ix.a : null; +} +// An issue whose file has no usable table of contents (under this version's rules). +function _issueNone(b) { + const ix = _issueMeta(b).issue || (b && b.issue); + return !!(ix && ix.v === ISSUE_INDEX_V && ix.none); } // ---- state: {href: 'r' (gelesen) | 's' (übersprungen)} and the article one was in ----------- @@ -123,9 +129,39 @@ window.addEventListener('online', () => Object.keys(_issuePending()).forEach(id // ---- the index ----------------------------------------------------------------------------------- -// The articles of a table of contents: its leaves, each under the nearest entry above it -// (the Rubrik); one per target. [{h, t, r}] -function _issueArticlesFromToc(toc) { +// The articles of a table of contents. An EPUB's (byFile): an article is a file — the +// first entry pointing into it; the entries after it into the same file are its sections +// (whatever their depth), and an entry whose children point into other files is a Rubrik. +// Otherwise (one file for all, a PDF's outline): the leaves, each under the entry above it. +// Entries that are word processors' leftovers (_Hlk…, _Toc…) don't count. [{h, t, r}] +const _issueJunk = label => /^_?(Hlk|Toc|Ref|GoBack)\d*$/i.test(String(label || '').trim()); + +function _issueArticlesFromToc(toc, byFile) { + toc = toc.filter(e => !_issueJunk(e.label)); + const fileOf = h => String(h || '').split('#')[0]; + if (byFile) { + const out = []; + const seen = new Set(); + for (let i = 0; i < toc.length; i++) { + const e = toc[i]; + if (!e.href) continue; + const f = fileOf(e.href); + let container = false; + for (let k = i + 1; k < toc.length && toc[k].depth > e.depth; k++) { + if (toc[k].href && fileOf(toc[k].href) !== f) { container = true; break; } + } + if (container || seen.has(f)) continue; + seen.add(f); + let r = ''; + for (let k = i - 1, d = e.depth; k >= 0 && d > 0; k--) { + if (toc[k].depth >= d) continue; + d = toc[k].depth; + if (!toc[k].href || fileOf(toc[k].href) !== f) { r = toc[k].label || ''; break; } + } + out.push({h: e.href, t: (e.label || '').replace(/\s+/g, ' ').trim() || 'Ohne Titel', r: r.replace(/\s+/g, ' ').trim()}); + } + if (out.length >= 2) return out; + } const out = []; const seen = new Set(); const parents = []; @@ -154,6 +190,9 @@ const _issueHtmlText = html => String(html || '') .replace(//gi, ' ') .replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, ' ') .replace(/<[^>]+>/g, ' ') + .replace(/&#x([0-9a-f]+);/gi, (m, h) => String.fromCharCode(parseInt(h, 16))) + .replace(/&#(\d+);/g, (m, d) => String.fromCharCode(parseInt(d, 10))) + .replace(/ /g, ' ') .replace(/ /g, ' ').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'"); async function _issueIndexEpub(bytes) { @@ -172,7 +211,7 @@ async function _issueIndexEpub(bytes) { mediaType: item.getAttribute('media-type') || '', properties: item.getAttribute('properties') || ''}; }); const spine = Array.prototype.map.call(opfDoc.querySelectorAll('spine > itemref'), r => (manifest[r.getAttribute('idref')] || {}).href).filter(Boolean); - let arts = _issueArticlesFromToc(await _parseEpubToc(zip, opfDoc, manifest)); + let arts = _issueArticlesFromToc(await _parseEpubToc(zip, opfDoc, manifest), true); const files = {}; const fileText = async path => { if (!(path in files)) { @@ -216,48 +255,356 @@ async function _issueIndexEpub(bytes) { for (let s = startIdx + 1; startIdx >= 0 && s < (endIdx < 0 ? spine.length : endIdx); s++) html += await fileText(spine[s]); if (nFile && nFrag) { const n = await fileText(nFile); html += n.slice(0, posIn(n, nFrag)); } } - Object.assign(arts[i], _issueTextStats(_issueHtmlText(html), arts[i].t)); + const st = _issueTextStats(_issueHtmlText(html), arts[i].t); + // The headings before the first paragraph that the title doesn't already say (a + // magazine's headline under its kicker): the teaser; else the text's beginning. + const body = html.replace(//i, ''); + const firstP = body.search(/]/i); + const heads = []; + const re = /]*>([\s\S]*?)<\/h[1-4]>/gi; + let m; + while ((m = re.exec(body)) && (firstP < 0 || m.index < firstP)) { + const t = _issueHtmlText(m[1]).replace(/\s+/g, ' ').trim(); + // Not one of the article's own numbered sections ("I. Verarmung …", "1. …"). + if (t && !/^([IVXL]+|\d+)\.\s/.test(t) && _issueNorm(arts[i].t).indexOf(_issueNorm(t)) < 0) heads.push(t); + } + if (heads.length) st.x = heads.join(' – '); + else if (firstP >= 0) st.x = _issueTextStats(_issueHtmlText(body.slice(firstP)), '').x; + Object.assign(arts[i], st); } return arts; } +const _issueNorm = s => String(s || '').toLowerCase().replace(/ß/g, 'ss').normalize('NFKD').replace(/[^a-z0-9]/g, ''); + async function _issueIndexPdf(bytes) { - if (typeof pdfjsLib === 'undefined' || typeof _parsePdfOutline !== 'function') return null; + if (typeof pdfjsLib === 'undefined') return null; const pdf = await pdfjsLib.getDocument({data: new Uint8Array(bytes)}).promise; - const outline = await pdf.getOutline(); - if (!outline || !outline.length) return []; - const arts = _issueArticlesFromToc(await _parsePdfOutline(pdf, outline)) - .filter(a => /^#pdf-page-\d+$/.test(a.h)); - arts.forEach(a => { a.p = parseInt(a.h.slice(10), 10); }); - arts.sort((a, b) => a.p - b.p); + try { + return await _issuePdfArticles(pdf); + } finally { + try { pdf.destroy(); } catch (e) {} + } +} + +// A PDF's articles: from its outline where it has a real one, else from the contents +// page printed in the issue (most magazines' PDFs have no outline). +async function _issuePdfArticles(pdf) { const pageText = {}; const textOf = async n => { if (!(n in pageText)) { try { - const page = await pdf.getPage(n); - const c = await page.getTextContent(); + const c = await (await pdf.getPage(n)).getTextContent(); pageText[n] = c.items.map(it => it.str).join(' '); } catch (e) { pageText[n] = ''; } } return pageText[n]; }; - for (let i = 0; i < arts.length; i++) { - const end = i + 1 < arts.length ? Math.max(arts[i].p, arts[i + 1].p - 1) : pdf.numPages; - arts[i].e = end; - let text = ''; - for (let n = arts[i].p; n <= Math.min(end, arts[i].p + 30); n++) text += ' ' + await textOf(n); - // Pages shared with the next article: counted for both (the next one starts on it). - const s = _issueTextStats(text, arts[i].t); - // The teaser: from where the title stands on the first page, if it does. - const first = (await textOf(arts[i].p)).replace(/\s+/g, ' '); - const k = first.toLowerCase().indexOf(arts[i].t.toLowerCase()); - if (k >= 0) s.x = _issueTextStats(first.slice(k + arts[i].t.length), '').x; - Object.assign(arts[i], s); + let arts = []; + const outline = await pdf.getOutline().catch(() => null); + if (outline && outline.length && typeof _parsePdfOutline === 'function') { + arts = _issueArticlesFromToc(await _parsePdfOutline(pdf, outline)) + .filter(a => /^#pdf-page-\d+$/.test(a.h)) + .map(a => Object.assign(a, {p: parseInt(a.h.slice(10), 10)})); + } + if (arts.length < 3) arts = await _issuePrintedToc(pdf, textOf); + if (arts.length < 2) return []; + arts.sort((a, b) => a.p - b.p); + for (let i = 0; i < arts.length; i++) { + const a = arts[i]; + a.h = '#pdf-page-' + a.p; + a.e = i + 1 < arts.length ? Math.max(a.p, arts[i + 1].p - 1) : pdf.numPages; + let text = ''; + for (let n = a.p; n <= Math.min(a.e, a.p + 40); n++) text += ' ' + await textOf(n); + const s = _issueTextStats(text, ''); + // From a printed contents page: no teaser (a page's running heads and captions come + // first in its text); the author stands there instead. + if (a.a !== undefined) s.x = a.x || ''; + else if (!a.x) { + // The teaser: what follows the title on its first page, if it stands there. + const first = (await textOf(a.p)).replace(/\s+/g, ' '); + const k = _issueFind(first, a.t); + s.x = _issueTextStats(k >= 0 ? first.slice(k) : '', '').x; + } else s.x = a.x; + Object.assign(a, s); } - try { pdf.destroy(); } catch (e) {} return arts; } +// Where `title` stands in `text` (letters and digits only, any case): the index in +// `text` just after it, or -1. +function _issueFind(text, title) { + const want = _issueNorm(title).slice(0, 40); + if (want.length < 4) return -1; + let norm = ''; + const map = []; + for (let i = 0; i < text.length; i++) { + const c = _issueNorm(text[i]); + for (let k = 0; k < c.length; k++) { norm += c[k]; map.push(i + 1); } + } + const at = norm.indexOf(want); + return at < 0 ? -1 : map[at + want.length - 1]; +} + +function _issueSpan(text, title) { + const end = _issueFind(text, title); + if (end < 0) return ''; + const want = _issueNorm(title).slice(0, 40); + let n = 0, i = end; + while (i > 0 && n < want.length) { i--; n += _issueNorm(text[i]).length; } + return text.slice(i, end); +} + +// The lines of a page as pdf.js lays its text out: pieces on one baseline that sit close +// together make a segment; far apart (another column), two. [{x, y, size, text}] +async function _issuePdfSegments(pdf, n) { + const page = await pdf.getPage(n); + const c = await page.getTextContent(); + const items = c.items.filter(it => it.str && it.str.trim()) + .map(it => ({x: it.transform[4], y: it.transform[5], w: it.width, size: Math.hypot(it.transform[2], it.transform[3]) || it.height || 10, s: it.str})); + items.sort((a, b) => (b.y - a.y) || (a.x - b.x)); + const segs = []; + let line = []; + const flush = () => { + if (!line.length) return; + line.sort((a, b) => a.x - b.x); + let seg = null; + for (const it of line) { + if (seg && it.x - seg.end > Math.max(24, it.size * 3)) { segs.push(seg); seg = null; } + if (!seg) seg = {x: it.x, y: it.y, size: it.size, text: it.s, end: it.x + it.w}; + else { + seg.text += (it.x - seg.end > it.size * 0.15 && !/\s$/.test(seg.text) && !/^\s/.test(it.s) ? ' ' : '') + it.s; + seg.end = Math.max(seg.end, it.x + it.w); + seg.size = Math.max(seg.size, it.size); + } + } + if (seg) segs.push(seg); + line = []; + }; + for (const it of items) { + if (line.length && Math.abs(line[0].y - it.y) > Math.min(line[0].size, it.size) * 0.5) flush(); + line.push(it); + } + flush(); + segs.forEach(s => { s.text = s.text.replace(/\s+/g, ' ').trim(); }); + return segs.filter(s => s.text); +} + +// The contents page printed in the issue. Its rows: a page number at the start +// ("14 Der Krieg gegen Palästina") or at the end, maybe with the author before the title +// in a column of its own ("Nathan Sperber | Beyond Neoliberalism? | 23"); the rows just +// under one, indented alike in its type size, continue it (more title, the author); +// capitals make a heading (the Rubrik). The printed numbers are matched to the file's +// pages by finding the titles there. +const _ISSUE_L = 'A-Za-zÀ-ÖØ-öø-ÿĀ-ſ'; +const _issueNameLike = t => t.length <= 45 && + /^[A-ZÀ-ÖØ-ÞĀ-ſ][A-Za-zÀ-ÖØ-öø-ÿĀ-ſ.'’-]*(\s+((und|and|&|von|van|de|der|del|y|bin)\s+)?[A-ZÀ-ÖØ-ÞĀ-ſ][A-Za-zÀ-ÖØ-öø-ÿĀ-ſ.'’-]*){1,4}$/.test(t); + +function _issueTocRows(segs, N) { + const rows = []; + segs.slice().sort((a, b) => (b.y - a.y) || (a.x - b.x)).forEach(s => { + const row = rows.filter(r => Math.abs(r.y - s.y) < Math.min(r.size, s.size) * 0.4)[0]; + if (row) { row.segs.push(s); row.size = Math.max(row.size, s.size); } else rows.push({y: s.y, size: s.size, segs: [s]}); + }); + const items = []; + const isNum = t => /^\d{1,3}$/.test(t) && parseInt(t, 10) >= 1 && parseInt(t, 10) <= N + 20; + const letters = t => new RegExp('[' + _ISSUE_L + ']{3}').test(t); + rows.forEach(r => { + const g = r.segs.sort((a, b) => a.x - b.x); + const last = g[g.length - 1]; + const textSegs = g.filter(x => !isNum(x.text)); + if (g.length >= 2 && isNum(last.text) && textSegs.length && letters(textSegs[textSegs.length - 1].text)) { + // … title | number (an author in a column before the title) + const title = textSegs[textSegs.length - 1]; + items.push({entry: true, printed: parseInt(last.text, 10), lines: [title.text], lead: false, + author: textSegs.length > 1 ? textSegs.slice(0, -1).map(x => x.text).join(' ') : '', + x: title.x, y: r.y, y0: r.y, size: title.size}); + return; + } + const tail = /^(.*\S)\s+(\d{1,3})$/.exec(last.text); + if (g.length >= 2 && tail && isNum(tail[2]) && letters(tail[1]) && letters(g[0].text)) { + // author | title 57 + items.push({entry: true, printed: parseInt(tail[2], 10), lines: [tail[1]], lead: false, + author: g.slice(0, -1).map(x => x.text).join(' '), x: last.x, y: r.y, y0: r.y, size: last.size}); + return; + } + if (g.length >= 2 && isNum(g[0].text) && letters(g[1].text)) { + items.push({entry: true, printed: parseInt(g[0].text, 10), lines: [g.slice(1).map(x => x.text).join(' ')], lead: true, + author: '', x: g[1].x, y: r.y, y0: r.y, size: g[1].size}); + return; + } + g.forEach(seg => { + let m = /^(\d{1,3})\s+(\S.*)$/.exec(seg.text); + let lead = true; + if (!m) { m = /^(.*\S)\s+(\d{1,3})$/.exec(seg.text); lead = false; } + const n = m ? parseInt(lead ? m[1] : m[2], 10) : 0; + const t = m ? (lead ? m[2] : m[1]) : ''; + if (m && n >= 1 && n <= N + 20 && letters(t)) { + items.push({entry: true, printed: n, lines: [t], lead, author: '', x: seg.x, y: seg.y, y0: seg.y, size: seg.size}); + } else items.push({entry: false, seg}); + }); + }); + return items; +} + +async function _issuePrintedToc(pdf, textOf) { + const N = pdf.numPages; + const caps = t => t.length >= 3 && t.length <= 40 && !/\d/.test(t) && + t.replace(new RegExp('[^' + _ISSUE_L + ']', 'g'), '').length >= 3 && t === t.toUpperCase(); + // The contents page: among the first pages the one with most numbered rows whose + // numbers aren't a mere count (1, 2, 3 … is a list, not a contents). + const pages = []; + for (let n = 1; n <= Math.min(16, N); n++) { + const items = _issueTocRows(await _issuePdfSegments(pdf, n), N); + const nums = items.filter(x => x.entry); + const sorted = nums.map(x => x.printed).sort((a, b) => a - b); + let steps = 0; + for (let i = 1; i < sorted.length; i++) if (sorted[i] - sorted[i - 1] === 1) steps++; + pages.push({n, items, score: nums.length * (steps > sorted.length * 0.6 ? 0.2 : 1)}); + } + let best = null; + pages.forEach(pg => { if (pg.score >= 4 && (!best || pg.score > best.score)) best = pg; }); + if (!best) return []; + // A contents running over several pages: the neighbours with entries too. + best.best = true; + const use = [best]; + for (let k = best.n; k < pages.length && pages[k].score >= 2; k++) use.push(pages[k]); + for (let k = best.n - 2; k >= 0 && pages[k].score >= 2; k--) use.unshift(pages[k]); + const isCaps = t => t === t.toUpperCase() && new RegExp('[' + _ISSUE_L + ']{3}').test(t); + const entries = []; + use.forEach(pg => { + const ents = []; + const headings = []; + const capsRun = []; // capitals just above, not used yet: a title's beginning or a heading + pg.items.forEach(it => { + if (it.entry) { + const e = it; + // A title in capitals may begin on the lines above its number ("DAS POLITISCHE FELD + // NACH LINKS / 14 VERSCHIEBEN"); a lone word on top of them is a heading (GESPRÄCH). + if (isCaps(e.lines[0])) { + const above = capsRun.filter(c => c.x >= e.x - 8 && c.x - e.x < 60 && c.y > e.y && c.y - e.y < c.size * 2.2 * (capsRun.length + 1)); + above.sort((a, b) => b.y - a.y); + let prev = e.y; + const chain = []; + for (let k = above.length - 1; k >= 0; k--) { + if (above[k].y - prev > above[k].size * 2.4) break; + chain.unshift(above[k]); + prev = above[k].y; + } + if (chain.length >= 1 && !/\s/.test(chain[0].text)) { e.kicker = chain[0].text; chain.shift(); } + chain.forEach(c => { c.used = true; }); + if (chain.length) e.lines[0] = chain.map(c => c.text).concat([e.lines[0]]).join(' ').replace(/(\w)- (?=\w)/g, '$1-'); + } + e.fromNeighbour = !pg.best; + ents.push(e); + return; + } + const s = it.seg; + if (/^\d+$/.test(s.text)) return; + if (caps(s.text)) { + const c = {y: s.y, x: s.x, t: s.text, text: s.text, size: s.size}; + headings.push(c); + capsRun.push(c); + return; + } + const e = ents.filter(x => s.x >= x.x - 8 && s.x - x.x < 60 && x.y - s.y > 0 && x.y - s.y < Math.max(x.size, s.size) * 3 && x.lines.length < 6).pop(); + if (!e) return; + const by = /^(Von|Mit|By|With)\s+(.+)$/.exec(s.text); + if (by && !e.author) { e.author = by[2]; e.y = s.y; e.done = true; return; } + if (e.done && /(,|&|\bund)$/.test(e.author)) { e.author += ' ' + s.text; e.y = s.y; return; } + if (e.done) return; + if (Math.abs(s.size - e.size) < 2.5) e.lines.push(s.text); // more title, or the author + else (e.sub = e.sub || []).push(s.text); // a line under it in other type + e.y = s.y; + }); + // Each entry's heading: the nearest capitals above it in the contents' type size (a + // title's beginning is no heading; a kicker on top of one is). + ents.forEach(e => { + const h = headings.filter(x => x.y > e.y0 && Math.abs(x.x - e.x) < 200 && x.size < e.size * 1.6 && + !x.used).sort((a, b) => a.y - b.y)[0]; + e.r = e.kicker || (h && !isCaps(e.lines[0]) ? h.t : ''); + }); + entries.push.apply(entries, ents); + }); + const sizes = entries.map(e => e.size).sort((a, b) => a - b); + const median = sizes[Math.floor(sizes.length / 2)] || 10; + for (let i = entries.length - 1; i >= 0; i--) if (entries[i].size > median * 1.6) entries.splice(i, 1); + const capsShare = entries.filter(e => isCaps(e.lines[0])).length / entries.length; + for (let i = entries.length - 1; i >= 0; i--) { + if (capsShare < 0.5 && isCaps(entries[i].lines[0]) && !/\s/.test(entries[i].lines[0])) entries.splice(i, 1); + } + if (entries.length < 3) return []; + entries.forEach(e => { + // An author on the last line: the title is the lines before it. + if (!e.author && e.lines.length > 1 && _issueNameLike(e.lines[e.lines.length - 1])) e.author = e.lines.pop(); + // "Name: Title" (manuskripte) + const m = /^([^:]{3,45}):\s+(.{3,})$/.exec(e.lines[0]); + if (!e.author && m && _issueNameLike(m[1])) { e.author = m[1]; e.lines[0] = m[2]; } + e.author = (e.author || '').replace(/\s+/g, ' ').trim(); + }); + // The printed numbers onto the file's pages: the offset most titles agree on. + const votes = {}; + const found = {}; + for (const e of entries) { + // The title as printed; where the number stands at the end of one run of text, maybe + // with a name before the title ("Marcus Verhagen Grenfell's Fire 121"). + const t = e.lines.join(' '); + const words = t.split(' '); + const cands = [t].concat(!e.lead && !e.author && words.length > 3 ? [words.slice(2).join(' '), words.slice(3).join(' ')] : []); + for (let off = -2; off <= 8; off++) { + const p = e.printed + off; + if (p < 1 || p > N) continue; + const text = await textOf(p); + const hits = cands.filter(c => _issueNorm(c).length >= 8 && _issueFind(text, c) >= 0); + const upper = hits.filter(c => { const sp = _issueSpan(text, c); return sp && /[A-ZÄÖÜ]/.test(sp) && sp === sp.toUpperCase(); }); + const hit = upper[0] || hits[0]; + if (hit) { + votes[off] = (votes[off] || 0) + 1; + (found[off] = found[off] || []).push([e, hit]); + } + } + } + const offs = Object.keys(votes).sort((a, b) => votes[b] - votes[a]); + const off = offs.length ? parseInt(offs[0], 10) : 0; + const titled = new Map((found[off] || []).map(x => [x[0], x[1]])); + const runningHead = async t => { + let n = 0; + for (const q of [Math.round(N * 0.3), Math.round(N * 0.55), Math.round(N * 0.8)]) if (_issueFind(await textOf(q), t) >= 0) n++; + return n >= 2; + }; + for (let i = entries.length - 1; i >= 0; i--) { + const e = entries[i]; + const t = e.lines.join(' '); + // On a further contents page, an entry its page doesn't confirm must look like a title: + // capitals, or a few words without a comma (not a sentence running on). + const plausible = isCaps(t) || (t.length <= 80 && !/,/.test(t) && /\s/.test(t) && /^[^a-zäöüß]/.test(t)); + if (await runningHead(t) || (e.fromNeighbour && !titled.get(e) && !plausible)) entries.splice(i, 1); + } + const out = []; + const seen = new Set(); + entries.forEach(e => { + const p = e.printed + off; + if (p < 1 || p > N) return; + const full = e.lines.join(' '); + const t = titled.get(e) || full; + let author = e.author; + let title = t; + if (!author && t !== full && full.indexOf(t) > 0) author = full.slice(0, full.indexOf(t)).trim(); + if (!author && !e.lead && !titled.get(e)) { + const w = full.split(' '); + for (const k of [2, 3]) { + if (w.length > k + 1 && _issueNameLike(w.slice(0, k).join(' '))) { author = w.slice(0, k).join(' '); title = w.slice(k).join(' '); break; } + } + } + const key = p + '|' + _issueNorm(title); + if (seen.has(key)) return; + seen.add(key); + out.push({p, t: title.replace(/\s+/g, ' ').trim(), r: (e.r || '').replace(/\s+/g, ' ').trim(), a: author, + x: e.sub ? e.sub.join(' ').replace(/(\w)- (?=[a-zäöüß])/g, '$1').replace(/\s+/g, ' ').trim() : ''}); + }); + return out; +} + // The file, decrypted (the browser's cache, or — in the apps — their store). async function _issueBytes(id) { let rec = typeof _getCachedBook === 'function' ? await _getCachedBook(id) : null; @@ -275,7 +622,7 @@ function _issueEnsureIndex(bookId) { const b = _lastBookListData.find(x => x.id === bookId); if (!b) return Promise.resolve(null); const have = _issueIndexOf(b); - if (have || (_issueMeta(b).issue && _issueMeta(b).issue.none)) return Promise.resolve(have); + if (have || _issueNone(b)) return Promise.resolve(have); if (_issueBuilding.has(bookId)) return _issueBuilding.get(bookId); const job = (async () => { const bytes = await _issueBytes(bookId); @@ -284,7 +631,7 @@ function _issueEnsureIndex(bookId) { const arts = pdf ? await _issueIndexPdf(bytes) : await _issueIndexEpub(bytes); if (!arts) return null; // An issue without a usable table of contents stays a whole (and isn't tried again). - const issue = arts.length >= 2 ? {v: 1, a: arts} : {v: 1, a: [], none: true}; + const issue = arts.length >= 2 ? {v: ISSUE_INDEX_V, a: arts} : {v: ISSUE_INDEX_V, a: [], none: true}; try { await _updateBookMeta(bookId, {issue}); } catch (e) { bookMetaCache[bookId] = Object.assign({}, bookMetaCache[bookId] || {}, {issue}); } @@ -300,8 +647,7 @@ function _issueEnsureIndex(bookId) { let _issueQueueBusy = false; function _issueIndexLater(books) { if (_issueQueueBusy || (typeof _LIB_EINK === 'function' && _LIB_EINK())) return; - const todo = books.filter(b => _isMagazine(b) && b.keyOk !== false && !_issueIndexOf(b) && - !(_issueMeta(b).issue && _issueMeta(b).issue.none)); + const todo = books.filter(b => _isMagazine(b) && b.keyOk !== false && !_issueIndexOf(b) && !_issueNone(b)); if (!todo.length) return; _issueQueueBusy = true; const idle = window.requestIdleCallback || (fn => setTimeout(fn, 1500)); @@ -325,7 +671,7 @@ function _issueView(b) { ${b.author ? `${_issueEsc(b.author)}` : ''} ${sum ? `${sum.total} Artikel · ${_issueMinutes(sum.arts.reduce((n, a) => n + (a.m || 0), 0))}` : ''}`; if (!sum) { - const none = _issueMeta(b).issue && _issueMeta(b).issue.none; + const none = _issueNone(b); return back + head + (none ? `

Dieses Heft hat kein Inhaltsverzeichnis, aus dem sich Artikel ablesen lassen – es liest sich als Ganzes.

` @@ -353,7 +699,7 @@ function _issueView(b) { `; }); return h; @@ -501,8 +847,9 @@ async function _issueReaderOpened(bookId, isPdf) { const next = _issueR.els[i + 1]; if (next && !next.hasAttribute('data-epub-src')) next.parentNode.insertBefore(aside, next); else { - const chapters = contentEl.querySelectorAll(':scope > [data-epub-src]'); - let ch = next ? next.previousElementSibling : chapters[chapters.length - 1]; + // The chapter just before the next article; the last article: its own (what follows + // it — notices, the imprint — is no part of it). + let ch = next ? next.previousElementSibling : el.closest('[data-epub-src]'); while (ch && !ch.hasAttribute('data-epub-src')) ch = ch.previousElementSibling; (ch || el.closest('[data-epub-src]') || contentEl).appendChild(aside); } diff --git a/static/js/sw.js b/static/js/sw.js index aae17be..ddaed8d 100644 --- a/static/js/sw.js +++ b/static/js/sw.js @@ -2,7 +2,7 @@ * diora service worker — caches the app shell for offline use. */ -const CACHE = 'diora-v127'; +const CACHE = 'diora-v128'; const SHELL = [ '/static/css/app.css', '/static/js/app/01-core.js',