Zeitschriften: Artikel richtig erkennen (GegenStandpunkt, PDF-Hefte)
- EPUB: ein Artikel ist eine Datei; die Einträge danach in dieselbe Datei sind seine Abschnitte, gleich auf welcher Ebene (beim GegenStandpunkt stehen Zwischenüberschriften teils ganz oben). Unterzeile = die Schlagzeile unter dem Titel, nicht nummerierte Abschnitte. - PDF: die meisten Hefte haben keine Lesezeichen – die Artikel kommen jetzt aus dem gedruckten Inhaltsverzeichnis (Jacobin, NLR, LuXemburg, manuskripte): Seitenzahl vorn oder hinten, Autor in eigener Spalte oder darunter („Von …“), Titel in Versalien über mehrere Zeilen, Rubriken; die gedruckten Seitenzahlen werden über die Titel auf die PDF-Seiten abgebildet. - Index-Version 2: schon gebaute Indizes werden neu gebaut. sw.js: diora-v128. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
ea7d62789f
commit
3c7be76a93
3 changed files with 414 additions and 44 deletions
|
|
@ -106,13 +106,36 @@ so is each article in it. Neither is reduced to the other
|
|||
articles with their state, a teaser and the reading time; the ⋮ / mark
|
||||
sets read / unread / skipped by hand.
|
||||
|
||||
**The index** comes from the file, on the device: an EPUB's table of contents
|
||||
(its leaves are the articles, their parents the Rubriken; without one, each
|
||||
spine file with text), a PDF's outline (page ranges). Teasers and reading times
|
||||
(200 words a minute) come from the text. It is kept in the encrypted meta as
|
||||
`issue: {v: 1, a: [{h, t, r, x, m, p?, e?}]}` (href, title, Rubrik, teaser,
|
||||
minutes, a PDF's first and last page), or `{v: 1, a: [], none: true}` for an
|
||||
issue without a usable table of contents (read as a whole). Built in the
|
||||
**The index** comes from the file, on the device:
|
||||
|
||||
- **EPUB**: the table of contents, by file — an article is a file (the first
|
||||
entry pointing into it); the entries after it into the same file are its
|
||||
sections, whatever their depth (GegenStandpunkt puts some at the top level);
|
||||
an entry whose children point into other files is a Rubrik. A one-file EPUB:
|
||||
the leaves. No table of contents: each spine file with text. The teaser is
|
||||
the headline under the title (the headings before the first paragraph that
|
||||
the title doesn't say, not numbered sections), else the first paragraph.
|
||||
- **PDF**: a real outline (word processors' `_Hlk…`/`_Toc…` leftovers don't
|
||||
count); most magazines' PDFs have none, so mostly **the contents page
|
||||
printed in the issue**: among the first 16 pages the one with most numbered
|
||||
rows (numbers that merely count 1, 2, 3 … are a list, not a contents), plus
|
||||
its neighbours with entries. A row: the page number at the start
|
||||
("14 Der Krieg gegen Palästina", Jacobin), or at the end, maybe with the
|
||||
author in a column before the title ("Nathan Sperber | Beyond Neoliberalism?
|
||||
| 23", NLR); lines under it in its type size continue the title, an author
|
||||
line ("Von …", a name) gives the author, lines in other type the teaser; a
|
||||
title in capitals may begin on the lines above its number, a lone word on top
|
||||
is its kicker (LuXemburg: GESPRÄCH, INFOGRAFIK); "Name: Titel"
|
||||
(manuskripte). Capitals in the contents' type are Rubriken. The printed
|
||||
numbers are mapped to the file's pages by the offset most titles agree on
|
||||
(found in the page's text, a headline in capitals preferred); running heads
|
||||
and the contents page's own head are dropped.
|
||||
|
||||
Reading times count 200 words a minute. It is kept in the encrypted meta as
|
||||
`issue: {v: 2, a: [{h, t, r, x, m, a?, p?, e?}]}` (href, title, Rubrik, teaser,
|
||||
minutes, author, a PDF's first and last page), or `{v: 2, a: [], none: true}`
|
||||
for an issue without a usable table of contents (read as a whole). An index of
|
||||
another version is built anew. Built in the
|
||||
background for issues without one (not on e-ink: there when the issue is
|
||||
opened, or it comes from another device).
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ISSUE_WPM = 200; // words per minute, for the reading times
|
||||
const ISSUE_INDEX_V = 2; // an index of another version is built anew
|
||||
const _issuePendingKey = 'diora_issue_pending';
|
||||
|
||||
function _issueEsc(s) {
|
||||
|
|
@ -26,7 +27,12 @@ const _issueMeta = b => (b && (bookMetaCache[b.id] || {})) || {};
|
|||
function _issueIndexOf(b) {
|
||||
const m = _issueMeta(b);
|
||||
const ix = m.issue || (b && b.issue);
|
||||
return ix && ix.a && ix.a.length ? ix.a : null;
|
||||
return ix && ix.v === ISSUE_INDEX_V && ix.a && ix.a.length ? ix.a : null;
|
||||
}
|
||||
// An issue whose file has no usable table of contents (under this version's rules).
|
||||
function _issueNone(b) {
|
||||
const ix = _issueMeta(b).issue || (b && b.issue);
|
||||
return !!(ix && ix.v === ISSUE_INDEX_V && ix.none);
|
||||
}
|
||||
|
||||
// ---- state: {href: 'r' (gelesen) | 's' (übersprungen)} and the article one was in -----------
|
||||
|
|
@ -123,9 +129,39 @@ window.addEventListener('online', () => Object.keys(_issuePending()).forEach(id
|
|||
|
||||
// ---- the index -----------------------------------------------------------------------------------
|
||||
|
||||
// The articles of a table of contents: its leaves, each under the nearest entry above it
|
||||
// (the Rubrik); one per target. [{h, t, r}]
|
||||
function _issueArticlesFromToc(toc) {
|
||||
// The articles of a table of contents. An EPUB's (byFile): an article is a file — the
|
||||
// first entry pointing into it; the entries after it into the same file are its sections
|
||||
// (whatever their depth), and an entry whose children point into other files is a Rubrik.
|
||||
// Otherwise (one file for all, a PDF's outline): the leaves, each under the entry above it.
|
||||
// Entries that are word processors' leftovers (_Hlk…, _Toc…) don't count. [{h, t, r}]
|
||||
const _issueJunk = label => /^_?(Hlk|Toc|Ref|GoBack)\d*$/i.test(String(label || '').trim());
|
||||
|
||||
function _issueArticlesFromToc(toc, byFile) {
|
||||
toc = toc.filter(e => !_issueJunk(e.label));
|
||||
const fileOf = h => String(h || '').split('#')[0];
|
||||
if (byFile) {
|
||||
const out = [];
|
||||
const seen = new Set();
|
||||
for (let i = 0; i < toc.length; i++) {
|
||||
const e = toc[i];
|
||||
if (!e.href) continue;
|
||||
const f = fileOf(e.href);
|
||||
let container = false;
|
||||
for (let k = i + 1; k < toc.length && toc[k].depth > e.depth; k++) {
|
||||
if (toc[k].href && fileOf(toc[k].href) !== f) { container = true; break; }
|
||||
}
|
||||
if (container || seen.has(f)) continue;
|
||||
seen.add(f);
|
||||
let r = '';
|
||||
for (let k = i - 1, d = e.depth; k >= 0 && d > 0; k--) {
|
||||
if (toc[k].depth >= d) continue;
|
||||
d = toc[k].depth;
|
||||
if (!toc[k].href || fileOf(toc[k].href) !== f) { r = toc[k].label || ''; break; }
|
||||
}
|
||||
out.push({h: e.href, t: (e.label || '').replace(/\s+/g, ' ').trim() || 'Ohne Titel', r: r.replace(/\s+/g, ' ').trim()});
|
||||
}
|
||||
if (out.length >= 2) return out;
|
||||
}
|
||||
const out = [];
|
||||
const seen = new Set();
|
||||
const parents = [];
|
||||
|
|
@ -154,6 +190,9 @@ const _issueHtmlText = html => String(html || '')
|
|||
.replace(/<head\b[\s\S]*?<\/head>/gi, ' ')
|
||||
.replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, ' ')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/&#x([0-9a-f]+);/gi, (m, h) => String.fromCharCode(parseInt(h, 16)))
|
||||
.replace(/&#(\d+);/g, (m, d) => String.fromCharCode(parseInt(d, 10)))
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/ /g, ' ').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'");
|
||||
|
||||
async function _issueIndexEpub(bytes) {
|
||||
|
|
@ -172,7 +211,7 @@ async function _issueIndexEpub(bytes) {
|
|||
mediaType: item.getAttribute('media-type') || '', properties: item.getAttribute('properties') || ''};
|
||||
});
|
||||
const spine = Array.prototype.map.call(opfDoc.querySelectorAll('spine > itemref'), r => (manifest[r.getAttribute('idref')] || {}).href).filter(Boolean);
|
||||
let arts = _issueArticlesFromToc(await _parseEpubToc(zip, opfDoc, manifest));
|
||||
let arts = _issueArticlesFromToc(await _parseEpubToc(zip, opfDoc, manifest), true);
|
||||
const files = {};
|
||||
const fileText = async path => {
|
||||
if (!(path in files)) {
|
||||
|
|
@ -216,48 +255,356 @@ async function _issueIndexEpub(bytes) {
|
|||
for (let s = startIdx + 1; startIdx >= 0 && s < (endIdx < 0 ? spine.length : endIdx); s++) html += await fileText(spine[s]);
|
||||
if (nFile && nFrag) { const n = await fileText(nFile); html += n.slice(0, posIn(n, nFrag)); }
|
||||
}
|
||||
Object.assign(arts[i], _issueTextStats(_issueHtmlText(html), arts[i].t));
|
||||
const st = _issueTextStats(_issueHtmlText(html), arts[i].t);
|
||||
// The headings before the first paragraph that the title doesn't already say (a
|
||||
// magazine's headline under its kicker): the teaser; else the text's beginning.
|
||||
const body = html.replace(/<head\b[\s\S]*?<\/head>/i, '');
|
||||
const firstP = body.search(/<p[\s>]/i);
|
||||
const heads = [];
|
||||
const re = /<h[1-4]\b[^>]*>([\s\S]*?)<\/h[1-4]>/gi;
|
||||
let m;
|
||||
while ((m = re.exec(body)) && (firstP < 0 || m.index < firstP)) {
|
||||
const t = _issueHtmlText(m[1]).replace(/\s+/g, ' ').trim();
|
||||
// Not one of the article's own numbered sections ("I. Verarmung …", "1. …").
|
||||
if (t && !/^([IVXL]+|\d+)\.\s/.test(t) && _issueNorm(arts[i].t).indexOf(_issueNorm(t)) < 0) heads.push(t);
|
||||
}
|
||||
if (heads.length) st.x = heads.join(' – ');
|
||||
else if (firstP >= 0) st.x = _issueTextStats(_issueHtmlText(body.slice(firstP)), '').x;
|
||||
Object.assign(arts[i], st);
|
||||
}
|
||||
return arts;
|
||||
}
|
||||
|
||||
const _issueNorm = s => String(s || '').toLowerCase().replace(/ß/g, 'ss').normalize('NFKD').replace(/[^a-z0-9]/g, '');
|
||||
|
||||
async function _issueIndexPdf(bytes) {
|
||||
if (typeof pdfjsLib === 'undefined' || typeof _parsePdfOutline !== 'function') return null;
|
||||
if (typeof pdfjsLib === 'undefined') return null;
|
||||
const pdf = await pdfjsLib.getDocument({data: new Uint8Array(bytes)}).promise;
|
||||
const outline = await pdf.getOutline();
|
||||
if (!outline || !outline.length) return [];
|
||||
const arts = _issueArticlesFromToc(await _parsePdfOutline(pdf, outline))
|
||||
.filter(a => /^#pdf-page-\d+$/.test(a.h));
|
||||
arts.forEach(a => { a.p = parseInt(a.h.slice(10), 10); });
|
||||
arts.sort((a, b) => a.p - b.p);
|
||||
try {
|
||||
return await _issuePdfArticles(pdf);
|
||||
} finally {
|
||||
try { pdf.destroy(); } catch (e) {}
|
||||
}
|
||||
}
|
||||
|
||||
// A PDF's articles: from its outline where it has a real one, else from the contents
|
||||
// page printed in the issue (most magazines' PDFs have no outline).
|
||||
async function _issuePdfArticles(pdf) {
|
||||
const pageText = {};
|
||||
const textOf = async n => {
|
||||
if (!(n in pageText)) {
|
||||
try {
|
||||
const page = await pdf.getPage(n);
|
||||
const c = await page.getTextContent();
|
||||
const c = await (await pdf.getPage(n)).getTextContent();
|
||||
pageText[n] = c.items.map(it => it.str).join(' ');
|
||||
} catch (e) { pageText[n] = ''; }
|
||||
}
|
||||
return pageText[n];
|
||||
};
|
||||
for (let i = 0; i < arts.length; i++) {
|
||||
const end = i + 1 < arts.length ? Math.max(arts[i].p, arts[i + 1].p - 1) : pdf.numPages;
|
||||
arts[i].e = end;
|
||||
let text = '';
|
||||
for (let n = arts[i].p; n <= Math.min(end, arts[i].p + 30); n++) text += ' ' + await textOf(n);
|
||||
// Pages shared with the next article: counted for both (the next one starts on it).
|
||||
const s = _issueTextStats(text, arts[i].t);
|
||||
// The teaser: from where the title stands on the first page, if it does.
|
||||
const first = (await textOf(arts[i].p)).replace(/\s+/g, ' ');
|
||||
const k = first.toLowerCase().indexOf(arts[i].t.toLowerCase());
|
||||
if (k >= 0) s.x = _issueTextStats(first.slice(k + arts[i].t.length), '').x;
|
||||
Object.assign(arts[i], s);
|
||||
let arts = [];
|
||||
const outline = await pdf.getOutline().catch(() => null);
|
||||
if (outline && outline.length && typeof _parsePdfOutline === 'function') {
|
||||
arts = _issueArticlesFromToc(await _parsePdfOutline(pdf, outline))
|
||||
.filter(a => /^#pdf-page-\d+$/.test(a.h))
|
||||
.map(a => Object.assign(a, {p: parseInt(a.h.slice(10), 10)}));
|
||||
}
|
||||
if (arts.length < 3) arts = await _issuePrintedToc(pdf, textOf);
|
||||
if (arts.length < 2) return [];
|
||||
arts.sort((a, b) => a.p - b.p);
|
||||
for (let i = 0; i < arts.length; i++) {
|
||||
const a = arts[i];
|
||||
a.h = '#pdf-page-' + a.p;
|
||||
a.e = i + 1 < arts.length ? Math.max(a.p, arts[i + 1].p - 1) : pdf.numPages;
|
||||
let text = '';
|
||||
for (let n = a.p; n <= Math.min(a.e, a.p + 40); n++) text += ' ' + await textOf(n);
|
||||
const s = _issueTextStats(text, '');
|
||||
// From a printed contents page: no teaser (a page's running heads and captions come
|
||||
// first in its text); the author stands there instead.
|
||||
if (a.a !== undefined) s.x = a.x || '';
|
||||
else if (!a.x) {
|
||||
// The teaser: what follows the title on its first page, if it stands there.
|
||||
const first = (await textOf(a.p)).replace(/\s+/g, ' ');
|
||||
const k = _issueFind(first, a.t);
|
||||
s.x = _issueTextStats(k >= 0 ? first.slice(k) : '', '').x;
|
||||
} else s.x = a.x;
|
||||
Object.assign(a, s);
|
||||
}
|
||||
try { pdf.destroy(); } catch (e) {}
|
||||
return arts;
|
||||
}
|
||||
|
||||
// Where `title` stands in `text` (letters and digits only, any case): the index in
|
||||
// `text` just after it, or -1.
|
||||
function _issueFind(text, title) {
|
||||
const want = _issueNorm(title).slice(0, 40);
|
||||
if (want.length < 4) return -1;
|
||||
let norm = '';
|
||||
const map = [];
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = _issueNorm(text[i]);
|
||||
for (let k = 0; k < c.length; k++) { norm += c[k]; map.push(i + 1); }
|
||||
}
|
||||
const at = norm.indexOf(want);
|
||||
return at < 0 ? -1 : map[at + want.length - 1];
|
||||
}
|
||||
|
||||
function _issueSpan(text, title) {
|
||||
const end = _issueFind(text, title);
|
||||
if (end < 0) return '';
|
||||
const want = _issueNorm(title).slice(0, 40);
|
||||
let n = 0, i = end;
|
||||
while (i > 0 && n < want.length) { i--; n += _issueNorm(text[i]).length; }
|
||||
return text.slice(i, end);
|
||||
}
|
||||
|
||||
// The lines of a page as pdf.js lays its text out: pieces on one baseline that sit close
|
||||
// together make a segment; far apart (another column), two. [{x, y, size, text}]
|
||||
async function _issuePdfSegments(pdf, n) {
|
||||
const page = await pdf.getPage(n);
|
||||
const c = await page.getTextContent();
|
||||
const items = c.items.filter(it => it.str && it.str.trim())
|
||||
.map(it => ({x: it.transform[4], y: it.transform[5], w: it.width, size: Math.hypot(it.transform[2], it.transform[3]) || it.height || 10, s: it.str}));
|
||||
items.sort((a, b) => (b.y - a.y) || (a.x - b.x));
|
||||
const segs = [];
|
||||
let line = [];
|
||||
const flush = () => {
|
||||
if (!line.length) return;
|
||||
line.sort((a, b) => a.x - b.x);
|
||||
let seg = null;
|
||||
for (const it of line) {
|
||||
if (seg && it.x - seg.end > Math.max(24, it.size * 3)) { segs.push(seg); seg = null; }
|
||||
if (!seg) seg = {x: it.x, y: it.y, size: it.size, text: it.s, end: it.x + it.w};
|
||||
else {
|
||||
seg.text += (it.x - seg.end > it.size * 0.15 && !/\s$/.test(seg.text) && !/^\s/.test(it.s) ? ' ' : '') + it.s;
|
||||
seg.end = Math.max(seg.end, it.x + it.w);
|
||||
seg.size = Math.max(seg.size, it.size);
|
||||
}
|
||||
}
|
||||
if (seg) segs.push(seg);
|
||||
line = [];
|
||||
};
|
||||
for (const it of items) {
|
||||
if (line.length && Math.abs(line[0].y - it.y) > Math.min(line[0].size, it.size) * 0.5) flush();
|
||||
line.push(it);
|
||||
}
|
||||
flush();
|
||||
segs.forEach(s => { s.text = s.text.replace(/\s+/g, ' ').trim(); });
|
||||
return segs.filter(s => s.text);
|
||||
}
|
||||
|
||||
// The contents page printed in the issue. Its rows: a page number at the start
|
||||
// ("14 Der Krieg gegen Palästina") or at the end, maybe with the author before the title
|
||||
// in a column of its own ("Nathan Sperber | Beyond Neoliberalism? | 23"); the rows just
|
||||
// under one, indented alike in its type size, continue it (more title, the author);
|
||||
// capitals make a heading (the Rubrik). The printed numbers are matched to the file's
|
||||
// pages by finding the titles there.
|
||||
const _ISSUE_L = 'A-Za-zÀ-ÖØ-öø-ÿĀ-ſ';
|
||||
const _issueNameLike = t => t.length <= 45 &&
|
||||
/^[A-ZÀ-ÖØ-ÞĀ-ſ][A-Za-zÀ-ÖØ-öø-ÿĀ-ſ.'’-]*(\s+((und|and|&|von|van|de|der|del|y|bin)\s+)?[A-ZÀ-ÖØ-ÞĀ-ſ][A-Za-zÀ-ÖØ-öø-ÿĀ-ſ.'’-]*){1,4}$/.test(t);
|
||||
|
||||
function _issueTocRows(segs, N) {
|
||||
const rows = [];
|
||||
segs.slice().sort((a, b) => (b.y - a.y) || (a.x - b.x)).forEach(s => {
|
||||
const row = rows.filter(r => Math.abs(r.y - s.y) < Math.min(r.size, s.size) * 0.4)[0];
|
||||
if (row) { row.segs.push(s); row.size = Math.max(row.size, s.size); } else rows.push({y: s.y, size: s.size, segs: [s]});
|
||||
});
|
||||
const items = [];
|
||||
const isNum = t => /^\d{1,3}$/.test(t) && parseInt(t, 10) >= 1 && parseInt(t, 10) <= N + 20;
|
||||
const letters = t => new RegExp('[' + _ISSUE_L + ']{3}').test(t);
|
||||
rows.forEach(r => {
|
||||
const g = r.segs.sort((a, b) => a.x - b.x);
|
||||
const last = g[g.length - 1];
|
||||
const textSegs = g.filter(x => !isNum(x.text));
|
||||
if (g.length >= 2 && isNum(last.text) && textSegs.length && letters(textSegs[textSegs.length - 1].text)) {
|
||||
// … title | number (an author in a column before the title)
|
||||
const title = textSegs[textSegs.length - 1];
|
||||
items.push({entry: true, printed: parseInt(last.text, 10), lines: [title.text], lead: false,
|
||||
author: textSegs.length > 1 ? textSegs.slice(0, -1).map(x => x.text).join(' ') : '',
|
||||
x: title.x, y: r.y, y0: r.y, size: title.size});
|
||||
return;
|
||||
}
|
||||
const tail = /^(.*\S)\s+(\d{1,3})$/.exec(last.text);
|
||||
if (g.length >= 2 && tail && isNum(tail[2]) && letters(tail[1]) && letters(g[0].text)) {
|
||||
// author | title 57
|
||||
items.push({entry: true, printed: parseInt(tail[2], 10), lines: [tail[1]], lead: false,
|
||||
author: g.slice(0, -1).map(x => x.text).join(' '), x: last.x, y: r.y, y0: r.y, size: last.size});
|
||||
return;
|
||||
}
|
||||
if (g.length >= 2 && isNum(g[0].text) && letters(g[1].text)) {
|
||||
items.push({entry: true, printed: parseInt(g[0].text, 10), lines: [g.slice(1).map(x => x.text).join(' ')], lead: true,
|
||||
author: '', x: g[1].x, y: r.y, y0: r.y, size: g[1].size});
|
||||
return;
|
||||
}
|
||||
g.forEach(seg => {
|
||||
let m = /^(\d{1,3})\s+(\S.*)$/.exec(seg.text);
|
||||
let lead = true;
|
||||
if (!m) { m = /^(.*\S)\s+(\d{1,3})$/.exec(seg.text); lead = false; }
|
||||
const n = m ? parseInt(lead ? m[1] : m[2], 10) : 0;
|
||||
const t = m ? (lead ? m[2] : m[1]) : '';
|
||||
if (m && n >= 1 && n <= N + 20 && letters(t)) {
|
||||
items.push({entry: true, printed: n, lines: [t], lead, author: '', x: seg.x, y: seg.y, y0: seg.y, size: seg.size});
|
||||
} else items.push({entry: false, seg});
|
||||
});
|
||||
});
|
||||
return items;
|
||||
}
|
||||
|
||||
async function _issuePrintedToc(pdf, textOf) {
|
||||
const N = pdf.numPages;
|
||||
const caps = t => t.length >= 3 && t.length <= 40 && !/\d/.test(t) &&
|
||||
t.replace(new RegExp('[^' + _ISSUE_L + ']', 'g'), '').length >= 3 && t === t.toUpperCase();
|
||||
// The contents page: among the first pages the one with most numbered rows whose
|
||||
// numbers aren't a mere count (1, 2, 3 … is a list, not a contents).
|
||||
const pages = [];
|
||||
for (let n = 1; n <= Math.min(16, N); n++) {
|
||||
const items = _issueTocRows(await _issuePdfSegments(pdf, n), N);
|
||||
const nums = items.filter(x => x.entry);
|
||||
const sorted = nums.map(x => x.printed).sort((a, b) => a - b);
|
||||
let steps = 0;
|
||||
for (let i = 1; i < sorted.length; i++) if (sorted[i] - sorted[i - 1] === 1) steps++;
|
||||
pages.push({n, items, score: nums.length * (steps > sorted.length * 0.6 ? 0.2 : 1)});
|
||||
}
|
||||
let best = null;
|
||||
pages.forEach(pg => { if (pg.score >= 4 && (!best || pg.score > best.score)) best = pg; });
|
||||
if (!best) return [];
|
||||
// A contents running over several pages: the neighbours with entries too.
|
||||
best.best = true;
|
||||
const use = [best];
|
||||
for (let k = best.n; k < pages.length && pages[k].score >= 2; k++) use.push(pages[k]);
|
||||
for (let k = best.n - 2; k >= 0 && pages[k].score >= 2; k--) use.unshift(pages[k]);
|
||||
const isCaps = t => t === t.toUpperCase() && new RegExp('[' + _ISSUE_L + ']{3}').test(t);
|
||||
const entries = [];
|
||||
use.forEach(pg => {
|
||||
const ents = [];
|
||||
const headings = [];
|
||||
const capsRun = []; // capitals just above, not used yet: a title's beginning or a heading
|
||||
pg.items.forEach(it => {
|
||||
if (it.entry) {
|
||||
const e = it;
|
||||
// A title in capitals may begin on the lines above its number ("DAS POLITISCHE FELD
|
||||
// NACH LINKS / 14 VERSCHIEBEN"); a lone word on top of them is a heading (GESPRÄCH).
|
||||
if (isCaps(e.lines[0])) {
|
||||
const above = capsRun.filter(c => c.x >= e.x - 8 && c.x - e.x < 60 && c.y > e.y && c.y - e.y < c.size * 2.2 * (capsRun.length + 1));
|
||||
above.sort((a, b) => b.y - a.y);
|
||||
let prev = e.y;
|
||||
const chain = [];
|
||||
for (let k = above.length - 1; k >= 0; k--) {
|
||||
if (above[k].y - prev > above[k].size * 2.4) break;
|
||||
chain.unshift(above[k]);
|
||||
prev = above[k].y;
|
||||
}
|
||||
if (chain.length >= 1 && !/\s/.test(chain[0].text)) { e.kicker = chain[0].text; chain.shift(); }
|
||||
chain.forEach(c => { c.used = true; });
|
||||
if (chain.length) e.lines[0] = chain.map(c => c.text).concat([e.lines[0]]).join(' ').replace(/(\w)- (?=\w)/g, '$1-');
|
||||
}
|
||||
e.fromNeighbour = !pg.best;
|
||||
ents.push(e);
|
||||
return;
|
||||
}
|
||||
const s = it.seg;
|
||||
if (/^\d+$/.test(s.text)) return;
|
||||
if (caps(s.text)) {
|
||||
const c = {y: s.y, x: s.x, t: s.text, text: s.text, size: s.size};
|
||||
headings.push(c);
|
||||
capsRun.push(c);
|
||||
return;
|
||||
}
|
||||
const e = ents.filter(x => s.x >= x.x - 8 && s.x - x.x < 60 && x.y - s.y > 0 && x.y - s.y < Math.max(x.size, s.size) * 3 && x.lines.length < 6).pop();
|
||||
if (!e) return;
|
||||
const by = /^(Von|Mit|By|With)\s+(.+)$/.exec(s.text);
|
||||
if (by && !e.author) { e.author = by[2]; e.y = s.y; e.done = true; return; }
|
||||
if (e.done && /(,|&|\bund)$/.test(e.author)) { e.author += ' ' + s.text; e.y = s.y; return; }
|
||||
if (e.done) return;
|
||||
if (Math.abs(s.size - e.size) < 2.5) e.lines.push(s.text); // more title, or the author
|
||||
else (e.sub = e.sub || []).push(s.text); // a line under it in other type
|
||||
e.y = s.y;
|
||||
});
|
||||
// Each entry's heading: the nearest capitals above it in the contents' type size (a
|
||||
// title's beginning is no heading; a kicker on top of one is).
|
||||
ents.forEach(e => {
|
||||
const h = headings.filter(x => x.y > e.y0 && Math.abs(x.x - e.x) < 200 && x.size < e.size * 1.6 &&
|
||||
!x.used).sort((a, b) => a.y - b.y)[0];
|
||||
e.r = e.kicker || (h && !isCaps(e.lines[0]) ? h.t : '');
|
||||
});
|
||||
entries.push.apply(entries, ents);
|
||||
});
|
||||
const sizes = entries.map(e => e.size).sort((a, b) => a - b);
|
||||
const median = sizes[Math.floor(sizes.length / 2)] || 10;
|
||||
for (let i = entries.length - 1; i >= 0; i--) if (entries[i].size > median * 1.6) entries.splice(i, 1);
|
||||
const capsShare = entries.filter(e => isCaps(e.lines[0])).length / entries.length;
|
||||
for (let i = entries.length - 1; i >= 0; i--) {
|
||||
if (capsShare < 0.5 && isCaps(entries[i].lines[0]) && !/\s/.test(entries[i].lines[0])) entries.splice(i, 1);
|
||||
}
|
||||
if (entries.length < 3) return [];
|
||||
entries.forEach(e => {
|
||||
// An author on the last line: the title is the lines before it.
|
||||
if (!e.author && e.lines.length > 1 && _issueNameLike(e.lines[e.lines.length - 1])) e.author = e.lines.pop();
|
||||
// "Name: Title" (manuskripte)
|
||||
const m = /^([^:]{3,45}):\s+(.{3,})$/.exec(e.lines[0]);
|
||||
if (!e.author && m && _issueNameLike(m[1])) { e.author = m[1]; e.lines[0] = m[2]; }
|
||||
e.author = (e.author || '').replace(/\s+/g, ' ').trim();
|
||||
});
|
||||
// The printed numbers onto the file's pages: the offset most titles agree on.
|
||||
const votes = {};
|
||||
const found = {};
|
||||
for (const e of entries) {
|
||||
// The title as printed; where the number stands at the end of one run of text, maybe
|
||||
// with a name before the title ("Marcus Verhagen Grenfell's Fire 121").
|
||||
const t = e.lines.join(' ');
|
||||
const words = t.split(' ');
|
||||
const cands = [t].concat(!e.lead && !e.author && words.length > 3 ? [words.slice(2).join(' '), words.slice(3).join(' ')] : []);
|
||||
for (let off = -2; off <= 8; off++) {
|
||||
const p = e.printed + off;
|
||||
if (p < 1 || p > N) continue;
|
||||
const text = await textOf(p);
|
||||
const hits = cands.filter(c => _issueNorm(c).length >= 8 && _issueFind(text, c) >= 0);
|
||||
const upper = hits.filter(c => { const sp = _issueSpan(text, c); return sp && /[A-ZÄÖÜ]/.test(sp) && sp === sp.toUpperCase(); });
|
||||
const hit = upper[0] || hits[0];
|
||||
if (hit) {
|
||||
votes[off] = (votes[off] || 0) + 1;
|
||||
(found[off] = found[off] || []).push([e, hit]);
|
||||
}
|
||||
}
|
||||
}
|
||||
const offs = Object.keys(votes).sort((a, b) => votes[b] - votes[a]);
|
||||
const off = offs.length ? parseInt(offs[0], 10) : 0;
|
||||
const titled = new Map((found[off] || []).map(x => [x[0], x[1]]));
|
||||
const runningHead = async t => {
|
||||
let n = 0;
|
||||
for (const q of [Math.round(N * 0.3), Math.round(N * 0.55), Math.round(N * 0.8)]) if (_issueFind(await textOf(q), t) >= 0) n++;
|
||||
return n >= 2;
|
||||
};
|
||||
for (let i = entries.length - 1; i >= 0; i--) {
|
||||
const e = entries[i];
|
||||
const t = e.lines.join(' ');
|
||||
// On a further contents page, an entry its page doesn't confirm must look like a title:
|
||||
// capitals, or a few words without a comma (not a sentence running on).
|
||||
const plausible = isCaps(t) || (t.length <= 80 && !/,/.test(t) && /\s/.test(t) && /^[^a-zäöüß]/.test(t));
|
||||
if (await runningHead(t) || (e.fromNeighbour && !titled.get(e) && !plausible)) entries.splice(i, 1);
|
||||
}
|
||||
const out = [];
|
||||
const seen = new Set();
|
||||
entries.forEach(e => {
|
||||
const p = e.printed + off;
|
||||
if (p < 1 || p > N) return;
|
||||
const full = e.lines.join(' ');
|
||||
const t = titled.get(e) || full;
|
||||
let author = e.author;
|
||||
let title = t;
|
||||
if (!author && t !== full && full.indexOf(t) > 0) author = full.slice(0, full.indexOf(t)).trim();
|
||||
if (!author && !e.lead && !titled.get(e)) {
|
||||
const w = full.split(' ');
|
||||
for (const k of [2, 3]) {
|
||||
if (w.length > k + 1 && _issueNameLike(w.slice(0, k).join(' '))) { author = w.slice(0, k).join(' '); title = w.slice(k).join(' '); break; }
|
||||
}
|
||||
}
|
||||
const key = p + '|' + _issueNorm(title);
|
||||
if (seen.has(key)) return;
|
||||
seen.add(key);
|
||||
out.push({p, t: title.replace(/\s+/g, ' ').trim(), r: (e.r || '').replace(/\s+/g, ' ').trim(), a: author,
|
||||
x: e.sub ? e.sub.join(' ').replace(/(\w)- (?=[a-zäöüß])/g, '$1').replace(/\s+/g, ' ').trim() : ''});
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
// The file, decrypted (the browser's cache, or — in the apps — their store).
|
||||
async function _issueBytes(id) {
|
||||
let rec = typeof _getCachedBook === 'function' ? await _getCachedBook(id) : null;
|
||||
|
|
@ -275,7 +622,7 @@ function _issueEnsureIndex(bookId) {
|
|||
const b = _lastBookListData.find(x => x.id === bookId);
|
||||
if (!b) return Promise.resolve(null);
|
||||
const have = _issueIndexOf(b);
|
||||
if (have || (_issueMeta(b).issue && _issueMeta(b).issue.none)) return Promise.resolve(have);
|
||||
if (have || _issueNone(b)) return Promise.resolve(have);
|
||||
if (_issueBuilding.has(bookId)) return _issueBuilding.get(bookId);
|
||||
const job = (async () => {
|
||||
const bytes = await _issueBytes(bookId);
|
||||
|
|
@ -284,7 +631,7 @@ function _issueEnsureIndex(bookId) {
|
|||
const arts = pdf ? await _issueIndexPdf(bytes) : await _issueIndexEpub(bytes);
|
||||
if (!arts) return null;
|
||||
// An issue without a usable table of contents stays a whole (and isn't tried again).
|
||||
const issue = arts.length >= 2 ? {v: 1, a: arts} : {v: 1, a: [], none: true};
|
||||
const issue = arts.length >= 2 ? {v: ISSUE_INDEX_V, a: arts} : {v: ISSUE_INDEX_V, a: [], none: true};
|
||||
try { await _updateBookMeta(bookId, {issue}); } catch (e) {
|
||||
bookMetaCache[bookId] = Object.assign({}, bookMetaCache[bookId] || {}, {issue});
|
||||
}
|
||||
|
|
@ -300,8 +647,7 @@ function _issueEnsureIndex(bookId) {
|
|||
let _issueQueueBusy = false;
|
||||
function _issueIndexLater(books) {
|
||||
if (_issueQueueBusy || (typeof _LIB_EINK === 'function' && _LIB_EINK())) return;
|
||||
const todo = books.filter(b => _isMagazine(b) && b.keyOk !== false && !_issueIndexOf(b) &&
|
||||
!(_issueMeta(b).issue && _issueMeta(b).issue.none));
|
||||
const todo = books.filter(b => _isMagazine(b) && b.keyOk !== false && !_issueIndexOf(b) && !_issueNone(b));
|
||||
if (!todo.length) return;
|
||||
_issueQueueBusy = true;
|
||||
const idle = window.requestIdleCallback || (fn => setTimeout(fn, 1500));
|
||||
|
|
@ -325,7 +671,7 @@ function _issueView(b) {
|
|||
${b.author ? `<span class="lib-meta">${_issueEsc(b.author)}</span>` : ''}
|
||||
${sum ? `<span class="lib-meta">${sum.total} Artikel · ${_issueMinutes(sum.arts.reduce((n, a) => n + (a.m || 0), 0))}</span>` : ''}</span></div>`;
|
||||
if (!sum) {
|
||||
const none = _issueMeta(b).issue && _issueMeta(b).issue.none;
|
||||
const none = _issueNone(b);
|
||||
return back + head + (none
|
||||
? `<p class="lib-empty">Dieses Heft hat kein Inhaltsverzeichnis, aus dem sich Artikel ablesen lassen – es liest sich als Ganzes.</p>
|
||||
<button type="button" class="iss-go" data-issue-act="read">▶ Lesen</button>`
|
||||
|
|
@ -353,7 +699,7 @@ function _issueView(b) {
|
|||
<button type="button" class="iss-art" data-issue-art="${i}">
|
||||
<span class="iss-n">${i + 1}</span>
|
||||
<span class="iss-art-text"><span class="iss-art-title">${_issueEsc(a.t)}</span>
|
||||
${a.x ? `<span class="iss-teaser">${_issueEsc(a.x)}</span>` : ''}</span>
|
||||
${a.a || a.x ? `<span class="iss-teaser">${_issueEsc([a.a, a.x].filter(Boolean).join(' · '))}</span>` : ''}</span>
|
||||
<span class="iss-min">${a.m ? a.m + ' min' : ''}</span></button></div>`;
|
||||
});
|
||||
return h;
|
||||
|
|
@ -501,8 +847,9 @@ async function _issueReaderOpened(bookId, isPdf) {
|
|||
const next = _issueR.els[i + 1];
|
||||
if (next && !next.hasAttribute('data-epub-src')) next.parentNode.insertBefore(aside, next);
|
||||
else {
|
||||
const chapters = contentEl.querySelectorAll(':scope > [data-epub-src]');
|
||||
let ch = next ? next.previousElementSibling : chapters[chapters.length - 1];
|
||||
// The chapter just before the next article; the last article: its own (what follows
|
||||
// it — notices, the imprint — is no part of it).
|
||||
let ch = next ? next.previousElementSibling : el.closest('[data-epub-src]');
|
||||
while (ch && !ch.hasAttribute('data-epub-src')) ch = ch.previousElementSibling;
|
||||
(ch || el.closest('[data-epub-src]') || contentEl).appendChild(aside);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* diora service worker — caches the app shell for offline use.
|
||||
*/
|
||||
|
||||
const CACHE = 'diora-v127';
|
||||
const CACHE = 'diora-v128';
|
||||
const SHELL = [
|
||||
'/static/css/app.css',
|
||||
'/static/js/app/01-core.js',
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue