From e3d597a3c081ee6bd6c05258f0a3cd734f7f0153 Mon Sep 17 00:00:00 2001 From: mrwnslz Date: Mon, 19 Oct 2026 17:07:51 +0200 Subject: [PATCH] Nachrichten: Lesehilfen ohne Sprachmodell (feeds/textproc.py, docs/news-text.md) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Floskeln je Quelle (Spendenboxen u. ä.) verschwinden aus dem Artikel - Überblick einer Meldung aus mehreren Quellen: Zusammenfassung (LexRank + MMR), was nur ein Bericht sagt, Wortwahl je Quelle, dieselbe Meldung in der anderen Sprache (GET /feeds/stories//overview/) - Themen-Chips über den Nachrichten (topics auf der ersten Seite, ?ids= filtert) - before= mit unkodiertem '+' gibt keinen 500 mehr - sw v133 Co-Authored-By: Claude Opus 5.5 --- docs/news-text.md | 46 ++++++ feeds/tests.py | 107 +++++++++++++- feeds/textproc.py | 353 +++++++++++++++++++++++++++++++++++++++++++++ feeds/urls.py | 1 + feeds/views.py | 118 ++++++++++++++- static/js/feeds.js | 88 ++++++++++- static/js/sw.js | 2 +- 7 files changed, 704 insertions(+), 11 deletions(-) create mode 100644 docs/news-text.md create mode 100644 feeds/textproc.py diff --git a/docs/news-text.md b/docs/news-text.md new file mode 100644 index 0000000..273c74c --- /dev/null +++ b/docs/news-text.md @@ -0,0 +1,46 @@ +# Lesehilfen für Nachrichten (feeds/textproc.py) + +Klassische Textverfahren, kein Sprachmodell; reines Python auf dem Server, für die Nachrichten +eines Nutzers auf einmal. Ausprobiert an echten Nachrichten (Oktober 2026), übernommen wurde, +was dort trug. + +## Floskeln + +Sätze, die eine Quelle unter mindestens drei ihrer letzten 40 Artikel setzt (Spendenboxen, +„Diese Nachricht wurde … gesendet“, Abo-Hinweise), gehören zu keinem Artikel. `GET +/feeds/items//` liefert den Artikel ohne die Absätze, die nur aus solchen Sätzen bestehen; der +Überblick sieht sie gar nicht erst. Je Quelle sechs Stunden zwischengespeichert. + +## Überblick einer Meldung + +`GET /feeds/stories//overview/` – nur für Meldungen aus mindestens zwei Quellen: + +- `summary`: drei Sätze aus allen Berichten, die am meisten von dem sagen, was alle sagen + (LexRank), ohne einander zu wiederholen (MMR); `[{source, text}]`; +- `novel`: je Quelle bis zu zwei Sätze, denen in keinem anderen Bericht etwas ähnelt (tf-idf-Kosinus + < 0,2), ohne die schon im Überblick; `{source: [text]}`; +- `words`: je Quelle ihre eigenen Wörter für das Ereignis (Log-Odds gegen die übrigen Berichte, mit + Dirichlet-Prior; nur großgeschriebene Formen, also Substantive und Namen); `{source: [word]}`; +- `related`: dieselbe Meldung in der anderen Sprache (Deutsch/Englisch) – Meldungen der letzten zwei + Tage, die mindestens drei seltene Namen teilen (nach idf gewichtet, Summe ≥ 17); Story-Objekte wie + in `/feeds/news/`. + +Die App zeigt das über dem Artikel als „Überblick aus N Quellen“, darunter die Meldungen in der +anderen Sprache. Android lädt die Überblicke ungelesener Meldungen mit vor (Repo.kt), damit sie auch +offline da sind. + +## Themen + +Die erste Seite von `GET /feeds/news/` hat `topics: [{label, ids}]`: Meldungen der letzten zwei +Tage, die von derselben Sache handeln, nicht nur vom selben Ereignis. Gruppiert nach Titel und +Anriss (Leader-Clustering gegen die Mittelpunkte, drei Runden). Ein Thema hat mindestens drei +Meldungen aus mindestens zwei Quellen, liegt dicht beieinander (mittlere Ähnlichkeit zum Mittelpunkt +≥ 0,3) und hat ein Etikett aus mindestens zwei großgeschriebenen Wörtern, die die meisten seiner +Meldungen teilen. Mit `category` nur der Anteil der Kategorie. `?ids=1,2,3` liefert genau diese +Meldungen (ein Thema antippen). + +## Verworfen + +Stimmung (SentiWS), steigende Begriffe, Fortsetzungsketten (Nachrichten laufen nach zwei Tagen ab), +Clickbait-Erkennung: im Versuch zu ungenau oder ohne Nutzen. Persönliche Reihenfolge: erst, wenn +genug gelesen ist, um daraus zu lernen. diff --git a/feeds/tests.py b/feeds/tests.py index aa0969c..9be6e51 100644 --- a/feeds/tests.py +++ b/feeds/tests.py @@ -4,12 +4,13 @@ from unittest import mock import feedparser from django.contrib.auth.models import User +from django.core.cache import cache from django.test import TestCase from django.utils import timezone from podcasts.views import FeedFetch -from . import cluster, fetch, logins, refresh +from . import cluster, fetch, logins, refresh, textproc from .models import FeedSettings, Item, SiteLogin, Source, Story @@ -434,3 +435,107 @@ class FolderOrderTest(TestCase): content_type='application/json') self.assertEqual(r.json()['folders'], ['Politik', 'Kunst', 'Code']) self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Politik', 'Kunst', 'Code']) + + +DONATE = 'Unterstützen Sie unabhängigen Journalismus mit einer Spende an unsere Redaktion.' +FRANCE = [ + ('Proteste in Frankreich: Gewalt bei Demonstrationen gegen Macron', 'Die Proteste in Frankreich gegen die Regierung Macron nehmen zu.'), + ('Frankreich: Proteste gegen Sparpläne weiten sich aus', 'In Frankreich wachsen die Proteste gegen die Sparpläne von Macron.'), + ('Macron verurteilt Gewalt bei Protesten in Frankreich', 'Präsident Macron verurteilt die Gewalt bei den Protesten in Frankreich.'), +] + + +class TextprocTest(TestCase): + def test_furniture_is_what_a_source_prints_under_many_articles(self): + texts = [f'Artikel Nummer {n} über etwas ganz anderes als die anderen. {DONATE}' for n in 'abcd'] + keys = textproc.furniture(texts) + self.assertEqual(textproc.without_furniture(texts[0], keys), 'Artikel Nummer a über etwas ganz anderes als die anderen.') + html = f'

Der eigentliche Text des Artikels steht hier.

{DONATE}

' + self.assertEqual(textproc.html_without_furniture(html, keys), '

Der eigentliche Text des Artikels steht hier.

') + self.assertEqual(textproc.furniture(texts[:2]), set()) # twice is no pattern + + def test_overview_of_two_reports(self): + shared = 'Die Regierung in Paris hat am Montag ein neues Sparpaket von vierzig Milliarden Euro vorgestellt. ' + a = shared + 'Gewerkschaften kündigten daraufhin landesweite Streiks für die kommende Woche an und riefen zu Demonstrationen auf.' + b = shared + 'Die Börse in Paris reagierte mit deutlichen Kursgewinnen bei den großen Banken des Landes am Nachmittag.' + docs = [('taz', a + ' Gewerkschaften Gewerkschaften'), ('FAZ', b + ' Börse Börse')] + self.assertTrue(textproc.summarize(docs, k=2)) + novel = textproc.novel(docs) + self.assertIn('Gewerkschaften', novel['taz'][0]) + self.assertIn('Börse', novel['FAZ'][0]) + base = ' Die Regierung stellte das Paket vor und die Opposition kritisierte es scharf.' * 3 + words = textproc.word_choice([('taz', base + ' Gewerkschaften streiken.' * 5), ('FAZ', base + ' Börse steigt.' * 5)]) + self.assertEqual(words, {'taz': ['Gewerkschaften'], 'FAZ': ['Börse']}) + + def test_topics_group_one_matter_from_several_sources(self): + stories = [(n, f'{t}. {s}', {src}) for n, ((t, s), src) in enumerate(zip(FRANCE, ['taz', 'nd', 'tagesschau']))] + stories.append((9, 'Bahn erhöht die Preise für das Deutschlandticket. Fahrgäste zahlen mehr.', {'taz'})) + got = textproc.topics(stories) + self.assertEqual(len(got), 1) + self.assertEqual(sorted(got[0]['ids']), [0, 1, 2]) + self.assertIn('Frankreich', got[0]['label']) + self.assertEqual(textproc.topics([(n, t, {'taz'}) for n, t, _ in stories[:3]]), []) # one source: no topic + + def test_related_story_in_the_other_language(self): + de = 'Der Papst Leo traf in Beirut den libanesischen Präsidenten Aoun und Ministerpräsident Salam.' + en = 'Pope Leo met Lebanese President Aoun and Prime Minister Salam in Beirut on Sunday.' + other = 'Die Bahn erhöht in Berlin die Preise für das Deutschlandticket deutlich.' + sl = [(1, textproc.language(de), textproc.names(de)), (2, textproc.language(en), textproc.names(en)), + (3, textproc.language(other), textproc.names(other))] + self.assertEqual((sl[0][1], sl[1][1]), ('de', 'en')) + got = textproc.related(sl, min_weight=1.0) + self.assertEqual([b for b, _ in got[1]], [2]) + self.assertNotIn(3, got) + + +class NewsTextApiTest(TestCase): + def setUp(self): + self.u = User.objects.create_user('reader', password='pw12345678') + self.client.force_login(self.u) + cache.clear() + now = timezone.now() + self.items = [] + for n, ((title, text), name) in enumerate(zip(FRANCE, ['taz', 'nd', 'tagesschau'])): + src = Source.objects.create(user=self.u, url=f'https://{name}/feed', title=name, kind='news', category='Welt') + story = Story.objects.create(user=self.u, first_at=now, last_at=now - timedelta(minutes=n)) + for k in range(4): # older articles of the source, each with the donation box + Item.objects.create(source=src, user=self.u, kind='news', guid_hash=f'{name}{k}', title=f'Alt {k}', + published=now - timedelta(days=1, minutes=k), + content=f'

Ein älterer Artikel Nummer {k} von {name}.

{DONATE}

') + i = Item.objects.create(source=src, user=self.u, kind='news', guid_hash=f'{name}x', title=title, + summary=text, published=now - timedelta(minutes=n), story=story, + content=f'

{text} Mehr dazu im Laufe des Tages von {name}.

{DONATE}

') + story.lead = i + story.save() + self.items.append(i) + + def test_first_page_has_topics_and_ids_filter(self): + r = self.client.get('/feeds/news/').json() + self.assertEqual(len(r['topics']), 1) + ids = r['topics'][0]['ids'] + self.assertEqual(len(ids), 3) + got = self.client.get('/feeds/news/?ids=' + ','.join(map(str, ids[:2]))).json() + self.assertEqual(sorted(s['id'] for s in got['stories']), sorted(ids[:2])) + self.assertNotIn('topics', got) + + def test_unencoded_plus_in_before_is_no_error(self): + when = timezone.now().isoformat() # +00:00, sent unencoded the '+' becomes a space + r = self.client.get('/feeds/news/?before=' + when.replace('+', ' ')) + self.assertEqual(r.status_code, 200) + self.assertEqual(self.client.get('/feeds/news/?before=garbled').status_code, 200) + + def test_article_comes_without_furniture(self): + got = self.client.get(f'/feeds/items/{self.items[0].id}/').json()['item'] + self.assertNotIn('Spende', got['content']) + self.assertIn('Proteste', got['content']) + + def test_overview_of_a_story_with_several_sources(self): + a, b = self.items[0], self.items[1] + b.story = a.story + b.save() + got = self.client.get(f'/feeds/stories/{a.story_id}/overview/').json() + self.assertTrue(got['summary']) + self.assertFalse(any('Spende' in x['text'] for x in got['summary'])) + self.assertEqual(got['related'], []) + single = self.client.get(f'/feeds/stories/{self.items[2].story_id}/overview/').json() + self.assertNotIn('summary', single) diff --git a/feeds/textproc.py b/feeds/textproc.py new file mode 100644 index 0000000..70872c8 --- /dev/null +++ b/feeds/textproc.py @@ -0,0 +1,353 @@ +"""Reading help for news, with classic text methods (no language model): + +- furniture: sentences a source prints under many of its articles (donation boxes, + "Diese Nachricht wurde … gesendet") are no part of an article; they are left out + when an article is shown and when a story is looked at; +- a story's overview: a summary drawn from all its reports (LexRank: the sentences + most like the others, without repeating each other — MMR), what only one report + says (its sentences unlike any other source's), and each source's own words for + the event (log-odds of a word in that report against the story's other reports); +- related stories in the other language (German/English): stories of the same days + that share names (capitalised words that aren't at a sentence's start), weighted by + how rare a name is among the day's stories; +- topics: stories grouped by their words when they're about the same matter, not + only the same event, labelled with the words they share most. + +Pure Python: these run on the server with the feeds, for one user's stories at a time. +""" + +import math +import re +from collections import Counter, defaultdict + +from . import fetch + +_SENT_END = re.compile(r'(?<=[.!?…])\s+(?=[„"»A-ZÄÖÜ0-9])') +_TOKEN = re.compile(r"[A-Za-zÄÖÜäöüßÀ-ÿ][A-Za-zÄÖÜäöüßÀ-ÿ0-9'’\-]+") + +STOP = set(''' +aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher +da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese +diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für +gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer +in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen +nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind +so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter +weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier +heute gestern morgen laut sagt sagte sagten erst mal neu video live ticker meldung bericht hatte hätte +außerdem diejenigen derjenige inzwischen zuvor derzeit dennoch trotzdem deswegen ebenfalls +wäre würde würden zudem sowohl jedoch allerdings bereits rund etwa demnach zufolge angaben unter +the and for with from that this have has are was were will would been into about after over their they +his her its our you your not but all can new says said more than when what who how why off out also had +one two which there been being some other could should would may might them then just only very +'''.split()) + +# Capitalised words that are no names (German nouns are capitalised too: those don't matter, +# they never match an English text; these would match both). +_NOT_NAMES = set(''' +Der Die Das Den Dem Des Ein Eine Einen Einem Einer Er Sie Es Wir Ihr Ich Und Oder Aber Auch Doch Mit Von +Nach Bei Für Auf Aus Im In Am Zum Zur Vor Seit Als Wie Wenn Dass Nicht Noch Nur So Da Dann Hier Jetzt +The A An And Or But With From For After Before In On At By To Of As It He She They We I This That These +Those Mr Mrs Ms Dr Monday Tuesday Wednesday Thursday Friday Saturday Sunday Montag Dienstag Mittwoch +Donnerstag Freitag Samstag Sonntag Sonnabend January February March April May June July August September +October November December Januar Februar März Juni Juli Oktober Dezember Reuters AFP AP dpa BBC CNN +'''.split()) + + +def plain(html): + return fetch.plain(html or '') + + +def sentences(text): + text = re.sub(r'\s+', ' ', text or '').strip() + return [s.strip() for s in _SENT_END.split(text) if 40 <= len(s.strip()) <= 400] + + +def _key(s): + """A sentence, the same however its numbers and spaces differ.""" + return re.sub(r'\d+', '0', re.sub(r'\s+', ' ', s.lower())).strip() + + +def words(text): + return [w.lower() for w in _TOKEN.findall(text or '') if len(w) > 2 and w.lower() not in STOP] + + +# ---- furniture ------------------------------------------------------------------------------------ + +FURNITURE_MIN = 3 # in this many of a source's articles: furniture + + +def furniture(texts): + """The sentences (keys) that stand in at least FURNITURE_MIN of these texts (one source's).""" + seen = Counter() + for t in texts: + seen.update({_key(s) for s in _units(t)}) + return {k for k, n in seen.items() if n >= FURNITURE_MIN} + + +def _units(text): + return [u.strip() for u in re.split(r'\n|(?<=[.!?])\s+', text or '') if len(u.strip()) > 15] + + +def without_furniture(text, keys): + if not keys: + return text + return ' '.join(u for u in _units(text) if _key(u) not in keys) + + +_BLOCK = re.compile(r'<(p|li|h[1-6]|blockquote|figcaption|div)\b[^>]*>(.*?)', re.S | re.I) + + +def html_without_furniture(html, keys): + """The article's HTML without the blocks whose text is (only) furniture.""" + if not keys or not html: + return html + + def keep(m): + inner = plain(m.group(2)) + units = _units(inner) + if units and all(_key(u) in keys for u in units): + return '' + return m.group(0) + return _BLOCK.sub(keep, html) + + +# ---- tf-idf over sentences ------------------------------------------------------------------------- + +def _vectors(texts): + toks = [Counter(words(t)) for t in texts] + df = Counter() + for t in toks: + df.update(t.keys()) + n = len(texts) + idf = {w: math.log((n + 1) / (d + 1)) + 1 for w, d in df.items()} + vecs = [] + for t in toks: + v = {w: (1 + math.log(c)) * idf[w] for w, c in t.items()} + norm = math.sqrt(sum(x * x for x in v.values())) or 1.0 + vecs.append({w: x / norm for w, x in v.items()}) + return vecs + + +def _cos(a, b): + if len(a) > len(b): + a, b = b, a + return sum(x * b.get(w, 0.0) for w, x in a.items()) + + +def summarize(docs, k=3, lam=0.7): + """docs: [(source, text)]. The k sentences that say most of what all say, in their order + of appearance: [(source, sentence)].""" + ss = [(src, s) for src, t in docs for s in sentences(t)[:30]] + if len(ss) <= k: + return ss + vecs = _vectors([s for _, s in ss]) + n = len(ss) + sim = [[_cos(vecs[a], vecs[b]) if a != b else 0.0 for b in range(n)] for a in range(n)] + links = [[1.0 if sim[a][b] > 0.1 else 0.0 for b in range(n)] for a in range(n)] + deg = [sum(r) or 1.0 for r in links] + rank = [1.0 / n] * n + for _ in range(40): + rank = [0.15 / n + 0.85 * sum(links[b][a] * rank[b] / deg[b] for b in range(n)) for a in range(n)] + top = max(rank) or 1.0 + chosen = [] + for _ in range(k): + best, score = None, -1e9 + for a in range(n): + if a in chosen: + continue + s = lam * rank[a] / top - (1 - lam) * max((sim[a][c] for c in chosen), default=0.0) + if s > score: + best, score = a, s + chosen.append(best) + return [ss[c] for c in sorted(chosen)] + + +def novel(docs, per_source=2, limit=0.2): + """What only one report says: per source, up to per_source of its sentences that no other + source has anything like (cosine < limit), the most central of its own first.""" + ss = [(src, s) for src, t in docs for s in sentences(t)[:40]] + if len({src for src, _ in ss}) < 2: + return {} + vecs = _vectors([s for _, s in ss]) + out = defaultdict(list) + for a, (src, s) in enumerate(ss): + others = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 != src] + if not others or max(others) >= limit: + continue + own = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 == src and b != a] + out[src].append((sum(own) / len(own) if own else 0.0, s)) + return {src: [s for _, s in sorted(v, key=lambda x: -x[0])[:per_source]] for src, v in out.items()} + + +def word_choice(docs, per_source=6): + """Each source's own words for the event: log-odds of a word in its report against the + other reports (with a prior from all of them), at least twice in its report.""" + forms = defaultdict(Counter) + counts = {} + for src, t in docs: + c = Counter() + for w in _TOKEN.findall(t or ''): + lw = w.lower() + if len(lw) > 3 and lw not in STOP: + c[lw] += 1 + forms[lw][w] += 1 + counts[src] = counts.get(src, Counter()) + c + if len(counts) < 2: + return {} + total = sum(counts.values(), Counter()) + a0 = sum(total.values()) or 1 + out = {} + for src, c in counts.items(): + n_i = sum(c.values()) or 1 + rest = total - c + n_j = sum(rest.values()) or 1 + scored = [] + for w, f in c.items(): + if f < 2: + continue + aw = total[w] / a0 * 100 + d = math.log((f + aw) / (n_i + 100 - f - aw)) - math.log((rest[w] + aw) / (n_j + 100 - rest[w] - aw)) + scored.append((d / math.sqrt(1 / (f + aw) + 1 / (rest[w] + aw)), w)) + # Nouns and names (as written: capitalised), not the verbs and particles around them. + best = [(z, forms[w].most_common(1)[0][0]) for z, w in sorted(scored, reverse=True) if z > 1.0] + out[src] = [w for z, w in best if w[:1].isupper()][:per_source] + return {k: v for k, v in out.items() if v} + + +# ---- names, languages, related stories ------------------------------------------------------------- + +_EN = set('the and of to in is that for with was said on as by at from'.split()) +_DE = set('der die das und ist nicht mit den von zu im sich auf für dem des eine sagte'.split()) + + +def language(text): + w = [x.lower() for x in _TOKEN.findall(text[:3000])] + en = sum(1 for x in w if x in _EN) + de = sum(1 for x in w if x in _DE) + return 'en' if en > de else 'de' + + +def names(text): + """Capitalised words not at a sentence's start (in German: names and nouns; in English: + names), as a set of lowercased forms.""" + out = set() + for s in sentences(text) or [text or '']: + toks = _TOKEN.findall(s) + for t in toks[1:]: + if t[0].isupper() and t not in _NOT_NAMES and len(t) > 2: + out.add(t.lower().rstrip('s’\'')) + return out + + +def related(stories, min_weight=17.0, per_story=3): + """stories: [(id, language, set of names)] of the same days. Pairs of stories in different + languages sharing rare names: {id: [(other id, weight)]}, weight = the shared names' idf.""" + df = Counter() + for _, _, ns in stories: + df.update(ns) + n = len(stories) + idf = {w: math.log((n + 1) / (d + 1)) for w, d in df.items()} + out = defaultdict(list) + for a in range(n): + ia, la, na = stories[a] + for b in range(a + 1, n): + ib, lb, nb = stories[b] + if la == lb: + continue + shared = na & nb + if len(shared) < 3: + continue + w = sum(idf[x] for x in shared) + if w >= min_weight: + out[ia].append((ib, round(w, 1))) + out[ib].append((ia, round(w, 1))) + return {k: sorted(v, key=lambda x: -x[1])[:per_story] for k, v in out.items()} + + +# ---- topics --------------------------------------------------------------------------------------------- + +NOT_TOPICS = set('breaking news free live watch video sekunden gebärdensprache podcast newsletter abo online'.split()) + + +TOPIC_MIN = 3 + + +def topics(stories, threshold=0.12, min_size=TOPIC_MIN, max_topics=8, rounds=3): + """stories: [(id, text, sources)] (titles and summaries, the set of source names). Groups of + stories about one matter, the largest first, each with a label (the words most of its stories + share): [{'label': str, 'ids': [ids]}]. Leader clustering against the groups' centroids, a + few rounds (each story to its nearest centroid, then the centroids anew); a group counts + when it has min_size stories from at least two sources.""" + if len(stories) < min_size: + return [] + vecs = _vectors([s[1] for s in stories]) + order = sorted(range(len(stories)), key=lambda i: -len(vecs[i])) + cents = [] + assign = [-1] * len(stories) + for _ in range(rounds): + changed = False + for i in order: + best, sim = -1, threshold + for g, c in enumerate(cents): + x = _cos(vecs[i], c) + if x > sim: + best, sim = g, x + if best < 0: + cents.append(dict(vecs[i])) + best = len(cents) - 1 + if assign[i] != best: + assign[i] = best + changed = True + # centroids anew from their members + acc = [Counter() for _ in cents] + for i, g in enumerate(assign): + for w, v in vecs[i].items(): + acc[g][w] += v + cents = [] + remap = {} + for g, a in enumerate(acc): + if a: + norm = math.sqrt(sum(v * v for v in a.values())) or 1.0 + remap[g] = len(cents) + cents.append({w: v / norm for w, v in a.items()}) + assign = [remap.get(g, -1) for g in assign] + if not changed: + break + groups = defaultdict(list) + for i, g in enumerate(assign): + groups[g].append(i) + out = [] + source_words = {w.lower() for s in stories for src in s[2] for w in _TOKEN.findall(src)} + for g in sorted(groups.values(), key=len, reverse=True): + if len(g) < min_size or len({src for i in g for src in stories[i][2]}) < 2: + continue + # Close enough together: the stories' mean likeness to their group's centre. + acc = Counter() + for i in g: + for w, v in vecs[i].items(): + acc[w] += v + norm = math.sqrt(sum(v * v for v in acc.values())) or 1.0 + centre = {w: v / norm for w, v in acc.items()} + if sum(_cos(vecs[i], centre) for i in g) / len(g) < 0.3: + continue + df = Counter() + for i in g: + df.update({w for w in words(stories[i][1]) if w not in NOT_TOPICS and w not in source_words}) + nice = [] + for w, c in df.most_common(8): + if c < max(2, len(g) // 2): + break + forms = Counter(t for i in g for t in _TOKEN.findall(stories[i][1]) if t.lower() == w) + form = forms.most_common(1)[0][0] if forms else w + # Nouns and names make a label, not "rückt" or "näher"; "Linke" once, not "Linken" too. + if form[:1].isupper() and not any(x.lower()[:5] == form.lower()[:5] for x in nice): + nice.append(form) + if len(nice) == 3: + break + if len(nice) < 2: # one shared word is a coincidence, not a topic + continue + out.append({'label': ' · '.join(nice), 'ids': [stories[i][0] for i in g]}) + if len(out) >= max_topics: + break + return out diff --git a/feeds/urls.py b/feeds/urls.py index c1aaf92..5edea27 100644 --- a/feeds/urls.py +++ b/feeds/urls.py @@ -15,6 +15,7 @@ urlpatterns = [ path('briefing/seen/', views.briefing_seen, name='feeds_briefing_seen'), path('news/', views.news, name='feeds_news'), path('stories//', views.story, name='feeds_story'), + path('stories//overview/', views.story_overview, name='feeds_story_overview'), path('blogs/', views.blogs, name='feeds_blogs'), path('starred/', views.starred, name='feeds_starred'), path('items/read/', views.mark_read, name='feeds_mark_read'), diff --git a/feeds/views.py b/feeds/views.py index 0068d2f..ead5a4f 100644 --- a/feeds/views.py +++ b/feeds/views.py @@ -12,8 +12,11 @@ Sources: GET /feeds/state/ sources, unread counts, settings POST /feeds/refresh/ refresh the account's sources now News: GET /feeds/briefing/ what's new since the briefing was last seen POST /feeds/briefing/seen/ seen: everything up to now counts as read - GET /feeds/news/?before=&category= stories, newest first + GET /feeds/news/?before=&category=&ids= stories, newest first (ids: just these, a topic's); + the first page also has the day's topics GET /feeds/stories// one story with all its items + GET /feeds/stories//overview/ summary, what only one report says, each source's + words, the story in the other language (feeds/textproc.py) Blogs: GET /feeds/blogs/?folder=&source=&unread=1&fav=1&before= (folder= alone: blogs without one; without folder/source the hidden blogs stay out) Items: GET /feeds/items// the article (fetches the page if the feed had a teaser) @@ -35,6 +38,7 @@ from xml.sax.saxutils import quoteattr import feedparser import requests +from django.core.cache import cache from django.db.models import Count from django.http import HttpResponse, JsonResponse from django.utils import timezone @@ -42,7 +46,7 @@ from django.utils.dateparse import parse_datetime from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods -from . import fetch, logins, refresh +from . import fetch, logins, refresh, textproc from .models import Item, SiteLogin, Source, Story PAGE = 40 @@ -71,6 +75,15 @@ def _body(request): return None +def _when(value): + """A datetime parameter; None when there's none or it's garbled (an unencoded '+' arrives as + a space).""" + try: + return parse_datetime((value or '').strip().replace(' ', '+')) + except ValueError: + return None + + def _iso(dt): return dt.isoformat() if dt else None @@ -479,14 +492,76 @@ def news(request): if (err := _auth(request)): return err qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at') - before = parse_datetime(request.GET.get('before') or '') + ids = [int(x) for x in request.GET.get('ids', '').split(',') if x.strip().isdigit()] + if ids: + stories = _stories(request.user, qs.filter(pk__in=ids[:PAGE])) + return JsonResponse({'stories': stories, 'more': False}) + before = _when(request.GET.get('before')) if before: qs = qs.filter(last_at__lt=before) category = request.GET.get('category') if category: qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category) stories = _stories(request.user, qs[:PAGE]) - return JsonResponse({'stories': stories, 'more': len(stories) == PAGE}) + out = {'stories': stories, 'more': len(stories) == PAGE} + if not before: + topics = _news_text(request.user)['topics'] + if category: + # The category's share of each topic. + mine = set(qs.values_list('pk', flat=True)[:500]) + topics = [t for t in ({**t, 'ids': [i for i in t['ids'] if i in mine]} for t in topics) + if len(t['ids']) >= textproc.TOPIC_MIN] + out['topics'] = topics + return JsonResponse(out) + + +# ---- reading help (feeds/textproc.py) + +FURNITURE_ITEMS = 40 # a source's recent articles that show its furniture +TEXT_DAYS = 2 # topics and related stories: of these days +TEXT_CACHE = 6 * 3600 + + +def _furniture(source): + key = f'feeds:furniture:{source.id}' + keys = cache.get(key) + if keys is None: + texts = [textproc.plain(c) for c in Item.objects.filter(source=source, gone=False) + .order_by('-published').values_list('content', flat=True)[:FURNITURE_ITEMS]] + keys = textproc.furniture(texts) + cache.set(key, keys, TEXT_CACHE) + return keys + + +def _clean_text(i): + """An item's text without its source's furniture, plain.""" + return textproc.without_furniture(textproc.plain(i.content) or i.summary, _furniture(i.source)) + + +def _news_text(user): + """The news of the last days, looked at as a whole: topics [{label, ids}] and related + stories in the other language {story id: [ids]}. Cached until a story changes.""" + since = timezone.now() - timedelta(days=TEXT_DAYS) + qs = Story.objects.filter(user=user, last_at__gte=since, items__gone=False).distinct() + latest = qs.order_by('-last_at').values_list('last_at', flat=True).first() + key = f'feeds:newstext:{user.id}:{qs.count()}:{latest.timestamp() if latest else 0}' + got = cache.get(key) + if got is not None: + return got + items = {} + for i in Item.objects.filter(story__in=qs, gone=False).select_related('source'): + items.setdefault(i.story_id, []).append(i) + tl, sl = [], [] + for sid, its in items.items(): + tl.append((sid, ' '.join(f'{i.title}. {i.summary}' for i in its), {i.source.title for i in its})) + text = ' '.join(f'{i.title}. {_clean_text(i)[:4000]}' for i in its) + sl.append((sid, textproc.language(text), textproc.names(text))) + got = { + 'topics': textproc.topics(tl), + 'related': {sid: [b for b, _ in lst] for sid, lst in textproc.related(sl).items()}, + } + cache.set(key, got, TEXT_CACHE) + return got @csrf_exempt @@ -500,6 +575,35 @@ def story(request, pk): return JsonResponse({'story': found[0]}) +@csrf_exempt +@require_http_methods(['GET']) +def story_overview(request, pk): + """What a story's reports say together: a summary from all of them, what only one says, + each source's own words, and the same story in the other language.""" + if (err := _auth(request)): + return err + s = Story.objects.filter(user=request.user, pk=pk).first() + if s is None: + return JsonResponse({'error': 'not found'}, status=404) + its = list(Item.objects.filter(story=s, gone=False).select_related('source').order_by('published')) + key = f'feeds:overview:{s.id}:{len(its)}:{sum(len(i.content) for i in its)}' + out = cache.get(key) + if out is None: + docs = [(i.source.title, _clean_text(i)) for i in its] + out = {} + if len({src for src, _ in docs}) >= 2: + summary = textproc.summarize(docs) + out['summary'] = [{'source': src, 'text': t} for src, t in summary] + said = {t for _, t in summary} + novel = {src: [t for t in ts if t not in said] for src, ts in textproc.novel(docs).items()} + out['novel'] = {src: ts for src, ts in novel.items() if ts} + out['words'] = textproc.word_choice(docs) + cache.set(key, out, TEXT_CACHE) + rel = _news_text(request.user)['related'].get(s.id, []) + related = _stories(request.user, Story.objects.filter(user=request.user, pk__in=rel)) if rel else [] + return JsonResponse({**out, 'related': related}) + + # ---- blogs, items @csrf_exempt @@ -522,7 +626,7 @@ def blogs(request): unread_only = request.GET.get('unread') == '1' if unread_only: qs = qs.filter(read_at=None) - before = parse_datetime(request.GET.get('before') or '') + before = _when(request.GET.get('before')) if before: qs = qs.filter(published__lt=before) # Unread: all of them at once (the page sorts them into days, favorites, rare blogs). @@ -550,7 +654,9 @@ def item(request, pk): return JsonResponse({'error': 'not found'}, status=404) if refresh.needs_article(i, i.source): refresh.load_article(i) - return JsonResponse({'item': _item_data(i, content=True)}) + d = _item_data(i, content=True) + d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source)) + return JsonResponse({'item': d}) @csrf_exempt diff --git a/static/js/feeds.js b/static/js/feeds.js index 8679d83..d2d00c4 100644 --- a/static/js/feeds.js +++ b/static/js/feeds.js @@ -21,6 +21,8 @@ briefing: null, stories: [], storiesMore: false, category: '', + topics: [], // the day's topics (/feeds/news/ first page): [{label, ids}] + topic: null, // the one shown, or null items: [], itemsMore: false, blogSource: '', blogUnread: true, blogFav: false, // main feed: only the favorite blogs @@ -87,8 +89,11 @@ async function loadNews(more) { const q = []; - if (more && st.stories.length) q.push('before=' + encodeURIComponent(st.stories[st.stories.length - 1].last_at)); - if (st.category) q.push('category=' + encodeURIComponent(st.category)); + if (st.topic) q.push('ids=' + st.topic.ids.join(',')); + else { + if (more && st.stories.length) q.push('before=' + encodeURIComponent(st.stories[st.stories.length - 1].last_at)); + if (st.category) q.push('category=' + encodeURIComponent(st.category)); + } const [news, briefing] = await Promise.all([ api('/feeds/news/' + (q.length ? '?' + q.join('&') : '')), more ? Promise.resolve(st.briefing) : api('/feeds/briefing/'), @@ -96,6 +101,7 @@ st.stories = more ? st.stories.concat(news.stories) : news.stories; st.storiesMore = news.more; st.briefing = briefing; + if (news.topics) st.topics = news.topics; } async function loadBlogs(more) { @@ -209,6 +215,14 @@ cats.map(c => '').join('') + ''; } + if (st.topics.length || st.topic) { + // Topics: stories of the last days about one matter (feeds/textproc.py). + const list = st.topic && !st.topics.some(t => t.label === st.topic.label) ? [st.topic].concat(st.topics) : st.topics; + h += '
Themen' + + list.map((t, n) => '').join('') + '
'; + st.topicList = list; + } if (!st.stories.length) return h + '

' + (st.loading ? 'Lädt …' : 'Gerade keine Nachrichten.') + '

'; let day = ''; st.stories.forEach(x => { @@ -596,6 +610,7 @@ readerItem = data.item; reader.innerHTML = readerShell(articleHtml(readerItem)); reader.scrollTop = 0; + if (readerStory) loadOverview(readerStory); progressSent = readerItem.progress || 0; if (readerItem.kind === 'news') { // News is read once opened; a blog post once read to 90 % (the server keeps where). @@ -638,6 +653,7 @@ } h += '

' + esc(i.title) + '

' + '

' + [i.author, i.source_title, when(i.published)].filter(Boolean).map(esc).join(' · ') + '

'; + if (story) h += '
' + (story.overview ? overviewHtml(story) : '') + '
'; const content = i.content || ''; if (i.image_url && content.indexOf(i.image_url) < 0) h += ''; h += '
' + (content || '

' + esc(i.summary) + '

') + '
'; @@ -661,6 +677,49 @@ return h; } + // What the story's reports say together (GET /feeds/stories//overview/, feeds/textproc.py): + // a summary from all of them, what only one says, each source's words, the same story in the + // other language. + function overviewHtml(story) { + const o = story.overview || {}; + let h = ''; + if (o.summary && o.summary.length) { + h += '
Überblick aus ' + story.sources + ' Quellen' + + o.summary.map(x => '

' + esc(x.text) + ' ' + esc(x.source) + '

').join(''); + const novel = Object.keys(o.novel || {}); + if (novel.length) { + h += '

Nur in einem Bericht

' + novel.map(src => o.novel[src].map(t => + '

' + esc(t) + ' ' + esc(src) + '

').join('')).join(''); + } + const words = Object.keys(o.words || {}); + if (words.length) { + h += '

Wortwahl

' + words.map(src => '

' + esc(src) + ': ' + + o.words[src].map(esc).join(', ') + '

').join(''); + } + h += '
'; + } + if (o.related && o.related.length) { + h += '

In der anderen Sprache

' + o.related.map(x => { + const lead = x.lead || {}; + return ''; + }).join('') + '
'; + } + return h; + } + + async function loadOverview(story) { + if (story.overview) return; + try { + story.overview = await api('/feeds/stories/' + story.id + '/overview/'); + } catch (e) { + return; // offline and never fetched: the article alone + } + const box = reader && !reader.hidden && readerStory === story && reader.querySelector('.fd-ov'); + if (box) box.innerHTML = overviewHtml(story); + } + function nextItem(i) { if (readerStory) return null; const list = readerList; @@ -688,6 +747,10 @@ const r = b.dataset.r; if (r === 'close') closeReader(); else if (r === 'item') openItem(parseInt(b.dataset.id, 10), readerStory); + else if (r === 'story') { + const other = readerStory && readerStory.overview && (readerStory.overview.related || []).find(x => x.id === parseInt(b.dataset.id, 10)); + if (other) openItem((other.lead || other.items[0]).id, other); + } else if (r === 'font') { let v = fontSize() + 2; if (v > 26) v = 14; @@ -776,7 +839,7 @@ // ---- events tab.addEventListener('click', async e => { - const t = e.target.closest('[data-view],[data-act],[data-story],[data-item],[data-cat],[data-edit],[data-folder],[data-flag],[data-edit-open]'); + const t = e.target.closest('[data-view],[data-act],[data-story],[data-item],[data-cat],[data-topic],[data-edit],[data-folder],[data-flag],[data-edit-open]'); if (!t || t.tagName === 'SELECT' || (t.tagName === 'INPUT' && t.type !== 'button')) return; if (t.dataset.view) { st.view = t.dataset.view; @@ -809,7 +872,14 @@ readerList = st.items; return openItem(parseInt(t.dataset.item, 10), null); } + if (t.dataset.topic !== undefined) { + const topic = (st.topicList || [])[parseInt(t.dataset.topic, 10)]; + st.topic = st.topic && topic && st.topic.label === topic.label ? null : topic || null; + window.scrollTo(0, 0); + return load(); + } if (t.dataset.cat !== undefined && !t.dataset.act) { + st.topic = null; st.category = t.dataset.cat; window.scrollTo(0, 0); return load(); @@ -1262,6 +1332,16 @@ '#fd-article .fd-art-body table { display: block; overflow-x: auto; border-collapse: collapse; font-size: 0.85em; }', '#fd-article .fd-art-body td, #fd-article .fd-art-body th { border: 1px solid var(--border, #333); padding: 4px 6px; }', '#fd-article .fd-sources { font-family: var(--font); }', + '#tab-feeds .fd-topics .fd-note { margin-right: 2px; }', + '#tab-feeds .fd-topics small { opacity: 0.6; }', + '#fd-article .fd-ov { font-size: 0.9em; }', + '#fd-article .fd-ov-box { margin: 0 0 18px; padding: 10px 14px; border: 1px solid var(--border, #2a2a2a); border-radius: 6px; }', + '#fd-article .fd-ov-box summary { font-family: var(--font); font-size: 12px; text-transform: uppercase; letter-spacing: 0.08em; color: var(--nui-muted, #9a9a9a); cursor: pointer; }', + '#fd-article .fd-ov p { margin: 8px 0; line-height: 1.45; }', + '#fd-article .fd-ov h4 { font-family: var(--font); font-size: 12px; font-weight: 400; text-transform: uppercase; letter-spacing: 0.08em; color: var(--nui-muted, #9a9a9a); margin: 14px 0 4px; }', + '#fd-article .fd-ov-src { font-family: var(--font); font-size: 11px; color: var(--nui-muted, #9a9a9a); white-space: nowrap; }', + '#fd-article .fd-ov-words { font-family: var(--font); font-size: 13px; }', + '#fd-article .fd-ov-rel { margin: 0 0 18px; }', '#fd-article .fd-art-foot { display: flex; flex-wrap: wrap; gap: 10px; justify-content: space-between; margin-top: 28px; font-family: var(--font); }', 'html.fd-reading #nui-dock { display: none; }', '@media (hover: hover) { #tab-feeds .fd-card:hover, #tab-feeds .fd-row:hover, #tab-feeds .fd-post:hover, #tab-feeds .fd-src:hover { background: var(--hover, #111); } }', @@ -1271,6 +1351,8 @@ 'body.eink #tab-feeds .fd-seg button.on, body.eink #tab-feeds .fd-badge { background: #000; color: #fff; }', 'body.eink .fd-row-meta, body.eink .fd-note, body.eink #tab-feeds .fd-card-sum, body.eink #tab-feeds .fd-read .fd-card-title, body.eink #tab-feeds .fd-read .fd-row-title { color: #333; }', 'body.eink #fd-article, body.eink #fd-article .fd-art-bar { background: #fff; color: #000; }', + 'body.eink #fd-article .fd-ov-box { border-color: #000; }', + 'body.eink #fd-article .fd-ov-src, body.eink #fd-article .fd-ov h4, body.eink #fd-article .fd-ov-box summary { color: #333; }', 'body.eink #fd-article .fd-art-body a, body.eink .fd-chip.on, body.eink .fd-icon.on, body.eink #tab-feeds .fd-brief-head h2 { color: #000; }', ].join('\n'); document.head.appendChild(style); diff --git a/static/js/sw.js b/static/js/sw.js index 70eccd4..7764bb1 100644 --- a/static/js/sw.js +++ b/static/js/sw.js @@ -2,7 +2,7 @@ * diora service worker — caches the app shell for offline use. */ -const CACHE = 'diora-v132'; +const CACHE = 'diora-v133'; const SHELL = [ '/static/css/app.css', '/static/js/app/01-core.js',