mrwnslz
e3d597a3c0
- Floskeln je Quelle (Spendenboxen u. ä.) verschwinden aus dem Artikel - Überblick einer Meldung aus mehreren Quellen: Zusammenfassung (LexRank + MMR), was nur ein Bericht sagt, Wortwahl je Quelle, dieselbe Meldung in der anderen Sprache (GET /feeds/stories/<id>/overview/) - Themen-Chips über den Nachrichten (topics auf der ersten Seite, ?ids= filtert) - before= mit unkodiertem '+' gibt keinen 500 mehr - sw v133 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
353 lines
15 KiB
Python
353 lines
15 KiB
Python
"""Reading help for news, with classic text methods (no language model):
|
||
|
||
- furniture: sentences a source prints under many of its articles (donation boxes,
|
||
"Diese Nachricht wurde … gesendet") are no part of an article; they are left out
|
||
when an article is shown and when a story is looked at;
|
||
- a story's overview: a summary drawn from all its reports (LexRank: the sentences
|
||
most like the others, without repeating each other — MMR), what only one report
|
||
says (its sentences unlike any other source's), and each source's own words for
|
||
the event (log-odds of a word in that report against the story's other reports);
|
||
- related stories in the other language (German/English): stories of the same days
|
||
that share names (capitalised words that aren't at a sentence's start), weighted by
|
||
how rare a name is among the day's stories;
|
||
- topics: stories grouped by their words when they're about the same matter, not
|
||
only the same event, labelled with the words they share most.
|
||
|
||
Pure Python: these run on the server with the feeds, for one user's stories at a time.
|
||
"""
|
||
|
||
import math
|
||
import re
|
||
from collections import Counter, defaultdict
|
||
|
||
from . import fetch
|
||
|
||
_SENT_END = re.compile(r'(?<=[.!?…])\s+(?=[„"»A-ZÄÖÜ0-9])')
|
||
_TOKEN = re.compile(r"[A-Za-zÄÖÜäöüßÀ-ÿ][A-Za-zÄÖÜäöüßÀ-ÿ0-9'’\-]+")
|
||
|
||
STOP = set('''
|
||
aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher
|
||
da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese
|
||
diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für
|
||
gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer
|
||
in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen
|
||
nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind
|
||
so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter
|
||
weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier
|
||
heute gestern morgen laut sagt sagte sagten erst mal neu video live ticker meldung bericht hatte hätte
|
||
außerdem diejenigen derjenige inzwischen zuvor derzeit dennoch trotzdem deswegen ebenfalls
|
||
wäre würde würden zudem sowohl jedoch allerdings bereits rund etwa demnach zufolge angaben unter
|
||
the and for with from that this have has are was were will would been into about after over their they
|
||
his her its our you your not but all can new says said more than when what who how why off out also had
|
||
one two which there been being some other could should would may might them then just only very
|
||
'''.split())
|
||
|
||
# Capitalised words that are no names (German nouns are capitalised too: those don't matter,
|
||
# they never match an English text; these would match both).
|
||
_NOT_NAMES = set('''
|
||
Der Die Das Den Dem Des Ein Eine Einen Einem Einer Er Sie Es Wir Ihr Ich Und Oder Aber Auch Doch Mit Von
|
||
Nach Bei Für Auf Aus Im In Am Zum Zur Vor Seit Als Wie Wenn Dass Nicht Noch Nur So Da Dann Hier Jetzt
|
||
The A An And Or But With From For After Before In On At By To Of As It He She They We I This That These
|
||
Those Mr Mrs Ms Dr Monday Tuesday Wednesday Thursday Friday Saturday Sunday Montag Dienstag Mittwoch
|
||
Donnerstag Freitag Samstag Sonntag Sonnabend January February March April May June July August September
|
||
October November December Januar Februar März Juni Juli Oktober Dezember Reuters AFP AP dpa BBC CNN
|
||
'''.split())
|
||
|
||
|
||
def plain(html):
|
||
return fetch.plain(html or '')
|
||
|
||
|
||
def sentences(text):
|
||
text = re.sub(r'\s+', ' ', text or '').strip()
|
||
return [s.strip() for s in _SENT_END.split(text) if 40 <= len(s.strip()) <= 400]
|
||
|
||
|
||
def _key(s):
|
||
"""A sentence, the same however its numbers and spaces differ."""
|
||
return re.sub(r'\d+', '0', re.sub(r'\s+', ' ', s.lower())).strip()
|
||
|
||
|
||
def words(text):
|
||
return [w.lower() for w in _TOKEN.findall(text or '') if len(w) > 2 and w.lower() not in STOP]
|
||
|
||
|
||
# ---- furniture ------------------------------------------------------------------------------------
|
||
|
||
FURNITURE_MIN = 3 # in this many of a source's articles: furniture
|
||
|
||
|
||
def furniture(texts):
|
||
"""The sentences (keys) that stand in at least FURNITURE_MIN of these texts (one source's)."""
|
||
seen = Counter()
|
||
for t in texts:
|
||
seen.update({_key(s) for s in _units(t)})
|
||
return {k for k, n in seen.items() if n >= FURNITURE_MIN}
|
||
|
||
|
||
def _units(text):
|
||
return [u.strip() for u in re.split(r'\n|(?<=[.!?])\s+', text or '') if len(u.strip()) > 15]
|
||
|
||
|
||
def without_furniture(text, keys):
|
||
if not keys:
|
||
return text
|
||
return ' '.join(u for u in _units(text) if _key(u) not in keys)
|
||
|
||
|
||
_BLOCK = re.compile(r'<(p|li|h[1-6]|blockquote|figcaption|div)\b[^>]*>(.*?)</\1>', re.S | re.I)
|
||
|
||
|
||
def html_without_furniture(html, keys):
|
||
"""The article's HTML without the blocks whose text is (only) furniture."""
|
||
if not keys or not html:
|
||
return html
|
||
|
||
def keep(m):
|
||
inner = plain(m.group(2))
|
||
units = _units(inner)
|
||
if units and all(_key(u) in keys for u in units):
|
||
return ''
|
||
return m.group(0)
|
||
return _BLOCK.sub(keep, html)
|
||
|
||
|
||
# ---- tf-idf over sentences -------------------------------------------------------------------------
|
||
|
||
def _vectors(texts):
|
||
toks = [Counter(words(t)) for t in texts]
|
||
df = Counter()
|
||
for t in toks:
|
||
df.update(t.keys())
|
||
n = len(texts)
|
||
idf = {w: math.log((n + 1) / (d + 1)) + 1 for w, d in df.items()}
|
||
vecs = []
|
||
for t in toks:
|
||
v = {w: (1 + math.log(c)) * idf[w] for w, c in t.items()}
|
||
norm = math.sqrt(sum(x * x for x in v.values())) or 1.0
|
||
vecs.append({w: x / norm for w, x in v.items()})
|
||
return vecs
|
||
|
||
|
||
def _cos(a, b):
|
||
if len(a) > len(b):
|
||
a, b = b, a
|
||
return sum(x * b.get(w, 0.0) for w, x in a.items())
|
||
|
||
|
||
def summarize(docs, k=3, lam=0.7):
|
||
"""docs: [(source, text)]. The k sentences that say most of what all say, in their order
|
||
of appearance: [(source, sentence)]."""
|
||
ss = [(src, s) for src, t in docs for s in sentences(t)[:30]]
|
||
if len(ss) <= k:
|
||
return ss
|
||
vecs = _vectors([s for _, s in ss])
|
||
n = len(ss)
|
||
sim = [[_cos(vecs[a], vecs[b]) if a != b else 0.0 for b in range(n)] for a in range(n)]
|
||
links = [[1.0 if sim[a][b] > 0.1 else 0.0 for b in range(n)] for a in range(n)]
|
||
deg = [sum(r) or 1.0 for r in links]
|
||
rank = [1.0 / n] * n
|
||
for _ in range(40):
|
||
rank = [0.15 / n + 0.85 * sum(links[b][a] * rank[b] / deg[b] for b in range(n)) for a in range(n)]
|
||
top = max(rank) or 1.0
|
||
chosen = []
|
||
for _ in range(k):
|
||
best, score = None, -1e9
|
||
for a in range(n):
|
||
if a in chosen:
|
||
continue
|
||
s = lam * rank[a] / top - (1 - lam) * max((sim[a][c] for c in chosen), default=0.0)
|
||
if s > score:
|
||
best, score = a, s
|
||
chosen.append(best)
|
||
return [ss[c] for c in sorted(chosen)]
|
||
|
||
|
||
def novel(docs, per_source=2, limit=0.2):
|
||
"""What only one report says: per source, up to per_source of its sentences that no other
|
||
source has anything like (cosine < limit), the most central of its own first."""
|
||
ss = [(src, s) for src, t in docs for s in sentences(t)[:40]]
|
||
if len({src for src, _ in ss}) < 2:
|
||
return {}
|
||
vecs = _vectors([s for _, s in ss])
|
||
out = defaultdict(list)
|
||
for a, (src, s) in enumerate(ss):
|
||
others = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 != src]
|
||
if not others or max(others) >= limit:
|
||
continue
|
||
own = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 == src and b != a]
|
||
out[src].append((sum(own) / len(own) if own else 0.0, s))
|
||
return {src: [s for _, s in sorted(v, key=lambda x: -x[0])[:per_source]] for src, v in out.items()}
|
||
|
||
|
||
def word_choice(docs, per_source=6):
|
||
"""Each source's own words for the event: log-odds of a word in its report against the
|
||
other reports (with a prior from all of them), at least twice in its report."""
|
||
forms = defaultdict(Counter)
|
||
counts = {}
|
||
for src, t in docs:
|
||
c = Counter()
|
||
for w in _TOKEN.findall(t or ''):
|
||
lw = w.lower()
|
||
if len(lw) > 3 and lw not in STOP:
|
||
c[lw] += 1
|
||
forms[lw][w] += 1
|
||
counts[src] = counts.get(src, Counter()) + c
|
||
if len(counts) < 2:
|
||
return {}
|
||
total = sum(counts.values(), Counter())
|
||
a0 = sum(total.values()) or 1
|
||
out = {}
|
||
for src, c in counts.items():
|
||
n_i = sum(c.values()) or 1
|
||
rest = total - c
|
||
n_j = sum(rest.values()) or 1
|
||
scored = []
|
||
for w, f in c.items():
|
||
if f < 2:
|
||
continue
|
||
aw = total[w] / a0 * 100
|
||
d = math.log((f + aw) / (n_i + 100 - f - aw)) - math.log((rest[w] + aw) / (n_j + 100 - rest[w] - aw))
|
||
scored.append((d / math.sqrt(1 / (f + aw) + 1 / (rest[w] + aw)), w))
|
||
# Nouns and names (as written: capitalised), not the verbs and particles around them.
|
||
best = [(z, forms[w].most_common(1)[0][0]) for z, w in sorted(scored, reverse=True) if z > 1.0]
|
||
out[src] = [w for z, w in best if w[:1].isupper()][:per_source]
|
||
return {k: v for k, v in out.items() if v}
|
||
|
||
|
||
# ---- names, languages, related stories -------------------------------------------------------------
|
||
|
||
_EN = set('the and of to in is that for with was said on as by at from'.split())
|
||
_DE = set('der die das und ist nicht mit den von zu im sich auf für dem des eine sagte'.split())
|
||
|
||
|
||
def language(text):
|
||
w = [x.lower() for x in _TOKEN.findall(text[:3000])]
|
||
en = sum(1 for x in w if x in _EN)
|
||
de = sum(1 for x in w if x in _DE)
|
||
return 'en' if en > de else 'de'
|
||
|
||
|
||
def names(text):
|
||
"""Capitalised words not at a sentence's start (in German: names and nouns; in English:
|
||
names), as a set of lowercased forms."""
|
||
out = set()
|
||
for s in sentences(text) or [text or '']:
|
||
toks = _TOKEN.findall(s)
|
||
for t in toks[1:]:
|
||
if t[0].isupper() and t not in _NOT_NAMES and len(t) > 2:
|
||
out.add(t.lower().rstrip('s’\''))
|
||
return out
|
||
|
||
|
||
def related(stories, min_weight=17.0, per_story=3):
|
||
"""stories: [(id, language, set of names)] of the same days. Pairs of stories in different
|
||
languages sharing rare names: {id: [(other id, weight)]}, weight = the shared names' idf."""
|
||
df = Counter()
|
||
for _, _, ns in stories:
|
||
df.update(ns)
|
||
n = len(stories)
|
||
idf = {w: math.log((n + 1) / (d + 1)) for w, d in df.items()}
|
||
out = defaultdict(list)
|
||
for a in range(n):
|
||
ia, la, na = stories[a]
|
||
for b in range(a + 1, n):
|
||
ib, lb, nb = stories[b]
|
||
if la == lb:
|
||
continue
|
||
shared = na & nb
|
||
if len(shared) < 3:
|
||
continue
|
||
w = sum(idf[x] for x in shared)
|
||
if w >= min_weight:
|
||
out[ia].append((ib, round(w, 1)))
|
||
out[ib].append((ia, round(w, 1)))
|
||
return {k: sorted(v, key=lambda x: -x[1])[:per_story] for k, v in out.items()}
|
||
|
||
|
||
# ---- topics ---------------------------------------------------------------------------------------------
|
||
|
||
NOT_TOPICS = set('breaking news free live watch video sekunden gebärdensprache podcast newsletter abo online'.split())
|
||
|
||
|
||
TOPIC_MIN = 3
|
||
|
||
|
||
def topics(stories, threshold=0.12, min_size=TOPIC_MIN, max_topics=8, rounds=3):
|
||
"""stories: [(id, text, sources)] (titles and summaries, the set of source names). Groups of
|
||
stories about one matter, the largest first, each with a label (the words most of its stories
|
||
share): [{'label': str, 'ids': [ids]}]. Leader clustering against the groups' centroids, a
|
||
few rounds (each story to its nearest centroid, then the centroids anew); a group counts
|
||
when it has min_size stories from at least two sources."""
|
||
if len(stories) < min_size:
|
||
return []
|
||
vecs = _vectors([s[1] for s in stories])
|
||
order = sorted(range(len(stories)), key=lambda i: -len(vecs[i]))
|
||
cents = []
|
||
assign = [-1] * len(stories)
|
||
for _ in range(rounds):
|
||
changed = False
|
||
for i in order:
|
||
best, sim = -1, threshold
|
||
for g, c in enumerate(cents):
|
||
x = _cos(vecs[i], c)
|
||
if x > sim:
|
||
best, sim = g, x
|
||
if best < 0:
|
||
cents.append(dict(vecs[i]))
|
||
best = len(cents) - 1
|
||
if assign[i] != best:
|
||
assign[i] = best
|
||
changed = True
|
||
# centroids anew from their members
|
||
acc = [Counter() for _ in cents]
|
||
for i, g in enumerate(assign):
|
||
for w, v in vecs[i].items():
|
||
acc[g][w] += v
|
||
cents = []
|
||
remap = {}
|
||
for g, a in enumerate(acc):
|
||
if a:
|
||
norm = math.sqrt(sum(v * v for v in a.values())) or 1.0
|
||
remap[g] = len(cents)
|
||
cents.append({w: v / norm for w, v in a.items()})
|
||
assign = [remap.get(g, -1) for g in assign]
|
||
if not changed:
|
||
break
|
||
groups = defaultdict(list)
|
||
for i, g in enumerate(assign):
|
||
groups[g].append(i)
|
||
out = []
|
||
source_words = {w.lower() for s in stories for src in s[2] for w in _TOKEN.findall(src)}
|
||
for g in sorted(groups.values(), key=len, reverse=True):
|
||
if len(g) < min_size or len({src for i in g for src in stories[i][2]}) < 2:
|
||
continue
|
||
# Close enough together: the stories' mean likeness to their group's centre.
|
||
acc = Counter()
|
||
for i in g:
|
||
for w, v in vecs[i].items():
|
||
acc[w] += v
|
||
norm = math.sqrt(sum(v * v for v in acc.values())) or 1.0
|
||
centre = {w: v / norm for w, v in acc.items()}
|
||
if sum(_cos(vecs[i], centre) for i in g) / len(g) < 0.3:
|
||
continue
|
||
df = Counter()
|
||
for i in g:
|
||
df.update({w for w in words(stories[i][1]) if w not in NOT_TOPICS and w not in source_words})
|
||
nice = []
|
||
for w, c in df.most_common(8):
|
||
if c < max(2, len(g) // 2):
|
||
break
|
||
forms = Counter(t for i in g for t in _TOKEN.findall(stories[i][1]) if t.lower() == w)
|
||
form = forms.most_common(1)[0][0] if forms else w
|
||
# Nouns and names make a label, not "rückt" or "näher"; "Linke" once, not "Linken" too.
|
||
if form[:1].isupper() and not any(x.lower()[:5] == form.lower()[:5] for x in nice):
|
||
nice.append(form)
|
||
if len(nice) == 3:
|
||
break
|
||
if len(nice) < 2: # one shared word is a coincidence, not a topic
|
||
continue
|
||
out.append({'label': ' · '.join(nice), 'ids': [stories[i][0] for i in g]})
|
||
if len(out) >= max_topics:
|
||
break
|
||
return out
|