354 lines
15 KiB
Python
354 lines
15 KiB
Python
|
"""Reading help for news, with classic text methods (no language model):
|
|||
|
|
|
|||
|
|
- furniture: sentences a source prints under many of its articles (donation boxes,
|
|||
|
|
"Diese Nachricht wurde … gesendet") are no part of an article; they are left out
|
|||
|
|
when an article is shown and when a story is looked at;
|
|||
|
|
- a story's overview: a summary drawn from all its reports (LexRank: the sentences
|
|||
|
|
most like the others, without repeating each other — MMR), what only one report
|
|||
|
|
says (its sentences unlike any other source's), and each source's own words for
|
|||
|
|
the event (log-odds of a word in that report against the story's other reports);
|
|||
|
|
- related stories in the other language (German/English): stories of the same days
|
|||
|
|
that share names (capitalised words that aren't at a sentence's start), weighted by
|
|||
|
|
how rare a name is among the day's stories;
|
|||
|
|
- topics: stories grouped by their words when they're about the same matter, not
|
|||
|
|
only the same event, labelled with the words they share most.
|
|||
|
|
|
|||
|
|
Pure Python: these run on the server with the feeds, for one user's stories at a time.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import math
|
|||
|
|
import re
|
|||
|
|
from collections import Counter, defaultdict
|
|||
|
|
|
|||
|
|
from . import fetch
|
|||
|
|
|
|||
|
|
_SENT_END = re.compile(r'(?<=[.!?…])\s+(?=[„"»A-ZÄÖÜ0-9])')
|
|||
|
|
_TOKEN = re.compile(r"[A-Za-zÄÖÜäöüßÀ-ÿ][A-Za-zÄÖÜäöüßÀ-ÿ0-9'’\-]+")
|
|||
|
|
|
|||
|
|
STOP = set('''
|
|||
|
|
aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher
|
|||
|
|
da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese
|
|||
|
|
diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für
|
|||
|
|
gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer
|
|||
|
|
in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen
|
|||
|
|
nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind
|
|||
|
|
so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter
|
|||
|
|
weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier
|
|||
|
|
heute gestern morgen laut sagt sagte sagten erst mal neu video live ticker meldung bericht hatte hätte
|
|||
|
|
außerdem diejenigen derjenige inzwischen zuvor derzeit dennoch trotzdem deswegen ebenfalls
|
|||
|
|
wäre würde würden zudem sowohl jedoch allerdings bereits rund etwa demnach zufolge angaben unter
|
|||
|
|
the and for with from that this have has are was were will would been into about after over their they
|
|||
|
|
his her its our you your not but all can new says said more than when what who how why off out also had
|
|||
|
|
one two which there been being some other could should would may might them then just only very
|
|||
|
|
'''.split())
|
|||
|
|
|
|||
|
|
# Capitalised words that are no names (German nouns are capitalised too: those don't matter,
|
|||
|
|
# they never match an English text; these would match both).
|
|||
|
|
_NOT_NAMES = set('''
|
|||
|
|
Der Die Das Den Dem Des Ein Eine Einen Einem Einer Er Sie Es Wir Ihr Ich Und Oder Aber Auch Doch Mit Von
|
|||
|
|
Nach Bei Für Auf Aus Im In Am Zum Zur Vor Seit Als Wie Wenn Dass Nicht Noch Nur So Da Dann Hier Jetzt
|
|||
|
|
The A An And Or But With From For After Before In On At By To Of As It He She They We I This That These
|
|||
|
|
Those Mr Mrs Ms Dr Monday Tuesday Wednesday Thursday Friday Saturday Sunday Montag Dienstag Mittwoch
|
|||
|
|
Donnerstag Freitag Samstag Sonntag Sonnabend January February March April May June July August September
|
|||
|
|
October November December Januar Februar März Juni Juli Oktober Dezember Reuters AFP AP dpa BBC CNN
|
|||
|
|
'''.split())
|
|||
|
|
|
|||
|
|
|
|||
|
|
def plain(html):
|
|||
|
|
return fetch.plain(html or '')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def sentences(text):
|
|||
|
|
text = re.sub(r'\s+', ' ', text or '').strip()
|
|||
|
|
return [s.strip() for s in _SENT_END.split(text) if 40 <= len(s.strip()) <= 400]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _key(s):
|
|||
|
|
"""A sentence, the same however its numbers and spaces differ."""
|
|||
|
|
return re.sub(r'\d+', '0', re.sub(r'\s+', ' ', s.lower())).strip()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def words(text):
|
|||
|
|
return [w.lower() for w in _TOKEN.findall(text or '') if len(w) > 2 and w.lower() not in STOP]
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---- furniture ------------------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
FURNITURE_MIN = 3 # in this many of a source's articles: furniture
|
|||
|
|
|
|||
|
|
|
|||
|
|
def furniture(texts):
|
|||
|
|
"""The sentences (keys) that stand in at least FURNITURE_MIN of these texts (one source's)."""
|
|||
|
|
seen = Counter()
|
|||
|
|
for t in texts:
|
|||
|
|
seen.update({_key(s) for s in _units(t)})
|
|||
|
|
return {k for k, n in seen.items() if n >= FURNITURE_MIN}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _units(text):
|
|||
|
|
return [u.strip() for u in re.split(r'\n|(?<=[.!?])\s+', text or '') if len(u.strip()) > 15]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def without_furniture(text, keys):
|
|||
|
|
if not keys:
|
|||
|
|
return text
|
|||
|
|
return ' '.join(u for u in _units(text) if _key(u) not in keys)
|
|||
|
|
|
|||
|
|
|
|||
|
|
_BLOCK = re.compile(r'<(p|li|h[1-6]|blockquote|figcaption|div)\b[^>]*>(.*?)</\1>', re.S | re.I)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def html_without_furniture(html, keys):
|
|||
|
|
"""The article's HTML without the blocks whose text is (only) furniture."""
|
|||
|
|
if not keys or not html:
|
|||
|
|
return html
|
|||
|
|
|
|||
|
|
def keep(m):
|
|||
|
|
inner = plain(m.group(2))
|
|||
|
|
units = _units(inner)
|
|||
|
|
if units and all(_key(u) in keys for u in units):
|
|||
|
|
return ''
|
|||
|
|
return m.group(0)
|
|||
|
|
return _BLOCK.sub(keep, html)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---- tf-idf over sentences -------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
def _vectors(texts):
|
|||
|
|
toks = [Counter(words(t)) for t in texts]
|
|||
|
|
df = Counter()
|
|||
|
|
for t in toks:
|
|||
|
|
df.update(t.keys())
|
|||
|
|
n = len(texts)
|
|||
|
|
idf = {w: math.log((n + 1) / (d + 1)) + 1 for w, d in df.items()}
|
|||
|
|
vecs = []
|
|||
|
|
for t in toks:
|
|||
|
|
v = {w: (1 + math.log(c)) * idf[w] for w, c in t.items()}
|
|||
|
|
norm = math.sqrt(sum(x * x for x in v.values())) or 1.0
|
|||
|
|
vecs.append({w: x / norm for w, x in v.items()})
|
|||
|
|
return vecs
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _cos(a, b):
|
|||
|
|
if len(a) > len(b):
|
|||
|
|
a, b = b, a
|
|||
|
|
return sum(x * b.get(w, 0.0) for w, x in a.items())
|
|||
|
|
|
|||
|
|
|
|||
|
|
def summarize(docs, k=3, lam=0.7):
|
|||
|
|
"""docs: [(source, text)]. The k sentences that say most of what all say, in their order
|
|||
|
|
of appearance: [(source, sentence)]."""
|
|||
|
|
ss = [(src, s) for src, t in docs for s in sentences(t)[:30]]
|
|||
|
|
if len(ss) <= k:
|
|||
|
|
return ss
|
|||
|
|
vecs = _vectors([s for _, s in ss])
|
|||
|
|
n = len(ss)
|
|||
|
|
sim = [[_cos(vecs[a], vecs[b]) if a != b else 0.0 for b in range(n)] for a in range(n)]
|
|||
|
|
links = [[1.0 if sim[a][b] > 0.1 else 0.0 for b in range(n)] for a in range(n)]
|
|||
|
|
deg = [sum(r) or 1.0 for r in links]
|
|||
|
|
rank = [1.0 / n] * n
|
|||
|
|
for _ in range(40):
|
|||
|
|
rank = [0.15 / n + 0.85 * sum(links[b][a] * rank[b] / deg[b] for b in range(n)) for a in range(n)]
|
|||
|
|
top = max(rank) or 1.0
|
|||
|
|
chosen = []
|
|||
|
|
for _ in range(k):
|
|||
|
|
best, score = None, -1e9
|
|||
|
|
for a in range(n):
|
|||
|
|
if a in chosen:
|
|||
|
|
continue
|
|||
|
|
s = lam * rank[a] / top - (1 - lam) * max((sim[a][c] for c in chosen), default=0.0)
|
|||
|
|
if s > score:
|
|||
|
|
best, score = a, s
|
|||
|
|
chosen.append(best)
|
|||
|
|
return [ss[c] for c in sorted(chosen)]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def novel(docs, per_source=2, limit=0.2):
|
|||
|
|
"""What only one report says: per source, up to per_source of its sentences that no other
|
|||
|
|
source has anything like (cosine < limit), the most central of its own first."""
|
|||
|
|
ss = [(src, s) for src, t in docs for s in sentences(t)[:40]]
|
|||
|
|
if len({src for src, _ in ss}) < 2:
|
|||
|
|
return {}
|
|||
|
|
vecs = _vectors([s for _, s in ss])
|
|||
|
|
out = defaultdict(list)
|
|||
|
|
for a, (src, s) in enumerate(ss):
|
|||
|
|
others = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 != src]
|
|||
|
|
if not others or max(others) >= limit:
|
|||
|
|
continue
|
|||
|
|
own = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 == src and b != a]
|
|||
|
|
out[src].append((sum(own) / len(own) if own else 0.0, s))
|
|||
|
|
return {src: [s for _, s in sorted(v, key=lambda x: -x[0])[:per_source]] for src, v in out.items()}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def word_choice(docs, per_source=6):
|
|||
|
|
"""Each source's own words for the event: log-odds of a word in its report against the
|
|||
|
|
other reports (with a prior from all of them), at least twice in its report."""
|
|||
|
|
forms = defaultdict(Counter)
|
|||
|
|
counts = {}
|
|||
|
|
for src, t in docs:
|
|||
|
|
c = Counter()
|
|||
|
|
for w in _TOKEN.findall(t or ''):
|
|||
|
|
lw = w.lower()
|
|||
|
|
if len(lw) > 3 and lw not in STOP:
|
|||
|
|
c[lw] += 1
|
|||
|
|
forms[lw][w] += 1
|
|||
|
|
counts[src] = counts.get(src, Counter()) + c
|
|||
|
|
if len(counts) < 2:
|
|||
|
|
return {}
|
|||
|
|
total = sum(counts.values(), Counter())
|
|||
|
|
a0 = sum(total.values()) or 1
|
|||
|
|
out = {}
|
|||
|
|
for src, c in counts.items():
|
|||
|
|
n_i = sum(c.values()) or 1
|
|||
|
|
rest = total - c
|
|||
|
|
n_j = sum(rest.values()) or 1
|
|||
|
|
scored = []
|
|||
|
|
for w, f in c.items():
|
|||
|
|
if f < 2:
|
|||
|
|
continue
|
|||
|
|
aw = total[w] / a0 * 100
|
|||
|
|
d = math.log((f + aw) / (n_i + 100 - f - aw)) - math.log((rest[w] + aw) / (n_j + 100 - rest[w] - aw))
|
|||
|
|
scored.append((d / math.sqrt(1 / (f + aw) + 1 / (rest[w] + aw)), w))
|
|||
|
|
# Nouns and names (as written: capitalised), not the verbs and particles around them.
|
|||
|
|
best = [(z, forms[w].most_common(1)[0][0]) for z, w in sorted(scored, reverse=True) if z > 1.0]
|
|||
|
|
out[src] = [w for z, w in best if w[:1].isupper()][:per_source]
|
|||
|
|
return {k: v for k, v in out.items() if v}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---- names, languages, related stories -------------------------------------------------------------
|
|||
|
|
|
|||
|
|
_EN = set('the and of to in is that for with was said on as by at from'.split())
|
|||
|
|
_DE = set('der die das und ist nicht mit den von zu im sich auf für dem des eine sagte'.split())
|
|||
|
|
|
|||
|
|
|
|||
|
|
def language(text):
|
|||
|
|
w = [x.lower() for x in _TOKEN.findall(text[:3000])]
|
|||
|
|
en = sum(1 for x in w if x in _EN)
|
|||
|
|
de = sum(1 for x in w if x in _DE)
|
|||
|
|
return 'en' if en > de else 'de'
|
|||
|
|
|
|||
|
|
|
|||
|
|
def names(text):
|
|||
|
|
"""Capitalised words not at a sentence's start (in German: names and nouns; in English:
|
|||
|
|
names), as a set of lowercased forms."""
|
|||
|
|
out = set()
|
|||
|
|
for s in sentences(text) or [text or '']:
|
|||
|
|
toks = _TOKEN.findall(s)
|
|||
|
|
for t in toks[1:]:
|
|||
|
|
if t[0].isupper() and t not in _NOT_NAMES and len(t) > 2:
|
|||
|
|
out.add(t.lower().rstrip('s’\''))
|
|||
|
|
return out
|
|||
|
|
|
|||
|
|
|
|||
|
|
def related(stories, min_weight=17.0, per_story=3):
|
|||
|
|
"""stories: [(id, language, set of names)] of the same days. Pairs of stories in different
|
|||
|
|
languages sharing rare names: {id: [(other id, weight)]}, weight = the shared names' idf."""
|
|||
|
|
df = Counter()
|
|||
|
|
for _, _, ns in stories:
|
|||
|
|
df.update(ns)
|
|||
|
|
n = len(stories)
|
|||
|
|
idf = {w: math.log((n + 1) / (d + 1)) for w, d in df.items()}
|
|||
|
|
out = defaultdict(list)
|
|||
|
|
for a in range(n):
|
|||
|
|
ia, la, na = stories[a]
|
|||
|
|
for b in range(a + 1, n):
|
|||
|
|
ib, lb, nb = stories[b]
|
|||
|
|
if la == lb:
|
|||
|
|
continue
|
|||
|
|
shared = na & nb
|
|||
|
|
if len(shared) < 3:
|
|||
|
|
continue
|
|||
|
|
w = sum(idf[x] for x in shared)
|
|||
|
|
if w >= min_weight:
|
|||
|
|
out[ia].append((ib, round(w, 1)))
|
|||
|
|
out[ib].append((ia, round(w, 1)))
|
|||
|
|
return {k: sorted(v, key=lambda x: -x[1])[:per_story] for k, v in out.items()}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---- topics ---------------------------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
NOT_TOPICS = set('breaking news free live watch video sekunden gebärdensprache podcast newsletter abo online'.split())
|
|||
|
|
|
|||
|
|
|
|||
|
|
TOPIC_MIN = 3
|
|||
|
|
|
|||
|
|
|
|||
|
|
def topics(stories, threshold=0.12, min_size=TOPIC_MIN, max_topics=8, rounds=3):
|
|||
|
|
"""stories: [(id, text, sources)] (titles and summaries, the set of source names). Groups of
|
|||
|
|
stories about one matter, the largest first, each with a label (the words most of its stories
|
|||
|
|
share): [{'label': str, 'ids': [ids]}]. Leader clustering against the groups' centroids, a
|
|||
|
|
few rounds (each story to its nearest centroid, then the centroids anew); a group counts
|
|||
|
|
when it has min_size stories from at least two sources."""
|
|||
|
|
if len(stories) < min_size:
|
|||
|
|
return []
|
|||
|
|
vecs = _vectors([s[1] for s in stories])
|
|||
|
|
order = sorted(range(len(stories)), key=lambda i: -len(vecs[i]))
|
|||
|
|
cents = []
|
|||
|
|
assign = [-1] * len(stories)
|
|||
|
|
for _ in range(rounds):
|
|||
|
|
changed = False
|
|||
|
|
for i in order:
|
|||
|
|
best, sim = -1, threshold
|
|||
|
|
for g, c in enumerate(cents):
|
|||
|
|
x = _cos(vecs[i], c)
|
|||
|
|
if x > sim:
|
|||
|
|
best, sim = g, x
|
|||
|
|
if best < 0:
|
|||
|
|
cents.append(dict(vecs[i]))
|
|||
|
|
best = len(cents) - 1
|
|||
|
|
if assign[i] != best:
|
|||
|
|
assign[i] = best
|
|||
|
|
changed = True
|
|||
|
|
# centroids anew from their members
|
|||
|
|
acc = [Counter() for _ in cents]
|
|||
|
|
for i, g in enumerate(assign):
|
|||
|
|
for w, v in vecs[i].items():
|
|||
|
|
acc[g][w] += v
|
|||
|
|
cents = []
|
|||
|
|
remap = {}
|
|||
|
|
for g, a in enumerate(acc):
|
|||
|
|
if a:
|
|||
|
|
norm = math.sqrt(sum(v * v for v in a.values())) or 1.0
|
|||
|
|
remap[g] = len(cents)
|
|||
|
|
cents.append({w: v / norm for w, v in a.items()})
|
|||
|
|
assign = [remap.get(g, -1) for g in assign]
|
|||
|
|
if not changed:
|
|||
|
|
break
|
|||
|
|
groups = defaultdict(list)
|
|||
|
|
for i, g in enumerate(assign):
|
|||
|
|
groups[g].append(i)
|
|||
|
|
out = []
|
|||
|
|
source_words = {w.lower() for s in stories for src in s[2] for w in _TOKEN.findall(src)}
|
|||
|
|
for g in sorted(groups.values(), key=len, reverse=True):
|
|||
|
|
if len(g) < min_size or len({src for i in g for src in stories[i][2]}) < 2:
|
|||
|
|
continue
|
|||
|
|
# Close enough together: the stories' mean likeness to their group's centre.
|
|||
|
|
acc = Counter()
|
|||
|
|
for i in g:
|
|||
|
|
for w, v in vecs[i].items():
|
|||
|
|
acc[w] += v
|
|||
|
|
norm = math.sqrt(sum(v * v for v in acc.values())) or 1.0
|
|||
|
|
centre = {w: v / norm for w, v in acc.items()}
|
|||
|
|
if sum(_cos(vecs[i], centre) for i in g) / len(g) < 0.3:
|
|||
|
|
continue
|
|||
|
|
df = Counter()
|
|||
|
|
for i in g:
|
|||
|
|
df.update({w for w in words(stories[i][1]) if w not in NOT_TOPICS and w not in source_words})
|
|||
|
|
nice = []
|
|||
|
|
for w, c in df.most_common(8):
|
|||
|
|
if c < max(2, len(g) // 2):
|
|||
|
|
break
|
|||
|
|
forms = Counter(t for i in g for t in _TOKEN.findall(stories[i][1]) if t.lower() == w)
|
|||
|
|
form = forms.most_common(1)[0][0] if forms else w
|
|||
|
|
# Nouns and names make a label, not "rückt" or "näher"; "Linke" once, not "Linken" too.
|
|||
|
|
if form[:1].isupper() and not any(x.lower()[:5] == form.lower()[:5] for x in nice):
|
|||
|
|
nice.append(form)
|
|||
|
|
if len(nice) == 3:
|
|||
|
|
break
|
|||
|
|
if len(nice) < 2: # one shared word is a coincidence, not a topic
|
|||
|
|
continue
|
|||
|
|
out.append({'label': ' · '.join(nice), 'ids': [stories[i][0] for i in g]})
|
|||
|
|
if len(out) >= max_topics:
|
|||
|
|
break
|
|||
|
|
return out
|