diff --git a/feeds/textproc.py b/feeds/textproc.py deleted file mode 100644 index 41a1e86..0000000 --- a/feeds/textproc.py +++ /dev/null @@ -1,352 +0,0 @@ -"""Reading help for news, with classic text methods (no language model): - -- furniture: sentences a source prints under many of its articles (donation boxes, - "Diese Nachricht wurde … gesendet") are no part of an article; they are left out - when an article is shown and when a story is looked at; -- a story's overview: a summary drawn from all its reports (LexRank: the sentences - most like the others, without repeating each other — MMR), what only one report - says (its sentences unlike any other source's), and each source's own words for - the event (log-odds of a word in that report against the story's other reports); -- related stories in the other language (German/English): stories of the same days - that share names (capitalised words that aren't at a sentence's start), weighted by - how rare a name is among the day's stories; -- topics: stories grouped by their words when they're about the same matter, not - only the same event, labelled with the words they share most. - -Pure Python: these run on the server with the feeds, for one user's stories at a time. -""" - -import math -import re -from collections import Counter, defaultdict - -from . import fetch - -_SENT_END = re.compile(r'(?<=[.!?…])\s+(?=[„"»A-ZÄÖÜ0-9])') -_TOKEN = re.compile(r"[A-Za-zÄÖÜäöüßÀ-ÿ][A-Za-zÄÖÜäöüßÀ-ÿ0-9'’\-]+") - -STOP = set(''' -aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher -da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese -diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für -gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer -in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen -nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind -so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter -weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier -heute gestern morgen laut sagt sagte sagten erst mal neu video live ticker meldung bericht hatte hätte -wäre würde würden zudem sowohl jedoch allerdings bereits rund etwa demnach zufolge angaben unter -the and for with from that this have has are was were will would been into about after over their they -his her its our you your not but all can new says said more than when what who how why off out also had -one two which there been being some other could should would may might them then just only very -'''.split()) - -# Capitalised words that are no names (German nouns are capitalised too: those don't matter, -# they never match an English text; these would match both). -_NOT_NAMES = set(''' -Der Die Das Den Dem Des Ein Eine Einen Einem Einer Er Sie Es Wir Ihr Ich Und Oder Aber Auch Doch Mit Von -Nach Bei Für Auf Aus Im In Am Zum Zur Vor Seit Als Wie Wenn Dass Nicht Noch Nur So Da Dann Hier Jetzt -The A An And Or But With From For After Before In On At By To Of As It He She They We I This That These -Those Mr Mrs Ms Dr Monday Tuesday Wednesday Thursday Friday Saturday Sunday Montag Dienstag Mittwoch -Donnerstag Freitag Samstag Sonntag Sonnabend January February March April May June July August September -October November December Januar Februar März Juni Juli Oktober Dezember Reuters AFP AP dpa BBC CNN -'''.split()) - - -def plain(html): - return fetch.plain(html or '') - - -def sentences(text): - text = re.sub(r'\s+', ' ', text or '').strip() - return [s.strip() for s in _SENT_END.split(text) if 40 <= len(s.strip()) <= 400] - - -def _key(s): - """A sentence, the same however its numbers and spaces differ.""" - return re.sub(r'\d+', '0', re.sub(r'\s+', ' ', s.lower())).strip() - - -def words(text): - return [w.lower() for w in _TOKEN.findall(text or '') if len(w) > 2 and w.lower() not in STOP] - - -# ---- furniture ------------------------------------------------------------------------------------ - -FURNITURE_MIN = 3 # in this many of a source's articles: furniture - - -def furniture(texts): - """The sentences (keys) that stand in at least FURNITURE_MIN of these texts (one source's).""" - seen = Counter() - for t in texts: - seen.update({_key(s) for s in _units(t)}) - return {k for k, n in seen.items() if n >= FURNITURE_MIN} - - -def _units(text): - return [u.strip() for u in re.split(r'\n|(?<=[.!?])\s+', text or '') if len(u.strip()) > 15] - - -def without_furniture(text, keys): - if not keys: - return text - return ' '.join(u for u in _units(text) if _key(u) not in keys) - - -_BLOCK = re.compile(r'<(p|li|h[1-6]|blockquote|figcaption|div)\b[^>]*>(.*?)', re.S | re.I) - - -def html_without_furniture(html, keys): - """The article's HTML without the blocks whose text is (only) furniture.""" - if not keys or not html: - return html - - def keep(m): - inner = plain(m.group(2)) - units = _units(inner) - if units and all(_key(u) in keys for u in units): - return '' - return m.group(0) - return _BLOCK.sub(keep, html) - - -# ---- tf-idf over sentences ------------------------------------------------------------------------- - -def _vectors(texts): - toks = [Counter(words(t)) for t in texts] - df = Counter() - for t in toks: - df.update(t.keys()) - n = len(texts) - idf = {w: math.log((n + 1) / (d + 1)) + 1 for w, d in df.items()} - vecs = [] - for t in toks: - v = {w: (1 + math.log(c)) * idf[w] for w, c in t.items()} - norm = math.sqrt(sum(x * x for x in v.values())) or 1.0 - vecs.append({w: x / norm for w, x in v.items()}) - return vecs - - -def _cos(a, b): - if len(a) > len(b): - a, b = b, a - return sum(x * b.get(w, 0.0) for w, x in a.items()) - - -def summarize(docs, k=3, lam=0.7): - """docs: [(source, text)]. The k sentences that say most of what all say, in their order - of appearance: [(source, sentence)].""" - ss = [(src, s) for src, t in docs for s in sentences(t)[:30]] - if len(ss) <= k: - return ss - vecs = _vectors([s for _, s in ss]) - n = len(ss) - sim = [[_cos(vecs[a], vecs[b]) if a != b else 0.0 for b in range(n)] for a in range(n)] - links = [[1.0 if sim[a][b] > 0.1 else 0.0 for b in range(n)] for a in range(n)] - deg = [sum(r) or 1.0 for r in links] - rank = [1.0 / n] * n - for _ in range(40): - rank = [0.15 / n + 0.85 * sum(links[b][a] * rank[b] / deg[b] for b in range(n)) for a in range(n)] - top = max(rank) or 1.0 - chosen = [] - for _ in range(k): - best, score = None, -1e9 - for a in range(n): - if a in chosen: - continue - s = lam * rank[a] / top - (1 - lam) * max((sim[a][c] for c in chosen), default=0.0) - if s > score: - best, score = a, s - chosen.append(best) - return [ss[c] for c in sorted(chosen)] - - -def novel(docs, per_source=2, limit=0.2): - """What only one report says: per source, up to per_source of its sentences that no other - source has anything like (cosine < limit), the most central of its own first.""" - ss = [(src, s) for src, t in docs for s in sentences(t)[:40]] - if len({src for src, _ in ss}) < 2: - return {} - vecs = _vectors([s for _, s in ss]) - out = defaultdict(list) - for a, (src, s) in enumerate(ss): - others = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 != src] - if not others or max(others) >= limit: - continue - own = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 == src and b != a] - out[src].append((sum(own) / len(own) if own else 0.0, s)) - return {src: [s for _, s in sorted(v, key=lambda x: -x[0])[:per_source]] for src, v in out.items()} - - -def word_choice(docs, per_source=6): - """Each source's own words for the event: log-odds of a word in its report against the - other reports (with a prior from all of them), at least twice in its report.""" - forms = defaultdict(Counter) - counts = {} - for src, t in docs: - c = Counter() - for w in _TOKEN.findall(t or ''): - lw = w.lower() - if len(lw) > 3 and lw not in STOP: - c[lw] += 1 - forms[lw][w] += 1 - counts[src] = counts.get(src, Counter()) + c - if len(counts) < 2: - return {} - total = sum(counts.values(), Counter()) - a0 = sum(total.values()) or 1 - out = {} - for src, c in counts.items(): - n_i = sum(c.values()) or 1 - rest = total - c - n_j = sum(rest.values()) or 1 - scored = [] - for w, f in c.items(): - if f < 2: - continue - aw = total[w] / a0 * 100 - d = math.log((f + aw) / (n_i + 100 - f - aw)) - math.log((rest[w] + aw) / (n_j + 100 - rest[w] - aw)) - scored.append((d / math.sqrt(1 / (f + aw) + 1 / (rest[w] + aw)), w)) - # Nouns and names (as written: capitalised), not the verbs and particles around them. - best = [(z, forms[w].most_common(1)[0][0]) for z, w in sorted(scored, reverse=True) if z > 1.0] - out[src] = [w for z, w in best if w[:1].isupper()][:per_source] - return {k: v for k, v in out.items() if v} - - -# ---- names, languages, related stories ------------------------------------------------------------- - -_EN = set('the and of to in is that for with was said on as by at from'.split()) -_DE = set('der die das und ist nicht mit den von zu im sich auf für dem des eine sagte'.split()) - - -def language(text): - w = [x.lower() for x in _TOKEN.findall(text[:3000])] - en = sum(1 for x in w if x in _EN) - de = sum(1 for x in w if x in _DE) - return 'en' if en > de else 'de' - - -def names(text): - """Capitalised words not at a sentence's start (in German: names and nouns; in English: - names), as a set of lowercased forms.""" - out = set() - for s in sentences(text) or [text or '']: - toks = _TOKEN.findall(s) - for t in toks[1:]: - if t[0].isupper() and t not in _NOT_NAMES and len(t) > 2: - out.add(t.lower().rstrip('s’\'')) - return out - - -def related(stories, min_weight=17.0, per_story=3): - """stories: [(id, language, set of names)] of the same days. Pairs of stories in different - languages sharing rare names: {id: [(other id, weight)]}, weight = the shared names' idf.""" - df = Counter() - for _, _, ns in stories: - df.update(ns) - n = len(stories) - idf = {w: math.log((n + 1) / (d + 1)) for w, d in df.items()} - out = defaultdict(list) - for a in range(n): - ia, la, na = stories[a] - for b in range(a + 1, n): - ib, lb, nb = stories[b] - if la == lb: - continue - shared = na & nb - if len(shared) < 3: - continue - w = sum(idf[x] for x in shared) - if w >= min_weight: - out[ia].append((ib, round(w, 1))) - out[ib].append((ia, round(w, 1))) - return {k: sorted(v, key=lambda x: -x[1])[:per_story] for k, v in out.items()} - - -# ---- topics --------------------------------------------------------------------------------------------- - -NOT_TOPICS = set('breaking news free live watch video sekunden gebärdensprache podcast newsletter abo online'.split()) - - -TOPIC_MIN = 3 - - -def topics(stories, threshold=0.12, min_size=TOPIC_MIN, max_topics=8, rounds=3): - """stories: [(id, text, sources)] (titles and summaries, the set of source names). Groups of - stories about one matter, the largest first, each with a label (the words most of its stories - share): [{'label': str, 'ids': [ids]}]. Leader clustering against the groups' centroids, a - few rounds (each story to its nearest centroid, then the centroids anew); a group counts - when it has min_size stories from at least two sources.""" - if len(stories) < min_size: - return [] - vecs = _vectors([s[1] for s in stories]) - order = sorted(range(len(stories)), key=lambda i: -len(vecs[i])) - cents = [] - assign = [-1] * len(stories) - for _ in range(rounds): - changed = False - for i in order: - best, sim = -1, threshold - for g, c in enumerate(cents): - x = _cos(vecs[i], c) - if x > sim: - best, sim = g, x - if best < 0: - cents.append(dict(vecs[i])) - best = len(cents) - 1 - if assign[i] != best: - assign[i] = best - changed = True - # centroids anew from their members - acc = [Counter() for _ in cents] - for i, g in enumerate(assign): - for w, v in vecs[i].items(): - acc[g][w] += v - cents = [] - remap = {} - for g, a in enumerate(acc): - if a: - norm = math.sqrt(sum(v * v for v in a.values())) or 1.0 - remap[g] = len(cents) - cents.append({w: v / norm for w, v in a.items()}) - assign = [remap.get(g, -1) for g in assign] - if not changed: - break - groups = defaultdict(list) - for i, g in enumerate(assign): - groups[g].append(i) - out = [] - source_words = {w.lower() for s in stories for src in s[2] for w in _TOKEN.findall(src)} - for g in sorted(groups.values(), key=len, reverse=True): - if len(g) < min_size or len({src for i in g for src in stories[i][2]}) < 2: - continue - # Close enough together: the stories' mean likeness to their group's centre. - acc = Counter() - for i in g: - for w, v in vecs[i].items(): - acc[w] += v - norm = math.sqrt(sum(v * v for v in acc.values())) or 1.0 - centre = {w: v / norm for w, v in acc.items()} - if sum(_cos(vecs[i], centre) for i in g) / len(g) < 0.3: - continue - df = Counter() - for i in g: - df.update({w for w in words(stories[i][1]) if w not in NOT_TOPICS and w not in source_words}) - nice = [] - for w, c in df.most_common(8): - if c < max(2, len(g) // 2): - break - forms = Counter(t for i in g for t in _TOKEN.findall(stories[i][1]) if t.lower() == w) - form = forms.most_common(1)[0][0] if forms else w - # Nouns and names make a label, not "rückt" or "näher"; "Linke" once, not "Linken" too. - if form[:1].isupper() and not any(x.lower()[:5] == form.lower()[:5] for x in nice): - nice.append(form) - if len(nice) == 3: - break - if len(nice) < 2: # one shared word is a coincidence, not a topic - continue - out.append({'label': ' · '.join(nice), 'ids': [stories[i][0] for i in g]}) - if len(out) >= max_topics: - break - return out diff --git a/feeds/urls.py b/feeds/urls.py index 5edea27..c1aaf92 100644 --- a/feeds/urls.py +++ b/feeds/urls.py @@ -15,7 +15,6 @@ urlpatterns = [ path('briefing/seen/', views.briefing_seen, name='feeds_briefing_seen'), path('news/', views.news, name='feeds_news'), path('stories//', views.story, name='feeds_story'), - path('stories//overview/', views.story_overview, name='feeds_story_overview'), path('blogs/', views.blogs, name='feeds_blogs'), path('starred/', views.starred, name='feeds_starred'), path('items/read/', views.mark_read, name='feeds_mark_read'), diff --git a/feeds/views.py b/feeds/views.py index aadc8fd..0068d2f 100644 --- a/feeds/views.py +++ b/feeds/views.py @@ -12,11 +12,8 @@ Sources: GET /feeds/state/ sources, unread counts, settings POST /feeds/refresh/ refresh the account's sources now News: GET /feeds/briefing/ what's new since the briefing was last seen POST /feeds/briefing/seen/ seen: everything up to now counts as read - GET /feeds/news/?before=&category=&ids= stories, newest first (ids: just these, a topic's); - the first page also has the day's topics + GET /feeds/news/?before=&category= stories, newest first GET /feeds/stories// one story with all its items - GET /feeds/stories//overview/ summary, what only one report says, each source's - words, the story in the other language (feeds/textproc.py) Blogs: GET /feeds/blogs/?folder=&source=&unread=1&fav=1&before= (folder= alone: blogs without one; without folder/source the hidden blogs stay out) Items: GET /feeds/items// the article (fetches the page if the feed had a teaser) @@ -38,7 +35,6 @@ from xml.sax.saxutils import quoteattr import feedparser import requests -from django.core.cache import cache from django.db.models import Count from django.http import HttpResponse, JsonResponse from django.utils import timezone @@ -46,7 +42,7 @@ from django.utils.dateparse import parse_datetime from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods -from . import fetch, logins, refresh, textproc +from . import fetch, logins, refresh from .models import Item, SiteLogin, Source, Story PAGE = 40 @@ -75,15 +71,6 @@ def _body(request): return None -def _when(value): - """A datetime parameter; None when there's none or it's garbled (an unencoded '+' arrives as - a space).""" - try: - return parse_datetime((value or '').strip().replace(' ', '+')) - except ValueError: - return None - - def _iso(dt): return dt.isoformat() if dt else None @@ -492,76 +479,14 @@ def news(request): if (err := _auth(request)): return err qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at') - ids = [int(x) for x in request.GET.get('ids', '').split(',') if x.strip().isdigit()] - if ids: - stories = _stories(request.user, qs.filter(pk__in=ids[:PAGE])) - return JsonResponse({'stories': stories, 'more': False}) - before = _when(request.GET.get('before')) + before = parse_datetime(request.GET.get('before') or '') if before: qs = qs.filter(last_at__lt=before) category = request.GET.get('category') if category: qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category) stories = _stories(request.user, qs[:PAGE]) - out = {'stories': stories, 'more': len(stories) == PAGE} - if not before: - topics = _news_text(request.user)['topics'] - if category: - # The category's share of each topic. - mine = set(qs.values_list('pk', flat=True)[:500]) - topics = [t for t in ({**t, 'ids': [i for i in t['ids'] if i in mine]} for t in topics) - if len(t['ids']) >= textproc.TOPIC_MIN] - out['topics'] = topics - return JsonResponse(out) - - -# ---- reading help (feeds/textproc.py) - -FURNITURE_ITEMS = 40 # a source's recent articles that show its furniture -TEXT_DAYS = 2 # topics and related stories: of these days -TEXT_CACHE = 6 * 3600 - - -def _furniture(source): - key = f'feeds:furniture:{source.id}' - keys = cache.get(key) - if keys is None: - texts = [textproc.plain(c) for c in Item.objects.filter(source=source, gone=False) - .order_by('-published').values_list('content', flat=True)[:FURNITURE_ITEMS]] - keys = textproc.furniture(texts) - cache.set(key, keys, TEXT_CACHE) - return keys - - -def _clean_text(i): - """An item's text without its source's furniture, plain.""" - return textproc.without_furniture(textproc.plain(i.content) or i.summary, _furniture(i.source)) - - -def _news_text(user): - """The news of the last days, looked at as a whole: topics [{label, ids}] and related - stories in the other language {story id: [ids]}. Cached until a story changes.""" - since = timezone.now() - timedelta(days=TEXT_DAYS) - qs = Story.objects.filter(user=user, last_at__gte=since, items__gone=False).distinct() - latest = qs.order_by('-last_at').values_list('last_at', flat=True).first() - key = f'feeds:newstext:{user.id}:{qs.count()}:{latest.timestamp() if latest else 0}' - got = cache.get(key) - if got is not None: - return got - items = {} - for i in Item.objects.filter(story__in=qs, gone=False).select_related('source'): - items.setdefault(i.story_id, []).append(i) - tl, sl = [], [] - for sid, its in items.items(): - tl.append((sid, ' '.join(f'{i.title}. {i.summary}' for i in its), {i.source.title for i in its})) - text = ' '.join(f'{i.title}. {_clean_text(i)[:4000]}' for i in its) - sl.append((sid, textproc.language(text), textproc.names(text))) - got = { - 'topics': textproc.topics(tl), - 'related': {sid: [b for b, _ in lst] for sid, lst in textproc.related(sl).items()}, - } - cache.set(key, got, TEXT_CACHE) - return got + return JsonResponse({'stories': stories, 'more': len(stories) == PAGE}) @csrf_exempt @@ -575,32 +500,6 @@ def story(request, pk): return JsonResponse({'story': found[0]}) -@csrf_exempt -@require_http_methods(['GET']) -def story_overview(request, pk): - """What a story's reports say together: a summary from all of them, what only one says, - each source's own words, and the same story in the other language.""" - if (err := _auth(request)): - return err - s = Story.objects.filter(user=request.user, pk=pk).first() - if s is None: - return JsonResponse({'error': 'not found'}, status=404) - its = list(Item.objects.filter(story=s, gone=False).select_related('source').order_by('published')) - key = f'feeds:overview:{s.id}:{len(its)}:{sum(len(i.content) for i in its)}' - out = cache.get(key) - if out is None: - docs = [(i.source.title, _clean_text(i)) for i in its] - out = {} - if len({src for src, _ in docs}) >= 2: - out['summary'] = [{'source': src, 'text': t} for src, t in textproc.summarize(docs)] - out['novel'] = textproc.novel(docs) - out['words'] = textproc.word_choice(docs) - cache.set(key, out, TEXT_CACHE) - rel = _news_text(request.user)['related'].get(s.id, []) - related = _stories(request.user, Story.objects.filter(user=request.user, pk__in=rel)) if rel else [] - return JsonResponse({**out, 'related': related}) - - # ---- blogs, items @csrf_exempt @@ -623,7 +522,7 @@ def blogs(request): unread_only = request.GET.get('unread') == '1' if unread_only: qs = qs.filter(read_at=None) - before = _when(request.GET.get('before')) + before = parse_datetime(request.GET.get('before') or '') if before: qs = qs.filter(published__lt=before) # Unread: all of them at once (the page sorts them into days, favorites, rare blogs). @@ -651,9 +550,7 @@ def item(request, pk): return JsonResponse({'error': 'not found'}, status=404) if refresh.needs_article(i, i.source): refresh.load_article(i) - d = _item_data(i, content=True) - d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source)) - return JsonResponse({'item': d}) + return JsonResponse({'item': _item_data(i, content=True)}) @csrf_exempt