feeds: unfertige Änderungen aus f2583cb wieder heraus (versehentlich mit committet, liegen weiter im Arbeitsbaum)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

feeds/textproc.py

@@ -1,352 +0,0 @@
-"""Reading help for news, with classic text methods (no language model):
-
-- furniture: sentences a source prints under many of its articles (donation boxes,
-  "Diese Nachricht wurde … gesendet") are no part of an article; they are left out
-  when an article is shown and when a story is looked at;
-- a story's overview: a summary drawn from all its reports (LexRank: the sentences
-  most like the others, without repeating each other — MMR), what only one report
-  says (its sentences unlike any other source's), and each source's own words for
-  the event (log-odds of a word in that report against the story's other reports);
-- related stories in the other language (German/English): stories of the same days
-  that share names (capitalised words that aren't at a sentence's start), weighted by
-  how rare a name is among the day's stories;
-- topics: stories grouped by their words when they're about the same matter, not
-  only the same event, labelled with the words they share most.
-
-Pure Python: these run on the server with the feeds, for one user's stories at a time.
-"""
-
-import math
-import re
-from collections import Counter, defaultdict
-
-from . import fetch
-
-_SENT_END = re.compile(r'(?<=[.!?…])\s+(?=[„"»A-ZÄÖÜ0-9])')
-_TOKEN = re.compile(r"[A-Za-zÄÖÜäöüßÀ-ÿ][A-Za-zÄÖÜäöüßÀ-ÿ0-9'’\-]+")
-
-STOP = set('''
-aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher
-da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese
-diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für
-gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer
-in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen
-nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind
-so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter
-weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier
-heute gestern morgen laut sagt sagte sagten erst mal neu video live ticker meldung bericht hatte hätte
-wäre würde würden zudem sowohl jedoch allerdings bereits rund etwa demnach zufolge angaben unter
-the and for with from that this have has are was were will would been into about after over their they
-his her its our you your not but all can new says said more than when what who how why off out also had
-one two which there been being some other could should would may might them then just only very
-'''.split())
-
-# Capitalised words that are no names (German nouns are capitalised too: those don't matter,
-# they never match an English text; these would match both).
-_NOT_NAMES = set('''
-Der Die Das Den Dem Des Ein Eine Einen Einem Einer Er Sie Es Wir Ihr Ich Und Oder Aber Auch Doch Mit Von
-Nach Bei Für Auf Aus Im In Am Zum Zur Vor Seit Als Wie Wenn Dass Nicht Noch Nur So Da Dann Hier Jetzt
-The A An And Or But With From For After Before In On At By To Of As It He She They We I This That These
-Those Mr Mrs Ms Dr Monday Tuesday Wednesday Thursday Friday Saturday Sunday Montag Dienstag Mittwoch
-Donnerstag Freitag Samstag Sonntag Sonnabend January February March April May June July August September
-October November December Januar Februar März Juni Juli Oktober Dezember Reuters AFP AP dpa BBC CNN
-'''.split())
-
-
-def plain(html):
-    return fetch.plain(html or '')
-
-
-def sentences(text):
-    text = re.sub(r'\s+', ' ', text or '').strip()
-    return [s.strip() for s in _SENT_END.split(text) if 40 <= len(s.strip()) <= 400]
-
-
-def _key(s):
-    """A sentence, the same however its numbers and spaces differ."""
-    return re.sub(r'\d+', '0', re.sub(r'\s+', ' ', s.lower())).strip()
-
-
-def words(text):
-    return [w.lower() for w in _TOKEN.findall(text or '') if len(w) > 2 and w.lower() not in STOP]
-
-
-# ---- furniture ------------------------------------------------------------------------------------
-
-FURNITURE_MIN = 3   # in this many of a source's articles: furniture
-
-
-def furniture(texts):
-    """The sentences (keys) that stand in at least FURNITURE_MIN of these texts (one source's)."""
-    seen = Counter()
-    for t in texts:
-        seen.update({_key(s) for s in _units(t)})
-    return {k for k, n in seen.items() if n >= FURNITURE_MIN}
-
-
-def _units(text):
-    return [u.strip() for u in re.split(r'\n|(?<=[.!?])\s+', text or '') if len(u.strip()) > 15]
-
-
-def without_furniture(text, keys):
-    if not keys:
-        return text
-    return ' '.join(u for u in _units(text) if _key(u) not in keys)
-
-
-_BLOCK = re.compile(r'<(p|li|h[1-6]|blockquote|figcaption|div)\b[^>]*>(.*?)</\1>', re.S | re.I)
-
-
-def html_without_furniture(html, keys):
-    """The article's HTML without the blocks whose text is (only) furniture."""
-    if not keys or not html:
-        return html
-
-    def keep(m):
-        inner = plain(m.group(2))
-        units = _units(inner)
-        if units and all(_key(u) in keys for u in units):
-            return ''
-        return m.group(0)
-    return _BLOCK.sub(keep, html)
-
-
-# ---- tf-idf over sentences -------------------------------------------------------------------------
-
-def _vectors(texts):
-    toks = [Counter(words(t)) for t in texts]
-    df = Counter()
-    for t in toks:
-        df.update(t.keys())
-    n = len(texts)
-    idf = {w: math.log((n + 1) / (d + 1)) + 1 for w, d in df.items()}
-    vecs = []
-    for t in toks:
-        v = {w: (1 + math.log(c)) * idf[w] for w, c in t.items()}
-        norm = math.sqrt(sum(x * x for x in v.values())) or 1.0
-        vecs.append({w: x / norm for w, x in v.items()})
-    return vecs
-
-
-def _cos(a, b):
-    if len(a) > len(b):
-        a, b = b, a
-    return sum(x * b.get(w, 0.0) for w, x in a.items())
-
-
-def summarize(docs, k=3, lam=0.7):
-    """docs: [(source, text)]. The k sentences that say most of what all say, in their order
-    of appearance: [(source, sentence)]."""
-    ss = [(src, s) for src, t in docs for s in sentences(t)[:30]]
-    if len(ss) <= k:
-        return ss
-    vecs = _vectors([s for _, s in ss])
-    n = len(ss)
-    sim = [[_cos(vecs[a], vecs[b]) if a != b else 0.0 for b in range(n)] for a in range(n)]
-    links = [[1.0 if sim[a][b] > 0.1 else 0.0 for b in range(n)] for a in range(n)]
-    deg = [sum(r) or 1.0 for r in links]
-    rank = [1.0 / n] * n
-    for _ in range(40):
-        rank = [0.15 / n + 0.85 * sum(links[b][a] * rank[b] / deg[b] for b in range(n)) for a in range(n)]
-    top = max(rank) or 1.0
-    chosen = []
-    for _ in range(k):
-        best, score = None, -1e9
-        for a in range(n):
-            if a in chosen:
-                continue
-            s = lam * rank[a] / top - (1 - lam) * max((sim[a][c] for c in chosen), default=0.0)
-            if s > score:
-                best, score = a, s
-        chosen.append(best)
-    return [ss[c] for c in sorted(chosen)]
-
-
-def novel(docs, per_source=2, limit=0.2):
-    """What only one report says: per source, up to per_source of its sentences that no other
-    source has anything like (cosine < limit), the most central of its own first."""
-    ss = [(src, s) for src, t in docs for s in sentences(t)[:40]]
-    if len({src for src, _ in ss}) < 2:
-        return {}
-    vecs = _vectors([s for _, s in ss])
-    out = defaultdict(list)
-    for a, (src, s) in enumerate(ss):
-        others = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 != src]
-        if not others or max(others) >= limit:
-            continue
-        own = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 == src and b != a]
-        out[src].append((sum(own) / len(own) if own else 0.0, s))
-    return {src: [s for _, s in sorted(v, key=lambda x: -x[0])[:per_source]] for src, v in out.items()}
-
-
-def word_choice(docs, per_source=6):
-    """Each source's own words for the event: log-odds of a word in its report against the
-    other reports (with a prior from all of them), at least twice in its report."""
-    forms = defaultdict(Counter)
-    counts = {}
-    for src, t in docs:
-        c = Counter()
-        for w in _TOKEN.findall(t or ''):
-            lw = w.lower()
-            if len(lw) > 3 and lw not in STOP:
-                c[lw] += 1
-                forms[lw][w] += 1
-        counts[src] = counts.get(src, Counter()) + c
-    if len(counts) < 2:
-        return {}
-    total = sum(counts.values(), Counter())
-    a0 = sum(total.values()) or 1
-    out = {}
-    for src, c in counts.items():
-        n_i = sum(c.values()) or 1
-        rest = total - c
-        n_j = sum(rest.values()) or 1
-        scored = []
-        for w, f in c.items():
-            if f < 2:
-                continue
-            aw = total[w] / a0 * 100
-            d = math.log((f + aw) / (n_i + 100 - f - aw)) - math.log((rest[w] + aw) / (n_j + 100 - rest[w] - aw))
-            scored.append((d / math.sqrt(1 / (f + aw) + 1 / (rest[w] + aw)), w))
-        # Nouns and names (as written: capitalised), not the verbs and particles around them.
-        best = [(z, forms[w].most_common(1)[0][0]) for z, w in sorted(scored, reverse=True) if z > 1.0]
-        out[src] = [w for z, w in best if w[:1].isupper()][:per_source]
-    return {k: v for k, v in out.items() if v}
-
-
-# ---- names, languages, related stories -------------------------------------------------------------
-
-_EN = set('the and of to in is that for with was said on as by at from'.split())
-_DE = set('der die das und ist nicht mit den von zu im sich auf für dem des eine sagte'.split())
-
-
-def language(text):
-    w = [x.lower() for x in _TOKEN.findall(text[:3000])]
-    en = sum(1 for x in w if x in _EN)
-    de = sum(1 for x in w if x in _DE)
-    return 'en' if en > de else 'de'
-
-
-def names(text):
-    """Capitalised words not at a sentence's start (in German: names and nouns; in English:
-    names), as a set of lowercased forms."""
-    out = set()
-    for s in sentences(text) or [text or '']:
-        toks = _TOKEN.findall(s)
-        for t in toks[1:]:
-            if t[0].isupper() and t not in _NOT_NAMES and len(t) > 2:
-                out.add(t.lower().rstrip('s’\''))
-    return out
-
-
-def related(stories, min_weight=17.0, per_story=3):
-    """stories: [(id, language, set of names)] of the same days. Pairs of stories in different
-    languages sharing rare names: {id: [(other id, weight)]}, weight = the shared names' idf."""
-    df = Counter()
-    for _, _, ns in stories:
-        df.update(ns)
-    n = len(stories)
-    idf = {w: math.log((n + 1) / (d + 1)) for w, d in df.items()}
-    out = defaultdict(list)
-    for a in range(n):
-        ia, la, na = stories[a]
-        for b in range(a + 1, n):
-            ib, lb, nb = stories[b]
-            if la == lb:
-                continue
-            shared = na & nb
-            if len(shared) < 3:
-                continue
-            w = sum(idf[x] for x in shared)
-            if w >= min_weight:
-                out[ia].append((ib, round(w, 1)))
-                out[ib].append((ia, round(w, 1)))
-    return {k: sorted(v, key=lambda x: -x[1])[:per_story] for k, v in out.items()}
-
-
-# ---- topics ---------------------------------------------------------------------------------------------
-
-NOT_TOPICS = set('breaking news free live watch video sekunden gebärdensprache podcast newsletter abo online'.split())
-
-
-TOPIC_MIN = 3
-
-
-def topics(stories, threshold=0.12, min_size=TOPIC_MIN, max_topics=8, rounds=3):
-    """stories: [(id, text, sources)] (titles and summaries, the set of source names). Groups of
-    stories about one matter, the largest first, each with a label (the words most of its stories
-    share): [{'label': str, 'ids': [ids]}]. Leader clustering against the groups' centroids, a
-    few rounds (each story to its nearest centroid, then the centroids anew); a group counts
-    when it has min_size stories from at least two sources."""
-    if len(stories) < min_size:
-        return []
-    vecs = _vectors([s[1] for s in stories])
-    order = sorted(range(len(stories)), key=lambda i: -len(vecs[i]))
-    cents = []
-    assign = [-1] * len(stories)
-    for _ in range(rounds):
-        changed = False
-        for i in order:
-            best, sim = -1, threshold
-            for g, c in enumerate(cents):
-                x = _cos(vecs[i], c)
-                if x > sim:
-                    best, sim = g, x
-            if best < 0:
-                cents.append(dict(vecs[i]))
-                best = len(cents) - 1
-            if assign[i] != best:
-                assign[i] = best
-                changed = True
-        # centroids anew from their members
-        acc = [Counter() for _ in cents]
-        for i, g in enumerate(assign):
-            for w, v in vecs[i].items():
-                acc[g][w] += v
-        cents = []
-        remap = {}
-        for g, a in enumerate(acc):
-            if a:
-                norm = math.sqrt(sum(v * v for v in a.values())) or 1.0
-                remap[g] = len(cents)
-                cents.append({w: v / norm for w, v in a.items()})
-        assign = [remap.get(g, -1) for g in assign]
-        if not changed:
-            break
-    groups = defaultdict(list)
-    for i, g in enumerate(assign):
-        groups[g].append(i)
-    out = []
-    source_words = {w.lower() for s in stories for src in s[2] for w in _TOKEN.findall(src)}
-    for g in sorted(groups.values(), key=len, reverse=True):
-        if len(g) < min_size or len({src for i in g for src in stories[i][2]}) < 2:
-            continue
-        # Close enough together: the stories' mean likeness to their group's centre.
-        acc = Counter()
-        for i in g:
-            for w, v in vecs[i].items():
-                acc[w] += v
-        norm = math.sqrt(sum(v * v for v in acc.values())) or 1.0
-        centre = {w: v / norm for w, v in acc.items()}
-        if sum(_cos(vecs[i], centre) for i in g) / len(g) < 0.3:
-            continue
-        df = Counter()
-        for i in g:
-            df.update({w for w in words(stories[i][1]) if w not in NOT_TOPICS and w not in source_words})
-        nice = []
-        for w, c in df.most_common(8):
-            if c < max(2, len(g) // 2):
-                break
-            forms = Counter(t for i in g for t in _TOKEN.findall(stories[i][1]) if t.lower() == w)
-            form = forms.most_common(1)[0][0] if forms else w
-            # Nouns and names make a label, not "rückt" or "näher"; "Linke" once, not "Linken" too.
-            if form[:1].isupper() and not any(x.lower()[:5] == form.lower()[:5] for x in nice):
-                nice.append(form)
-            if len(nice) == 3:
-                break
-        if len(nice) < 2:   # one shared word is a coincidence, not a topic
-            continue
-        out.append({'label': ' · '.join(nice), 'ids': [stories[i][0] for i in g]})
-        if len(out) >= max_topics:
-            break
-    return out

feeds/urls.py

@@ -15,7 +15,6 @@ urlpatterns = [
     path('briefing/seen/', views.briefing_seen, name='feeds_briefing_seen'),
     path('news/', views.news, name='feeds_news'),
     path('stories/<int:pk>/', views.story, name='feeds_story'),
-    path('stories/<int:pk>/overview/', views.story_overview, name='feeds_story_overview'),
     path('blogs/', views.blogs, name='feeds_blogs'),
     path('starred/', views.starred, name='feeds_starred'),
     path('items/read/', views.mark_read, name='feeds_mark_read'),

feeds/views.py

@@ -12,11 +12,8 @@ Sources:   GET  /feeds/state/                 sources, unread counts, settings
            POST /feeds/refresh/               refresh the account's sources now
 News:      GET  /feeds/briefing/              what's new since the briefing was last seen
            POST /feeds/briefing/seen/         seen: everything up to now counts as read
-           GET  /feeds/news/?before=&category=&ids=  stories, newest first (ids: just these, a topic's);
+           GET  /feeds/news/?before=&category=  stories, newest first
-                                              the first page also has the day's topics
            GET  /feeds/stories/<id>/          one story with all its items
-           GET  /feeds/stories/<id>/overview/ summary, what only one report says, each source's
-                                              words, the story in the other language (feeds/textproc.py)
 Blogs:     GET  /feeds/blogs/?folder=&source=&unread=1&fav=1&before=   (folder= alone: blogs without one;
                                               without folder/source the hidden blogs stay out)
 Items:     GET  /feeds/items/<id>/            the article (fetches the page if the feed had a teaser)
@@ -38,7 +35,6 @@ from xml.sax.saxutils import quoteattr
 
 import feedparser
 import requests
-from django.core.cache import cache
 from django.db.models import Count
 from django.http import HttpResponse, JsonResponse
 from django.utils import timezone
@@ -46,7 +42,7 @@ from django.utils.dateparse import parse_datetime
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_http_methods
 
-from . import fetch, logins, refresh, textproc
+from . import fetch, logins, refresh
 from .models import Item, SiteLogin, Source, Story
 
 PAGE = 40
@@ -75,15 +71,6 @@ def _body(request):
         return None
 
 
-def _when(value):
-    """A datetime parameter; None when there's none or it's garbled (an unencoded '+' arrives as
-    a space)."""
-    try:
-        return parse_datetime((value or '').strip().replace(' ', '+'))
-    except ValueError:
-        return None
-
-
 def _iso(dt):
     return dt.isoformat() if dt else None
 
@@ -492,76 +479,14 @@ def news(request):
     if (err := _auth(request)):
         return err
     qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at')
-    ids = [int(x) for x in request.GET.get('ids', '').split(',') if x.strip().isdigit()]
+    before = parse_datetime(request.GET.get('before') or '')
-    if ids:
-        stories = _stories(request.user, qs.filter(pk__in=ids[:PAGE]))
-        return JsonResponse({'stories': stories, 'more': False})
-    before = _when(request.GET.get('before'))
     if before:
         qs = qs.filter(last_at__lt=before)
     category = request.GET.get('category')
     if category:
         qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category)
     stories = _stories(request.user, qs[:PAGE])
-    out = {'stories': stories, 'more': len(stories) == PAGE}
+    return JsonResponse({'stories': stories, 'more': len(stories) == PAGE})
-    if not before:
-        topics = _news_text(request.user)['topics']
-        if category:
-            # The category's share of each topic.
-            mine = set(qs.values_list('pk', flat=True)[:500])
-            topics = [t for t in ({**t, 'ids': [i for i in t['ids'] if i in mine]} for t in topics)
-                      if len(t['ids']) >= textproc.TOPIC_MIN]
-        out['topics'] = topics
-    return JsonResponse(out)
-
-
-# ---- reading help (feeds/textproc.py)
-
-FURNITURE_ITEMS = 40          # a source's recent articles that show its furniture
-TEXT_DAYS = 2                 # topics and related stories: of these days
-TEXT_CACHE = 6 * 3600
-
-
-def _furniture(source):
-    key = f'feeds:furniture:{source.id}'
-    keys = cache.get(key)
-    if keys is None:
-        texts = [textproc.plain(c) for c in Item.objects.filter(source=source, gone=False)
-                 .order_by('-published').values_list('content', flat=True)[:FURNITURE_ITEMS]]
-        keys = textproc.furniture(texts)
-        cache.set(key, keys, TEXT_CACHE)
-    return keys
-
-
-def _clean_text(i):
-    """An item's text without its source's furniture, plain."""
-    return textproc.without_furniture(textproc.plain(i.content) or i.summary, _furniture(i.source))
-
-
-def _news_text(user):
-    """The news of the last days, looked at as a whole: topics [{label, ids}] and related
-    stories in the other language {story id: [ids]}. Cached until a story changes."""
-    since = timezone.now() - timedelta(days=TEXT_DAYS)
-    qs = Story.objects.filter(user=user, last_at__gte=since, items__gone=False).distinct()
-    latest = qs.order_by('-last_at').values_list('last_at', flat=True).first()
-    key = f'feeds:newstext:{user.id}:{qs.count()}:{latest.timestamp() if latest else 0}'
-    got = cache.get(key)
-    if got is not None:
-        return got
-    items = {}
-    for i in Item.objects.filter(story__in=qs, gone=False).select_related('source'):
-        items.setdefault(i.story_id, []).append(i)
-    tl, sl = [], []
-    for sid, its in items.items():
-        tl.append((sid, ' '.join(f'{i.title}. {i.summary}' for i in its), {i.source.title for i in its}))
-        text = ' '.join(f'{i.title}. {_clean_text(i)[:4000]}' for i in its)
-        sl.append((sid, textproc.language(text), textproc.names(text)))
-    got = {
-        'topics': textproc.topics(tl),
-        'related': {sid: [b for b, _ in lst] for sid, lst in textproc.related(sl).items()},
-    }
-    cache.set(key, got, TEXT_CACHE)
-    return got
 
 
 @csrf_exempt
@@ -575,32 +500,6 @@ def story(request, pk):
     return JsonResponse({'story': found[0]})
 
 
-@csrf_exempt
-@require_http_methods(['GET'])
-def story_overview(request, pk):
-    """What a story's reports say together: a summary from all of them, what only one says,
-    each source's own words, and the same story in the other language."""
-    if (err := _auth(request)):
-        return err
-    s = Story.objects.filter(user=request.user, pk=pk).first()
-    if s is None:
-        return JsonResponse({'error': 'not found'}, status=404)
-    its = list(Item.objects.filter(story=s, gone=False).select_related('source').order_by('published'))
-    key = f'feeds:overview:{s.id}:{len(its)}:{sum(len(i.content) for i in its)}'
-    out = cache.get(key)
-    if out is None:
-        docs = [(i.source.title, _clean_text(i)) for i in its]
-        out = {}
-        if len({src for src, _ in docs}) >= 2:
-            out['summary'] = [{'source': src, 'text': t} for src, t in textproc.summarize(docs)]
-            out['novel'] = textproc.novel(docs)
-            out['words'] = textproc.word_choice(docs)
-        cache.set(key, out, TEXT_CACHE)
-    rel = _news_text(request.user)['related'].get(s.id, [])
-    related = _stories(request.user, Story.objects.filter(user=request.user, pk__in=rel)) if rel else []
-    return JsonResponse({**out, 'related': related})
-
-
 # ---- blogs, items
 
 @csrf_exempt
@@ -623,7 +522,7 @@ def blogs(request):
     unread_only = request.GET.get('unread') == '1'
     if unread_only:
         qs = qs.filter(read_at=None)
-    before = _when(request.GET.get('before'))
+    before = parse_datetime(request.GET.get('before') or '')
     if before:
         qs = qs.filter(published__lt=before)
     # Unread: all of them at once (the page sorts them into days, favorites, rare blogs).
@@ -651,9 +550,7 @@ def item(request, pk):
         return JsonResponse({'error': 'not found'}, status=404)
     if refresh.needs_article(i, i.source):
         refresh.load_article(i)
-    d = _item_data(i, content=True)
+    return JsonResponse({'item': _item_data(i, content=True)})
-    d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source))
-    return JsonResponse({'item': d})
 
 
 @csrf_exempt