Feed-Reader (Server): Nachrichten und Blogs, Volltext, Meldungen bündeln

Neue App feeds: Quellen sind entweder Nachrichten oder Blog.
- Abruf wie bei Podcasts (ETag, Body-Hash, Backoff), refresh_newsfeeds für den Cron
- Anrisstexte: der Artikel wird von der Seite geholt (trafilatura), HTML bereinigt,
  relative Links absolut
- Nachrichten: gleiche Meldung aus mehreren Quellen wird eine Story (tf-idf, je
  Artikel verglichen), Briefing „Seit letztem Mal“ (Top-Themen, dann nach Ressort),
  verfallen nach news_keep_days (Standard 2)
- Blogs: Ungelesen-Liste je Blog, bleiben bis gelesen
- API unter /feeds/: Quellen suchen/abonnieren/OPML, News, Stories, Blogs,
  Artikel, gelesen, merken

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

diora/settings.py

@@ -31,6 +31,7 @@ INSTALLED_APPS = [
     'books',
     'gpodder',
     'tts',
+    'feeds',
 ]
 
 EBOOK_MAX_BYTES = 50 * 1024 * 1024  # 50 MB

diora/urls.py

@@ -11,6 +11,7 @@ urlpatterns = [
     path('accounts/', include('accounts.urls')),
     path('podcasts/', include('podcasts.urls')),
     path('books/', include('books.urls')),
+    path('feeds/', include('feeds.urls')),
     path('tts/', include('tts.urls')),
     path('api/2/', include('gpodder.urls')),
     path('api/sync/', sync_snapshot, name='api_sync'),

feeds/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class FeedsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'feeds'

feeds/cluster.py

@@ -0,0 +1,149 @@
+"""Bundling news items about the same event into stories.
+
+Each item is described by the words of its title (counted twice) and summary,
+without stop words and with a crude German/English suffix cut. A new item joins
+the most similar story of the last WINDOW hours (cosine of tf-idf vectors,
+document frequencies over those stories) if that similarity reaches THRESHOLD
+and they share at least MIN_SHARED words; otherwise it starts a story of its own.
+A story many sources report on is a top story.
+"""
+
+import math
+import re
+from collections import Counter
+from datetime import timedelta
+
+from .models import Item, Story
+
+WINDOW = timedelta(hours=36)
+THRESHOLD = 0.25
+MIN_SHARED = 2
+
+STOP = set('''
+aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher
+da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese
+diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für
+gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer
+in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen
+nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind
+so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter
+weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier
+heute gestern morgen laut sagt sagte erst mal neu alt video live ticker liveblog news meldung bericht
+the and for with from that this have has are was were will would been into about after over their they
+his her its our you your not but all can new says said more than when what who how why off out
+tot tote toten getötet sterben stirbt gestorben menschen mensch leute millionen milliarden prozent euro
+deutschland deutsche deutschen land länder stadt welt woche wochen monat monate tag tage tagen uhr
+frau frauen mann männer kinder erste ersten zweite neuer neues großen große groß kleine wegen gibt geben
+'''.split())
+
+_WORD = re.compile(r'[a-zäöüß][a-zäöüß0-9\-]+')
+_SUFFIXES = ('ungen', 'ung', 'en', 'er', 'es', 'e', 's', 'n')
+
+
+def _stem(w):
+    if len(w) > 5:
+        for s in _SUFFIXES:
+            if w.endswith(s) and len(w) - len(s) >= 4:
+                return w[:-len(s)]
+    return w
+
+
+def words(text):
+    out = []
+    for w in _WORD.findall((text or '').lower()):
+        w = w.strip('-')
+        if len(w) < 3 or w in STOP:
+            continue
+        out.append(_stem(w))
+    return out
+
+
+def item_terms(title, summary):
+    return Counter(words(title) * 2 + words(summary[:300]))
+
+
+def _cosine(a, b, idf):
+    dot = sum(a[w] * b[w] * idf.get(w, 1.0) ** 2 for w in a.keys() & b.keys())
+    if not dot:
+        return 0.0
+    na = math.sqrt(sum((n * idf.get(w, 1.0)) ** 2 for w, n in a.items()))
+    nb = math.sqrt(sum((n * idf.get(w, 1.0)) ** 2 for w, n in b.items()))
+    return dot / (na * nb)
+
+
+class Clusterer:
+    """Assigns one user's new news items to stories (load once, add items, done).
+
+    An item is compared with every item already in a story: the story's score
+    is the mean of the best match and the average match, so one stray shared
+    word can't pull an item in, and a story doesn't grow into a magnet the way
+    a pooled word list would. Items of the same source join only when very
+    close (a source rarely reports one event twice, but often two similar ones)."""
+
+    SAME_SOURCE = 0.5
+
+    def __init__(self, user, now):
+        self.user = user
+        self.stories = list(Story.objects.filter(user=user, last_at__gte=now - WINDOW))
+        self.members = {s.pk: [] for s in self.stories}   # story id -> [(source id, terms)]
+        for i in Item.objects.filter(story__in=self.stories, gone=False).only('story_id', 'source_id', 'title', 'summary'):
+            self.members[i.story_id].append((i.source_id, item_terms(i.title, i.summary)))
+        self._df = Counter()
+        for ms in self.members.values():
+            for _, t in ms:
+                self._df.update(t.keys())
+        self._n = sum(len(ms) for ms in self.members.values())
+
+    def _idf(self):
+        n = self._n + 1
+        return {w: math.log((n + 1) / (d + 1)) + 1 for w, d in self._df.items()}
+
+    def score(self, t, source_id, members, idf):
+        sims = []
+        for sid, mt in members:
+            if len(t.keys() & mt.keys()) < MIN_SHARED:
+                sims.append(0.0)
+                continue
+            sim = _cosine(t, mt, idf)
+            if sid == source_id and sim < self.SAME_SOURCE:
+                sim = 0.0
+            sims.append(sim)
+        if not sims or max(sims) == 0:
+            return 0.0
+        return (max(sims) + sum(sims) / len(sims)) / 2
+
+    def add(self, item):
+        t = item_terms(item.title, item.summary)
+        best, best_sim = None, 0.0
+        if t:
+            idf = self._idf()
+            for s in self.stories:
+                if abs((s.last_at - item.published).total_seconds()) > WINDOW.total_seconds():
+                    continue
+                sim = self.score(t, item.source_id, self.members[s.pk], idf)
+                if sim > best_sim:
+                    best, best_sim = s, sim
+        if best is not None and best_sim >= THRESHOLD:
+            self._join(best, item)
+        else:
+            best = Story.objects.create(user=self.user, lead=item, first_at=item.published,
+                                        last_at=item.published, size=1, sources=1)
+            self.stories.append(best)
+            self.members[best.pk] = []
+            item.story = best
+            item.save(update_fields=['story'])
+        self.members[best.pk].append((item.source_id, t))
+        self._df.update(t.keys())
+        self._n += 1
+        return best
+
+    def _join(self, s, item):
+        item.story = s
+        item.save(update_fields=['story'])
+        s.size = Item.objects.filter(story=s).count()
+        s.sources = Item.objects.filter(story=s).values('source').distinct().count()
+        s.first_at = min(s.first_at, item.published)
+        s.last_at = max(s.last_at, item.published)
+        if s.lead is None or (not s.lead.image_url and item.image_url):
+            s.lead = item
+        s.save(update_fields=['size', 'sources', 'first_at', 'last_at', 'lead'])

feeds/fetch.py

@@ -0,0 +1,231 @@
+"""Reading feeds: entries into items, the whole article from its page, finding a
+site's feed. Downloading the feed itself is podcasts.views._fetch_feed (ETag,
+body hash, redirects), shared with the podcast refresh."""
+
+import calendar
+import hashlib
+import html
+import logging
+import re
+from datetime import datetime, timezone as dt_timezone
+from html.parser import HTMLParser
+from urllib.parse import urljoin, urlparse
+
+import requests
+from django.utils import timezone
+
+from podcasts.shownotes import _Sanitizer, html_to_text
+
+log = logging.getLogger(__name__)
+
+USER_AGENT = 'Mozilla/5.0 (compatible; diora feed reader; +https://diora.creamfresh.xyz)'
+PAGE_TIMEOUT = 15
+# Shorter text than this in the feed counts as a teaser: the article is fetched ("auto").
+TEASER_CHARS = 700
+SUMMARY_CHARS = 400
+
+
+class _AbsoluteSanitizer(_Sanitizer):
+    """The podcast sanitizer, with links and images made absolute first
+    (article HTML is full of relative ones, which the allowlist would drop)."""
+
+    def __init__(self, base):
+        super().__init__()
+        self.base = base
+
+    def _start(self, tag, attrs, closed):
+        if tag == 'img':
+            # Lazy-loaded images keep the real address in data-src, src is a placeholder.
+            a = dict(attrs)
+            real = a.get('data-src') or a.get('data-original')
+            if real:
+                attrs = [(n, v) for n, v in attrs if n not in ('src', 'data-src', 'data-original')] + [('src', real)]
+        fixed = []
+        for name, value in attrs:
+            if name in ('href', 'src') and value and not value.strip().startswith(('data:', 'javascript:', '#')):
+                value = urljoin(self.base, value.strip())
+            fixed.append((name, value))
+        super()._start(tag, fixed, closed)
+
+
+def clean_html(text, base=''):
+    if not text:
+        return ''
+    s = _AbsoluteSanitizer(base)
+    s.feed(text)
+    return s.result().strip()
+
+
+def plain(text):
+    return re.sub(r'\s+', ' ', html_to_text(text or '')).strip()
+
+
+def _short(text, n=SUMMARY_CHARS):
+    if len(text) <= n:
+        return text
+    cut = text[:n].rsplit(' ', 1)[0]
+    return cut + ' …'
+
+
+def _when(entry):
+    """The entry's time (published, else updated), never in the future."""
+    now = timezone.now()
+    for key in ('published_parsed', 'updated_parsed', 'created_parsed'):
+        t = entry.get(key)
+        if t:
+            try:
+                when = datetime.fromtimestamp(calendar.timegm(t), dt_timezone.utc)
+            except (OverflowError, ValueError, TypeError):
+                continue
+            return min(when, now)
+    return now
+
+
+class _FirstImage(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.src = ''
+
+    def handle_starttag(self, tag, attrs):
+        if tag == 'img' and not self.src:
+            a = dict(attrs)
+            w, h = a.get('width', ''), a.get('height', '')
+            if w in ('1', '0') or h in ('1', '0'):
+                return  # tracking pixel
+            self.src = a.get('src') or ''
+
+
+def _image(entry, content_html, base):
+    for m in entry.get('media_thumbnail') or []:
+        if m.get('url'):
+            return m['url']
+    for m in entry.get('media_content') or []:
+        if m.get('url') and (m.get('medium') == 'image' or (m.get('type') or '').startswith('image/')):
+            return m['url']
+    for enc in entry.get('enclosures') or []:
+        if (enc.get('type') or '').startswith('image/') and enc.get('href'):
+            return enc['href']
+    p = _FirstImage()
+    try:
+        p.feed(content_html or '')
+    except Exception:
+        pass
+    return urljoin(base, p.src) if p.src else ''
+
+
+def guid_hash(entry):
+    key = entry.get('id') or entry.get('link') or f"{entry.get('title', '')}|{entry.get('published', '')}"
+    return hashlib.sha256(key.encode('utf-8', 'replace')).hexdigest()
+
+
+def parse_entry(entry, feed_url):
+    """A feedparser entry -> the fields of an Item (without source/user)."""
+    link = entry.get('link') or ''
+    base = link or feed_url
+    contents = [c.get('value') or '' for c in entry.get('content') or []]
+    content_html = max(contents, key=len) if contents else ''
+    summary_html = entry.get('summary') or ''
+    if len(plain(summary_html)) > len(plain(content_html)):
+        content_html = summary_html
+    content = clean_html(content_html, base)
+    text = plain(content)
+    summary = plain(summary_html) or text
+    title = plain(html.unescape(entry.get('title') or '')) or _short(text, 80) or 'Ohne Titel'
+    return {
+        'guid_hash': guid_hash(entry),
+        'url': link[:2000],
+        'title': title[:500],
+        'author': plain(entry.get('author') or '')[:300],
+        'published': _when(entry),
+        'summary': _short(summary),
+        'content': content,
+        'image_url': _image(entry, content_html, base)[:2000],
+    }
+
+
+def is_teaser(content_html):
+    return len(plain(content_html)) < TEASER_CHARS
+
+
+def _drop_title(html_text):
+    """trafilatura repeats the headline as the first <h1>; the reader shows it already."""
+    return re.sub(r'^\s*(<img[^>]*>\s*)?<h1>.*?</h1>', lambda m: m.group(1) or '', html_text, count=1, flags=re.S)
+
+
+def fetch_article(url):
+    """The article on `url` as (clean HTML, image URL), or None. trafilatura
+    finds the main text and leaves out navigation, ads, comments."""
+    import trafilatura  # heavy import, only where articles are fetched
+
+    try:
+        r = requests.get(url, timeout=PAGE_TIMEOUT, headers={'User-Agent': USER_AGENT})
+    except requests.RequestException as e:
+        log.info('article %s: %s', url, type(e).__name__)
+        return None
+    if r.status_code >= 400 or 'html' not in r.headers.get('Content-Type', 'text/html'):
+        return None
+    page = r.text
+    try:
+        body = trafilatura.extract(page, url=r.url, output_format='html', include_images=True,
+                                   include_links=True, include_tables=True, favor_precision=True)
+    except Exception:
+        log.exception('article %s: extraction failed', url)
+        return None
+    if not body:
+        return None
+    body = re.sub(r'^\s*<html>\s*<body>|</body>\s*</html>\s*$', '', body.strip())
+    content = clean_html(_drop_title(body), r.url)
+    if len(plain(content)) < 200:
+        return None
+    image = ''
+    try:
+        meta = trafilatura.extract_metadata(page, default_url=r.url)
+        image = (meta.image if meta else '') or ''
+    except Exception:
+        pass
+    return content, image
+
+
+# ---- finding a site's feed
+
+FEED_TYPES = ('application/rss+xml', 'application/atom+xml', 'application/feed+json', 'application/xml', 'text/xml')
+
+
+class _FeedLinks(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.feeds = []
+        self.icon = ''
+
+    def handle_starttag(self, tag, attrs):
+        if tag != 'link':
+            return
+        a = {k.lower(): (v or '') for k, v in attrs}
+        rel = a.get('rel', '').lower().split()
+        if 'alternate' in rel and a.get('type', '').lower() in FEED_TYPES and a.get('href'):
+            self.feeds.append((a['href'], a.get('title', '')))
+        if ('icon' in rel or 'apple-touch-icon' in rel) and a.get('href') and not self.icon:
+            self.icon = a['href']
+
+
+def discover(page_html, base):
+    """[(feed url, title)] a page links to, and its icon URL."""
+    p = _FeedLinks()
+    try:
+        p.feed(page_html)
+    except Exception:
+        pass
+    seen, out = set(), []
+    for href, title in p.feeds:
+        u = urljoin(base, href)
+        if u not in seen:
+            seen.add(u)
+            out.append((u, title))
+    return out, (urljoin(base, p.icon) if p.icon else '')
+
+
+def site_icon(site_url):
+    if not site_url:
+        return ''
+    u = urlparse(site_url)
+    return f'{u.scheme}://{u.netloc}/favicon.ico' if u.scheme and u.netloc else ''

feeds/management/commands/refresh_newsfeeds.py

@@ -0,0 +1,25 @@
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from feeds.models import Source
+from feeds.refresh import refresh
+from podcasts.views import _in_backoff_q
+
+
+class Command(BaseCommand):
+    help = 'Refresh the feed reader\'s sources (news and blogs)'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--source-id', type=int, help='Refresh only this source (even when backing off)')
+
+    def handle(self, *args, **options):
+        qs = Source.objects.select_related('user').order_by('last_refreshed_at')
+        if options['source_id']:
+            qs = qs.filter(pk=options['source_id'])
+        else:
+            qs = qs.exclude(_in_backoff_q(timezone.now()))
+        if not qs.exists():
+            self.stdout.write('No sources to refresh.')
+            return
+        refreshed, failed, new = refresh(qs)
+        self.stdout.write(f'{refreshed} sources refreshed, {failed} failed, +{new} items')

feeds/migrations/0001_initial.py

@@ -0,0 +1,111 @@
+# Generated by Django 6.1 on 2026-10-04 17:54
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='FeedSettings',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('news_seen_at', models.DateTimeField(blank=True, null=True)),
+                ('news_keep_days', models.PositiveIntegerField(default=2)),
+                ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='feed_settings', to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Source',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('url', models.URLField(max_length=1000)),
+                ('title', models.CharField(max_length=300)),
+                ('site_url', models.URLField(blank=True, max_length=1000)),
+                ('icon_url', models.URLField(blank=True, max_length=1000)),
+                ('kind', models.CharField(choices=[('news', 'Nachrichten'), ('blog', 'Blog')], default='news', max_length=10)),
+                ('category', models.CharField(blank=True, max_length=100)),
+                ('fulltext', models.CharField(choices=[('auto', 'Automatisch'), ('always', 'Immer'), ('never', 'Nie')], default='auto', max_length=10)),
+                ('added_at', models.DateTimeField(auto_now_add=True)),
+                ('last_refreshed_at', models.DateTimeField(blank=True, null=True)),
+                ('http_etag', models.CharField(blank=True, max_length=500)),
+                ('http_last_modified', models.CharField(blank=True, max_length=100)),
+                ('http_body_hash', models.CharField(blank=True, max_length=64)),
+                ('last_error', models.CharField(blank=True, max_length=500)),
+                ('last_error_at', models.DateTimeField(blank=True, null=True)),
+                ('error_count', models.PositiveIntegerField(default=0)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_sources', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'ordering': ['title'],
+                'unique_together': {('user', 'url')},
+            },
+        ),
+        migrations.CreateModel(
+            name='Item',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('kind', models.CharField(choices=[('news', 'Nachrichten'), ('blog', 'Blog')], max_length=10)),
+                ('guid_hash', models.CharField(max_length=64)),
+                ('url', models.URLField(blank=True, max_length=2000)),
+                ('title', models.CharField(max_length=500)),
+                ('author', models.CharField(blank=True, max_length=300)),
+                ('published', models.DateTimeField(db_index=True)),
+                ('fetched_at', models.DateTimeField(auto_now_add=True)),
+                ('summary', models.TextField(blank=True)),
+                ('content', models.TextField(blank=True)),
+                ('content_full', models.BooleanField(default=False)),
+                ('fulltext_failed', models.BooleanField(default=False)),
+                ('image_url', models.URLField(blank=True, max_length=2000)),
+                ('read_at', models.DateTimeField(blank=True, null=True)),
+                ('starred', models.BooleanField(default=False)),
+                ('gone', models.BooleanField(default=False)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_items', to=settings.AUTH_USER_MODEL)),
+                ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='feeds.source')),
+            ],
+            options={
+                'ordering': ['-published'],
+            },
+        ),
+        migrations.CreateModel(
+            name='Story',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('first_at', models.DateTimeField()),
+                ('last_at', models.DateTimeField(db_index=True)),
+                ('size', models.PositiveIntegerField(default=1)),
+                ('sources', models.PositiveIntegerField(default=1)),
+                ('terms', models.TextField(blank=True)),
+                ('lead', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='feeds.item')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_stories', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'ordering': ['-last_at'],
+            },
+        ),
+        migrations.AddField(
+            model_name='item',
+            name='story',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='items', to='feeds.story'),
+        ),
+        migrations.AddIndex(
+            model_name='item',
+            index=models.Index(fields=['user', 'kind', 'published'], name='feeds_item_user_id_1a8bce_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='item',
+            index=models.Index(fields=['user', 'kind', 'read_at'], name='feeds_item_user_id_0007dd_idx'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='item',
+            unique_together={('source', 'guid_hash')},
+        ),
+    ]

feeds/models.py

@@ -0,0 +1,106 @@
+from django.contrib.auth.models import User
+from django.db import models
+
+
+class Source(models.Model):
+    """A subscribed feed. Every source is either news or a blog, and that shapes
+    everything after it: news gets bundled into stories, shows up in the
+    "Seit letztem Mal" briefing and expires after a few days; blog articles stay
+    until read and are listed per blog."""
+    NEWS = 'news'
+    BLOG = 'blog'
+    KINDS = [(NEWS, 'Nachrichten'), (BLOG, 'Blog')]
+
+    # Whether the whole article is fetched from its page: "auto" when the feed
+    # only carries a teaser.
+    FULLTEXT_AUTO = 'auto'
+    FULLTEXT_ALWAYS = 'always'
+    FULLTEXT_NEVER = 'never'
+    FULLTEXTS = [(FULLTEXT_AUTO, 'Automatisch'), (FULLTEXT_ALWAYS, 'Immer'), (FULLTEXT_NEVER, 'Nie')]
+
+    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_sources')
+    url = models.URLField(max_length=1000)
+    title = models.CharField(max_length=300)
+    site_url = models.URLField(max_length=1000, blank=True)
+    icon_url = models.URLField(max_length=1000, blank=True)
+    kind = models.CharField(max_length=10, choices=KINDS, default=NEWS)
+    # News: the section it belongs to (Politik, Tech …), groups the briefing.
+    category = models.CharField(max_length=100, blank=True)
+    fulltext = models.CharField(max_length=10, choices=FULLTEXTS, default=FULLTEXT_AUTO)
+    added_at = models.DateTimeField(auto_now_add=True)
+
+    # Refresh bookkeeping, as for podcast feeds (see podcasts.views._fetch_feed).
+    last_refreshed_at = models.DateTimeField(null=True, blank=True)
+    http_etag = models.CharField(max_length=500, blank=True)
+    http_last_modified = models.CharField(max_length=100, blank=True)
+    http_body_hash = models.CharField(max_length=64, blank=True)
+    last_error = models.CharField(max_length=500, blank=True)
+    last_error_at = models.DateTimeField(null=True, blank=True)
+    error_count = models.PositiveIntegerField(default=0)
+
+    class Meta:
+        unique_together = ('user', 'url')
+        ordering = ['title']
+
+    def __str__(self):
+        return f'{self.title} ({self.kind}, user={self.user_id})'
+
+
+class Story(models.Model):
+    """News items about the same event from one or more sources (see cluster.py)."""
+    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_stories')
+    # The item the story is shown with (the first one, or the longest text).
+    lead = models.ForeignKey('Item', null=True, blank=True, on_delete=models.SET_NULL, related_name='+')
+    first_at = models.DateTimeField()
+    last_at = models.DateTimeField(db_index=True)
+    size = models.PositiveIntegerField(default=1)      # items in it
+    sources = models.PositiveIntegerField(default=1)   # distinct sources among them
+    # Words that describe it (space-separated, for matching new items).
+    terms = models.TextField(blank=True)
+
+    class Meta:
+        ordering = ['-last_at']
+
+
+class Item(models.Model):
+    source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name='items')
+    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_items')
+    kind = models.CharField(max_length=10, choices=Source.KINDS)  # the source's, at the time
+    guid_hash = models.CharField(max_length=64)  # sha256 of the entry's id (or link)
+    url = models.URLField(max_length=2000, blank=True)
+    title = models.CharField(max_length=500)
+    author = models.CharField(max_length=300, blank=True)
+    published = models.DateTimeField(db_index=True)
+    fetched_at = models.DateTimeField(auto_now_add=True)
+    summary = models.TextField(blank=True)   # plain text, short
+    content = models.TextField(blank=True)   # sanitized HTML
+    # The content came from the article's page, not the feed.
+    content_full = models.BooleanField(default=False)
+    # Fetching the page failed or gave nothing: don't try on every open.
+    fulltext_failed = models.BooleanField(default=False)
+    image_url = models.URLField(max_length=2000, blank=True)
+    read_at = models.DateTimeField(null=True, blank=True)
+    starred = models.BooleanField(default=False)
+    story = models.ForeignKey(Story, null=True, blank=True, on_delete=models.SET_NULL, related_name='items')
+    # Expired news: the row stays a while without its text, so the entry isn't
+    # taken in again as new while the feed still carries it (refresh.expire).
+    gone = models.BooleanField(default=False)
+
+    class Meta:
+        unique_together = ('source', 'guid_hash')
+        indexes = [
+            models.Index(fields=['user', 'kind', 'published']),
+            models.Index(fields=['user', 'kind', 'read_at']),
+        ]
+        ordering = ['-published']
+
+    def __str__(self):
+        return self.title
+
+
+class FeedSettings(models.Model):
+    user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='feed_settings')
+    # The last time the news briefing was seen; what came after is "Seit letztem Mal".
+    news_seen_at = models.DateTimeField(null=True, blank=True)
+    # News older than this goes (unless starred).
+    news_keep_days = models.PositiveIntegerField(default=2)

feeds/refresh.py

@@ -0,0 +1,222 @@
+"""Refreshing sources: new entries become items, teasers get their article,
+news is bundled into stories, old news expires."""
+
+import logging
+import time
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
+from datetime import timedelta
+
+from django.db import IntegrityError, transaction
+from django.utils import timezone
+
+from podcasts.views import FeedFetchError, _clear_error, _error_text, _fetch_feed
+
+from . import fetch
+from .cluster import Clusterer
+from .models import FeedSettings, Item, Source, Story
+
+log = logging.getLogger(__name__)
+
+FEED_WORKERS = 12
+ARTICLE_WORKERS = 8
+ARTICLES_PER_RUN = 80        # articles fetched per refresh at most; the rest when opened
+BLOG_KEEP = 1000             # items kept per blog
+BLOG_FIRST_UNREAD_DAYS = 14  # a new blog's older articles start out read
+GONE_KEEP_DAYS = 30          # expired news rows kept (without text) against re-adding
+
+
+def settings_for(user):
+    s, _ = FeedSettings.objects.get_or_create(user=user)
+    return s
+
+
+def _apply(source, result, now, keep_days):
+    """Write one download; returns the new items."""
+    if result.status == 304:
+        source.last_refreshed_at = now
+        _clear_error(source)
+        source.save(update_fields=['last_refreshed_at', 'last_error', 'last_error_at', 'error_count',
+                                   'http_etag', 'http_last_modified'])
+        return []
+    parsed = result.parsed
+    feed = parsed.feed
+    if feed.get('title') and (not source.title or source.title == source.url):
+        source.title = fetch.plain(feed.title)[:300]
+    if feed.get('link'):
+        source.site_url = feed.link[:1000]
+    if not source.icon_url:
+        icon = (feed.get('image') or {}).get('href') or feed.get('icon') or fetch.site_icon(source.site_url)
+        source.icon_url = (icon or '')[:1000]
+    if result.permanent_url and result.permanent_url != source.url and \
+            not Source.objects.filter(user=source.user, url=result.permanent_url).exists():
+        source.url = result.permanent_url[:1000]
+    source.http_etag = result.etag
+    source.http_last_modified = result.last_modified
+    source.http_body_hash = result.body_hash
+    source.last_refreshed_at = now
+    _clear_error(source)
+    first = not source.items.exists()
+    source.save()
+
+    known = set(source.items.values_list('guid_hash', flat=True))
+    oldest_kept = None
+    if source.kind == Source.BLOG and len(known) >= BLOG_KEEP:
+        oldest_kept = source.items.order_by('published').values_list('published', flat=True).first()
+    news_cutoff = now - timedelta(days=keep_days)
+    new = []
+    for entry in parsed.entries:
+        fields = fetch.parse_entry(entry, source.url)
+        if fields['guid_hash'] in known:
+            continue
+        known.add(fields['guid_hash'])
+        if source.kind == Source.NEWS and fields['published'] < news_cutoff:
+            continue
+        if oldest_kept and fields['published'] < oldest_kept:
+            continue
+        item = Item(source=source, user=source.user, kind=source.kind, **fields)
+        if source.kind == Source.BLOG and first and fields['published'] < now - timedelta(days=BLOG_FIRST_UNREAD_DAYS):
+            item.read_at = now
+        new.append(item)
+    if new:
+        try:
+            with transaction.atomic():
+                Item.objects.bulk_create(new)
+        except IntegrityError:
+            # Two entries of one feed with the same id: one by one, skipping the doubles.
+            kept = []
+            for item in new:
+                try:
+                    with transaction.atomic():
+                        item.save()
+                    kept.append(item)
+                except IntegrityError:
+                    pass
+            new = kept
+    return new
+
+
+def needs_article(item, source):
+    if not item.url or item.content_full or item.fulltext_failed:
+        return False
+    if source.fulltext == Source.FULLTEXT_NEVER:
+        return False
+    return source.fulltext == Source.FULLTEXT_ALWAYS or fetch.is_teaser(item.content)
+
+
+def load_article(item):
+    """Fetch the article for one item now (when opened); saves it."""
+    got = fetch.fetch_article(item.url)
+    _store_article(item, got)
+    return item
+
+
+def _store_article(item, got):
+    if got is None:
+        item.fulltext_failed = True
+        item.save(update_fields=['fulltext_failed'])
+        return
+    content, image = got
+    if len(fetch.plain(content)) <= len(fetch.plain(item.content)):
+        item.fulltext_failed = True  # the feed had more
+        item.save(update_fields=['fulltext_failed'])
+        return
+    item.content = content
+    item.content_full = True
+    if not item.image_url and image:
+        item.image_url = image[:2000]
+    item.save(update_fields=['content', 'content_full', 'image_url'])
+
+
+def _articles(items, deadline):
+    todo = [i for i in items if needs_article(i, i.source)][:ARTICLES_PER_RUN]
+    if not todo:
+        return
+    pool = ThreadPoolExecutor(ARTICLE_WORKERS)
+    futures = {pool.submit(fetch.fetch_article, i.url): i for i in todo}
+    pending = set(futures)
+    try:
+        while pending:
+            left = None if deadline is None else deadline - time.monotonic()
+            if left is not None and left <= 0:
+                break
+            done, pending = wait(pending, timeout=left, return_when=FIRST_COMPLETED)
+            for f in done:
+                try:
+                    _store_article(futures[f], f.result())
+                except Exception:
+                    log.exception('article for item %s', futures[f].pk)
+    finally:
+        pool.shutdown(wait=False, cancel_futures=True)
+
+
+def expire(user, now=None):
+    """News past the user's keep time loses its text (rows stay GONE_KEEP_DAYS
+    as markers), stories without visible items go."""
+    now = now or timezone.now()
+    keep = settings_for(user).news_keep_days
+    Item.objects.filter(user=user, kind=Source.NEWS, gone=False, starred=False,
+                        published__lt=now - timedelta(days=keep)).update(
+        gone=True, content='', summary='', story=None)
+    Item.objects.filter(user=user, gone=True, published__lt=now - timedelta(days=GONE_KEEP_DAYS)).delete()
+    Story.objects.filter(user=user).exclude(items__gone=False).delete()
+    # Blogs: the oldest read articles beyond BLOG_KEEP per blog.
+    for source in Source.objects.filter(user=user, kind=Source.BLOG):
+        ids = list(source.items.order_by('-published').values_list('id', flat=True)[BLOG_KEEP:])
+        if ids:
+            Item.objects.filter(id__in=ids, starred=False).exclude(read_at=None).delete()
+
+
+def refresh(sources, deadline=None):
+    """Refresh `sources` (downloads in parallel, writes one by one).
+    Returns (refreshed, failed, new items)."""
+    sources = list(sources)
+    now = timezone.now()
+    keep = {}
+    refreshed = failed = 0
+    new_items = []
+    pool = ThreadPoolExecutor(FEED_WORKERS)
+    try:
+        futures = {
+            pool.submit(_fetch_feed, s.url, s.http_etag, s.http_last_modified, s.http_body_hash,
+                        user_agent=fetch.USER_AGENT): s
+            for s in sources
+        }
+        try:
+            timeout = None if deadline is None else max(0.0, deadline - time.monotonic())
+            for f in as_completed(futures, timeout=timeout):
+                s = futures[f]
+                try:
+                    result = f.result()
+                    if s.user_id not in keep:
+                        keep[s.user_id] = settings_for(s.user).news_keep_days
+                    new_items += _apply(s, result, now, keep[s.user_id])
+                    refreshed += 1
+                except Exception as e:
+                    if not isinstance(e, FeedFetchError):
+                        log.exception('feed source %s', s.pk)
+                    s.last_error = _error_text(e)
+                    s.last_error_at = now
+                    s.error_count += 1
+                    s.save(update_fields=['last_error', 'last_error_at', 'error_count'])
+                    failed += 1
+        except TimeoutError:
+            pass
+    finally:
+        # Past the deadline: don't wait for the slow ones (they stay as they are).
+        pool.shutdown(wait=False, cancel_futures=True)
+
+    for item in new_items:
+        item.source = next(s for s in sources if s.pk == item.source_id)
+    _articles(new_items, deadline)
+
+    by_user = {}
+    for item in new_items:
+        if item.kind == Source.NEWS:
+            by_user.setdefault(item.user_id, []).append(item)
+    for user_id, items in by_user.items():
+        c = Clusterer(items[0].user, now)
+        for item in sorted(items, key=lambda i: i.published):
+            c.add(item)
+    for user_id in {s.user_id for s in sources}:
+        expire(next(s.user for s in sources if s.user_id == user_id), now)
+    return refreshed, failed, len(new_items)

feeds/tests.py

@@ -0,0 +1,179 @@
+import json
+from datetime import timedelta
+from unittest import mock
+
+import feedparser
+from django.contrib.auth.models import User
+from django.test import TestCase
+from django.utils import timezone
+
+from podcasts.views import FeedFetch
+
+from . import cluster, fetch, refresh
+from .models import FeedSettings, Item, Source, Story
+
+
+def rss(items, title='Testfeed', link='https://example.org/'):
+    body = ''.join(
+        f'<item><guid>{i["guid"]}</guid><title>{i["title"]}</title><link>{i.get("link", "https://example.org/" + i["guid"])}</link>'
+        f'<pubDate>{i.get("date", timezone.now().strftime("%a, %d %b %Y %H:%M:%S +0000"))}</pubDate>'
+        f'<description><![CDATA[{i.get("text", "")}]]></description></item>'
+        for i in items)
+    return f'<?xml version="1.0"?><rss version="2.0"><channel><title>{title}</title><link>{link}</link>{body}</channel></rss>'.encode()
+
+
+def fetched(content, url='https://example.org/feed'):
+    return FeedFetch(200, feedparser.parse(content), 'etag', '', url, url, 'hash')
+
+
+LONG = '<p>' + 'Ein langer Absatz mit sehr viel Text. ' * 40 + '</p>'
+
+
+class FetchTest(TestCase):
+    def test_relative_links_become_absolute_and_scripts_go(self):
+        out = fetch.clean_html('<p>Hi <a href="/a">x</a><img src="i.jpg"><script>bad()</script>'
+                               '<img data-src="/real.jpg" src="placeholder.gif"></p>', 'https://ex.org/post/1')
+        self.assertIn('href="https://ex.org/a"', out)
+        self.assertIn('src="https://ex.org/post/i.jpg"', out)
+        self.assertIn('src="https://ex.org/real.jpg"', out)
+        self.assertNotIn('placeholder', out)
+        self.assertNotIn('bad()', out)
+
+    def test_parse_entry(self):
+        e = feedparser.parse(rss([{'guid': 'g1', 'title': 'A &amp; B', 'text': '<p>Kurz <img src="/p.jpg"></p>'}])).entries[0]
+        f = fetch.parse_entry(e, 'https://example.org/feed')
+        self.assertEqual(f['title'], 'A & B')
+        self.assertEqual(f['image_url'], 'https://example.org/p.jpg')
+        self.assertTrue(fetch.is_teaser(f['content']))
+
+    def test_drop_repeated_headline(self):
+        self.assertEqual(fetch._drop_title('<img src="x"><h1>Titel</h1><p>Text</p>'), '<img src="x"><p>Text</p>')
+
+    def test_discover(self):
+        feeds, icon = fetch.discover('<link rel="alternate" type="application/rss+xml" href="/feed" title="F">'
+                                     '<link rel="icon" href="/fav.png">', 'https://ex.org/')
+        self.assertEqual(feeds, [('https://ex.org/feed', 'F')])
+        self.assertEqual(icon, 'https://ex.org/fav.png')
+
+
+class ClusterTest(TestCase):
+    def setUp(self):
+        self.u = User.objects.create_user('u', password='pw12345678')
+        self.a = Source.objects.create(user=self.u, url='https://a/', title='A')
+        self.b = Source.objects.create(user=self.u, url='https://b/', title='B')
+        self.c = Source.objects.create(user=self.u, url='https://c/', title='C')
+
+    def item(self, source, title, summary=''):
+        return Item.objects.create(source=source, user=self.u, kind='news', guid_hash=title[:60],
+                                   title=title, summary=summary, published=timezone.now())
+
+    def test_same_event_bundled_other_apart(self):
+        c = cluster.Clusterer(self.u, timezone.now())
+        s1 = c.add(self.item(self.a, 'Oktoberfest endet mit neuem Besucherrekord in München'))
+        s2 = c.add(self.item(self.b, 'Rekord-Wiesn: 7,4 Millionen Besucher auf dem Oktoberfest in München'))
+        s3 = c.add(self.item(self.c, 'Bahn führt Familienticket dauerhaft ein'))
+        self.assertEqual(s1, s2)
+        self.assertNotEqual(s1, s3)
+        s1.refresh_from_db()
+        self.assertEqual((s1.size, s1.sources), (2, 2))
+
+    def test_same_source_needs_to_be_very_close(self):
+        c = cluster.Clusterer(self.u, timezone.now())
+        s1 = c.add(self.item(self.a, 'Nations League: Pressekonferenz vor Irland gegen Israel abgebrochen'))
+        s2 = c.add(self.item(self.a, 'Nations League: England schlägt Kroatien deutlich'))
+        self.assertNotEqual(s1, s2)
+
+
+class ApiTest(TestCase):
+    def setUp(self):
+        self.u = User.objects.create_user('tester', password='pw12345678')
+        self.client.force_login(self.u)
+
+    def post(self, url, data=None):
+        return self.client.post(url, json.dumps(data or {}), content_type='application/json')
+
+    def add(self, kind, feed, url='https://example.org/feed'):
+        with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed, url)), \
+                mock.patch('feeds.fetch.fetch_article', return_value=(LONG, 'https://example.org/img.jpg')):
+            r = self.post('/feeds/sources/add/', {'url': url, 'kind': kind, 'category': 'Politik'})
+        self.assertEqual(r.status_code, 200)
+        return r.json()['source']
+
+    def test_requires_login(self):
+        self.client.logout()
+        self.assertEqual(self.client.get('/feeds/state/').status_code, 401)
+
+    def test_add_news_fetches_articles_and_bundles(self):
+        s = self.add('news', rss([{'guid': 'a', 'title': 'Merz besucht Kiew und sagt Hilfe zu', 'text': 'Kurz.'},
+                                   {'guid': 'b', 'title': 'Bahn Familienticket dauerhaft', 'text': 'Kurz.'}]))
+        self.assertEqual(s['title'], 'Testfeed')
+        self.assertEqual(s['unread'], 2)
+        items = Item.objects.filter(user=self.u)
+        self.assertTrue(all(i.content_full for i in items))
+        self.assertEqual(Story.objects.filter(user=self.u).count(), 2)
+        news = self.client.get('/feeds/news/').json()
+        self.assertEqual(len(news['stories']), 2)
+        self.assertEqual(news['stories'][0]['category'], 'Politik')
+        b = self.client.get('/feeds/briefing/').json()
+        self.assertEqual(b['total'], 2)
+        self.assertEqual(b['sections'][0]['category'], 'Politik')
+        self.post('/feeds/briefing/seen/')
+        self.assertFalse(Item.objects.filter(user=self.u, read_at=None).exists())
+        self.assertEqual(self.client.get('/feeds/briefing/').json()['total'], 0)
+
+    def test_blog_keeps_full_feed_text_and_lists(self):
+        self.add('blog', rss([{'guid': 'p1', 'title': 'Ein Artikel', 'text': LONG}]))
+        item = Item.objects.get(user=self.u)
+        self.assertFalse(item.content_full)  # the feed had the whole text
+        self.assertIsNone(item.story)
+        blogs = self.client.get('/feeds/blogs/?unread=1').json()
+        self.assertEqual([i['title'] for i in blogs['items']], ['Ein Artikel'])
+        detail = self.client.get(f'/feeds/items/{item.id}/').json()['item']
+        self.assertIn('langer Absatz', detail['content'])
+        self.post('/feeds/items/read/', {'ids': [item.id]})
+        self.assertEqual(self.client.get('/feeds/blogs/?unread=1').json()['items'], [])
+        self.post(f'/feeds/items/{item.id}/star/', {'starred': True})
+        self.assertEqual(len(self.client.get('/feeds/starred/').json()['items']), 1)
+
+    def test_old_news_expires_and_is_not_taken_in_again(self):
+        old = (timezone.now() - timedelta(days=3)).strftime('%a, %d %b %Y %H:%M:%S +0000')
+        feed = rss([{'guid': 'new', 'title': 'Heute passiert', 'text': LONG},
+                    {'guid': 'old', 'title': 'Vorgestern passiert', 'text': LONG, 'date': old}])
+        src = self.add('news', feed)
+        self.assertEqual(Item.objects.filter(user=self.u).count(), 1)  # too old to take in
+        Item.objects.filter(user=self.u).update(published=timezone.now() - timedelta(days=3))
+        refresh.expire(self.u)
+        self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0)
+        self.assertEqual(Story.objects.filter(user=self.u).count(), 0)
+        source = Source.objects.get(pk=src['id'])
+        with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed)):
+            refresh.refresh([source])
+        self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0)
+
+    def test_switching_kind_moves_items(self):
+        src = self.add('news', rss([{'guid': 'a', 'title': 'Etwas', 'text': LONG}]))
+        self.post(f'/feeds/sources/{src["id"]}/', {'kind': 'blog'})
+        item = Item.objects.get(user=self.u)
+        self.assertEqual((item.kind, item.story), ('blog', None))
+        self.assertEqual(Story.objects.filter(user=self.u).count(), 0)
+
+    def test_other_users_items_invisible(self):
+        self.add('blog', rss([{'guid': 'p1', 'title': 'Privat', 'text': LONG}]))
+        item = Item.objects.get(user=self.u)
+        other = User.objects.create_user('o', password='pw12345678')
+        self.client.force_login(other)
+        self.assertEqual(self.client.get(f'/feeds/items/{item.id}/').status_code, 404)
+        self.assertEqual(self.client.get('/feeds/blogs/').json()['items'], [])
+        self.post('/feeds/items/read/', {'ids': [item.id]})
+        item.refresh_from_db()
+        self.assertIsNone(item.read_at)
+
+    def test_opml_export(self):
+        self.add('blog', rss([]), url='https://blog.example/feed')
+        r = self.client.get('/feeds/export.opml')
+        self.assertIn(b'xmlUrl="https://blog.example/feed"', r.content)
+        self.assertIn(b'text="Blogs"', r.content)
+
+    def test_settings(self):
+        self.post('/feeds/settings/', {'news_keep_days': 5})
+        self.assertEqual(FeedSettings.objects.get(user=self.u).news_keep_days, 5)

feeds/urls.py

@@ -0,0 +1,24 @@
+from django.urls import path
+
+from . import views
+
+urlpatterns = [
+    path('state/', views.state, name='feeds_state'),
+    path('discover/', views.discover, name='feeds_discover'),
+    path('sources/add/', views.add_source, name='feeds_add_source'),
+    path('sources/bulk/', views.add_sources, name='feeds_add_sources'),
+    path('sources/<int:pk>/', views.edit_source, name='feeds_edit_source'),
+    path('sources/<int:pk>/delete/', views.delete_source, name='feeds_delete_source'),
+    path('export.opml', views.export_opml, name='feeds_export_opml'),
+    path('refresh/', views.refresh_now, name='feeds_refresh'),
+    path('briefing/', views.briefing, name='feeds_briefing'),
+    path('briefing/seen/', views.briefing_seen, name='feeds_briefing_seen'),
+    path('news/', views.news, name='feeds_news'),
+    path('stories/<int:pk>/', views.story, name='feeds_story'),
+    path('blogs/', views.blogs, name='feeds_blogs'),
+    path('starred/', views.starred, name='feeds_starred'),
+    path('items/read/', views.mark_read, name='feeds_mark_read'),
+    path('items/<int:pk>/', views.item, name='feeds_item'),
+    path('items/<int:pk>/star/', views.star, name='feeds_star'),
+    path('settings/', views.settings, name='feeds_settings'),
+]

feeds/views.py

@@ -0,0 +1,445 @@
+"""The feed reader's JSON API (session or Bearer token, like the podcast API).
+
+Sources:   GET  /feeds/state/                 sources, unread counts, settings
+           POST /feeds/discover/              {url} -> the feeds a page or URL offers
+           POST /feeds/sources/add/           {url, kind, category, title?}
+           POST /feeds/sources/bulk/          {sources: [{url, title, kind, category}]} (OPML import)
+           POST /feeds/sources/<id>/          {title, kind, category, fulltext}
+           POST /feeds/sources/<id>/delete/
+           GET  /feeds/export.opml
+           POST /feeds/refresh/               refresh the account's sources now
+News:      GET  /feeds/briefing/              what's new since the briefing was last seen
+           POST /feeds/briefing/seen/         seen: everything up to now counts as read
+           GET  /feeds/news/?before=&category=  stories, newest first
+           GET  /feeds/stories/<id>/          one story with all its items
+Blogs:     GET  /feeds/blogs/?source=&unread=1&before=
+Items:     GET  /feeds/items/<id>/            the article (fetches the page if the feed had a teaser)
+           POST /feeds/items/read/            {ids, read}
+           POST /feeds/items/<id>/star/       {starred}
+           GET  /feeds/starred/
+Settings:  POST /feeds/settings/              {news_keep_days}
+"""
+
+import json
+import time
+from datetime import timedelta
+from xml.sax.saxutils import quoteattr
+
+import feedparser
+import requests
+from django.db.models import Count
+from django.http import HttpResponse, JsonResponse
+from django.utils import timezone
+from django.utils.dateparse import parse_datetime
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import require_http_methods
+
+from . import fetch, refresh
+from .models import Item, Source, Story
+
+PAGE = 40
+BRIEFING_TOP = 8
+BRIEFING_PER_CATEGORY = 5
+REFRESH_WAIT = 40  # seconds an explicit refresh may take
+CATEGORY_NONE = 'Allgemein'
+
+
+def _auth(request):
+    if not request.user.is_authenticated:
+        return JsonResponse({'error': 'authentication required'}, status=401)
+    return None
+
+
+def _body(request):
+    try:
+        return json.loads(request.body or b'{}')
+    except (ValueError, UnicodeDecodeError):
+        return None
+
+
+def _iso(dt):
+    return dt.isoformat() if dt else None
+
+
+def _source_data(s, unread=0):
+    return {
+        'id': s.id, 'url': s.url, 'title': s.title, 'site_url': s.site_url, 'icon_url': s.icon_url,
+        'kind': s.kind, 'category': s.category, 'fulltext': s.fulltext, 'unread': unread,
+        'last_refreshed_at': _iso(s.last_refreshed_at), 'last_error': s.last_error, 'error_count': s.error_count,
+    }
+
+
+def _item_data(i, content=False):
+    d = {
+        'id': i.id, 'source': i.source_id, 'source_title': i.source.title, 'kind': i.kind,
+        'url': i.url, 'title': i.title, 'author': i.author, 'published': _iso(i.published),
+        'summary': i.summary, 'image_url': i.image_url, 'read': i.read_at is not None,
+        'starred': i.starred, 'story': i.story_id,
+    }
+    if content:
+        d['content'] = i.content
+        d['content_full'] = i.content_full
+    return d
+
+
+def _story_data(s, items):
+    items = sorted(items, key=lambda i: i.published)
+    lead = next((i for i in items if i.id == s.lead_id), items[0] if items else None)
+    return {
+        'id': s.id, 'first_at': _iso(s.first_at), 'last_at': _iso(s.last_at),
+        'size': len(items), 'sources': len({i.source_id for i in items}),
+        'category': (lead.source.category if lead else '') or CATEGORY_NONE,
+        'unread': sum(1 for i in items if i.read_at is None),
+        'lead': _item_data(lead) if lead else None,
+        'items': [_item_data(i) for i in items],
+    }
+
+
+def _stories(user, qs):
+    """Story dicts for a Story queryset (items fetched in one go)."""
+    stories = list(qs)
+    items = {}
+    for i in Item.objects.filter(story__in=stories, gone=False).select_related('source'):
+        items.setdefault(i.story_id, []).append(i)
+    return [_story_data(s, items[s.id]) for s in stories if s.id in items]
+
+
+# ---- sources
+
+@csrf_exempt
+@require_http_methods(['GET'])
+def state(request):
+    if (err := _auth(request)):
+        return err
+    unread = dict(Item.objects.filter(user=request.user, gone=False, read_at=None)
+                  .values_list('source').annotate(n=Count('id')))
+    sources = list(Source.objects.filter(user=request.user))
+    s = refresh.settings_for(request.user)
+    since = s.news_seen_at or (timezone.now() - timedelta(days=1))
+    new_news = Story.objects.filter(user=request.user, last_at__gt=since,
+                                    items__gone=False, items__read_at=None).distinct().count()
+    return JsonResponse({
+        'sources': [_source_data(x, unread.get(x.id, 0)) for x in sources],
+        'categories': sorted({x.category for x in sources if x.kind == Source.NEWS and x.category}),
+        'news_new': new_news,
+        'blogs_unread': sum(n for sid, n in unread.items() if any(x.id == sid and x.kind == Source.BLOG for x in sources)),
+        'settings': {'news_keep_days': s.news_keep_days, 'news_seen_at': _iso(s.news_seen_at)},
+    })
+
+
+def _probe(url):
+    """(feed url, title, entries) if `url` is a feed, else ([candidate feeds], icon)."""
+    r = requests.get(url, timeout=fetch.PAGE_TIMEOUT, headers={'User-Agent': fetch.USER_AGENT})
+    r.raise_for_status()
+    ctype = r.headers.get('Content-Type', '')
+    if 'html' in ctype and b'<rss' not in r.content[:2000] and b'<feed' not in r.content[:2000]:
+        return None, fetch.discover(r.text, r.url)
+    parsed = feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
+    if parsed.get('bozo') and not parsed.entries and not parsed.feed.get('title'):
+        return None, ([], '')
+    return (r.url, fetch.plain(parsed.feed.get('title') or '') or r.url, parsed), None
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def discover(request):
+    """The feeds behind a URL: itself if it's a feed, else the ones the page links to."""
+    if (err := _auth(request)):
+        return err
+    body = _body(request) or {}
+    url = (body.get('url') or '').strip()
+    if not url:
+        return JsonResponse({'error': 'Adresse fehlt'}, status=400)
+    if not url.startswith(('http://', 'https://')):
+        url = 'https://' + url
+    try:
+        feed, page = _probe(url)
+        if feed is None:
+            candidates, _icon = page
+            found = []
+            for u, title in candidates[:6]:
+                try:
+                    f, _ = _probe(u)
+                except requests.RequestException:
+                    continue
+                if f:
+                    found.append(f)
+            if not found:
+                return JsonResponse({'error': 'Dort ist kein Feed zu finden.'}, status=404)
+        else:
+            found = [feed]
+    except requests.RequestException:
+        return JsonResponse({'error': 'Die Adresse ist nicht erreichbar.'}, status=502)
+    out = []
+    for u, title, parsed in found:
+        entries = parsed.entries[:5]
+        teaser = sum(1 for e in entries if fetch.is_teaser(fetch.parse_entry(e, u)['content']))
+        out.append({
+            'url': u, 'title': title,
+            'subscribed': Source.objects.filter(user=request.user, url=u).exists(),
+            'samples': [fetch.plain(e.get('title') or '') for e in entries],
+            # Many entries a day suggests news, few long ones a blog.
+            'suggested_kind': Source.NEWS if len(parsed.entries) >= 25 or teaser >= 3 else Source.BLOG,
+        })
+    return JsonResponse({'feeds': out})
+
+
+def _kind(v):
+    return v if v in (Source.NEWS, Source.BLOG) else None
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def add_source(request):
+    if (err := _auth(request)):
+        return err
+    body = _body(request) or {}
+    url = (body.get('url') or '').strip()
+    kind = _kind(body.get('kind'))
+    if not url.startswith(('http://', 'https://')) or not kind:
+        return JsonResponse({'error': 'Adresse und Art (Nachrichten/Blog) nötig'}, status=400)
+    source, created = Source.objects.get_or_create(
+        user=request.user, url=url[:1000],
+        defaults={'title': (body.get('title') or url)[:300], 'kind': kind,
+                  'category': (body.get('category') or '').strip()[:100]})
+    if created:
+        refresh.refresh([source], deadline=time.monotonic() + 25)
+        source.refresh_from_db()
+    return JsonResponse({'source': _source_data(source, source.items.filter(read_at=None, gone=False).count()),
+                         'created': created})
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def add_sources(request):
+    """Many at once (an OPML file the page read); refreshed by the next run."""
+    if (err := _auth(request)):
+        return err
+    body = _body(request) or {}
+    added = 0
+    for x in body.get('sources') or []:
+        url = (x.get('url') or '').strip()
+        kind = _kind(x.get('kind'))
+        if not url.startswith(('http://', 'https://')) or not kind:
+            continue
+        _, created = Source.objects.get_or_create(
+            user=request.user, url=url[:1000],
+            defaults={'title': (x.get('title') or url)[:300], 'kind': kind,
+                      'category': (x.get('category') or '').strip()[:100]})
+        added += created
+    sources = Source.objects.filter(user=request.user, last_refreshed_at=None)
+    refresh.refresh(sources, deadline=time.monotonic() + REFRESH_WAIT)
+    return JsonResponse({'added': added})
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def edit_source(request, pk):
+    if (err := _auth(request)):
+        return err
+    source = Source.objects.filter(user=request.user, pk=pk).first()
+    body = _body(request)
+    if source is None or body is None:
+        return JsonResponse({'error': 'not found'}, status=404)
+    if 'title' in body and (body['title'] or '').strip():
+        source.title = body['title'].strip()[:300]
+    if 'category' in body:
+        source.category = (body['category'] or '').strip()[:100]
+    if body.get('fulltext') in dict(Source.FULLTEXTS):
+        source.fulltext = body['fulltext']
+    new_kind = _kind(body.get('kind'))
+    if new_kind and new_kind != source.kind:
+        source.kind = new_kind
+        # Its items follow; as blog articles they leave their stories.
+        source.items.update(kind=new_kind, **({'story': None} if new_kind == Source.BLOG else {}))
+        Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
+    source.save()
+    return JsonResponse({'source': _source_data(source)})
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def delete_source(request, pk):
+    if (err := _auth(request)):
+        return err
+    Source.objects.filter(user=request.user, pk=pk).delete()
+    Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
+    return JsonResponse({'ok': True})
+
+
+@csrf_exempt
+@require_http_methods(['GET'])
+def export_opml(request):
+    if (err := _auth(request)):
+        return err
+    groups = {}
+    for s in Source.objects.filter(user=request.user).order_by('kind', 'category', 'title'):
+        label = ('Nachrichten' + (f' – {s.category}' if s.category else '')) if s.kind == Source.NEWS else 'Blogs'
+        groups.setdefault(label, []).append(s)
+    lines = ['<?xml version="1.0" encoding="UTF-8"?>', '<opml version="2.0">',
+             '<head><title>diora Feeds</title></head>', '<body>']
+    for label, sources in groups.items():
+        lines.append(f'<outline text={quoteattr(label)}>')
+        for s in sources:
+            lines.append(f'<outline type="rss" text={quoteattr(s.title)} xmlUrl={quoteattr(s.url)}'
+                         + (f' htmlUrl={quoteattr(s.site_url)}' if s.site_url else '') + '/>')
+        lines.append('</outline>')
+    lines += ['</body>', '</opml>']
+    resp = HttpResponse('\n'.join(lines), content_type='text/x-opml; charset=utf-8')
+    resp['Content-Disposition'] = 'attachment; filename="diora-feeds.opml"'
+    return resp
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def refresh_now(request):
+    if (err := _auth(request)):
+        return err
+    refreshed, failed, new = refresh.refresh(Source.objects.filter(user=request.user),
+                                             deadline=time.monotonic() + REFRESH_WAIT)
+    return JsonResponse({'refreshed': refreshed, 'failed': failed, 'new': new})
+
+
+# ---- news
+
+@csrf_exempt
+@require_http_methods(['GET'])
+def briefing(request):
+    """Since the briefing was last seen (else the last day): the top stories
+    (reported by several sources), then the rest by section, newest first."""
+    if (err := _auth(request)):
+        return err
+    s = refresh.settings_for(request.user)
+    since = s.news_seen_at or (timezone.now() - timedelta(days=1))
+    qs = Story.objects.filter(user=request.user, last_at__gt=since,
+                              items__gone=False, items__read_at=None).distinct()
+    stories = _stories(request.user, qs)
+    stories.sort(key=lambda x: (-x['sources'], -x['size'], x['last_at']), reverse=False)
+    top = [x for x in stories if x['sources'] >= 2][:BRIEFING_TOP]
+    top_ids = {x['id'] for x in top}
+    sections = {}
+    for x in sorted(stories, key=lambda x: x['last_at'], reverse=True):
+        if x['id'] not in top_ids:
+            sections.setdefault(x['category'], []).append(x)
+    return JsonResponse({
+        'since': _iso(since), 'total': len(stories), 'top': top,
+        'sections': [{'category': c, 'count': len(v), 'stories': v[:BRIEFING_PER_CATEGORY]}
+                     for c, v in sorted(sections.items(), key=lambda kv: (kv[0] == CATEGORY_NONE, kv[0]))],
+    })
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def briefing_seen(request):
+    if (err := _auth(request)):
+        return err
+    now = timezone.now()
+    s = refresh.settings_for(request.user)
+    Item.objects.filter(user=request.user, kind=Source.NEWS, read_at=None, fetched_at__lte=now).update(read_at=now)
+    s.news_seen_at = now
+    s.save(update_fields=['news_seen_at'])
+    return JsonResponse({'news_seen_at': _iso(now)})
+
+
+@csrf_exempt
+@require_http_methods(['GET'])
+def news(request):
+    if (err := _auth(request)):
+        return err
+    qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at')
+    before = parse_datetime(request.GET.get('before') or '')
+    if before:
+        qs = qs.filter(last_at__lt=before)
+    category = request.GET.get('category')
+    if category:
+        qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category)
+    stories = _stories(request.user, qs[:PAGE])
+    return JsonResponse({'stories': stories, 'more': len(stories) == PAGE})
+
+
+@csrf_exempt
+@require_http_methods(['GET'])
+def story(request, pk):
+    if (err := _auth(request)):
+        return err
+    found = _stories(request.user, Story.objects.filter(user=request.user, pk=pk))
+    if not found:
+        return JsonResponse({'error': 'not found'}, status=404)
+    return JsonResponse({'story': found[0]})
+
+
+# ---- blogs, items
+
+@csrf_exempt
+@require_http_methods(['GET'])
+def blogs(request):
+    if (err := _auth(request)):
+        return err
+    qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source')
+    if request.GET.get('source', '').isdigit():
+        qs = qs.filter(source_id=int(request.GET['source']))
+    if request.GET.get('unread') == '1':
+        qs = qs.filter(read_at=None)
+    before = parse_datetime(request.GET.get('before') or '')
+    if before:
+        qs = qs.filter(published__lt=before)
+    items = list(qs.order_by('-published')[:PAGE])
+    return JsonResponse({'items': [_item_data(i) for i in items], 'more': len(items) == PAGE})
+
+
+@csrf_exempt
+@require_http_methods(['GET'])
+def starred(request):
+    if (err := _auth(request)):
+        return err
+    items = Item.objects.filter(user=request.user, starred=True).select_related('source').order_by('-published')
+    return JsonResponse({'items': [_item_data(i) for i in items[:200]]})
+
+
+@csrf_exempt
+@require_http_methods(['GET'])
+def item(request, pk):
+    if (err := _auth(request)):
+        return err
+    i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source').first()
+    if i is None:
+        return JsonResponse({'error': 'not found'}, status=404)
+    if refresh.needs_article(i, i.source):
+        refresh.load_article(i)
+    return JsonResponse({'item': _item_data(i, content=True)})
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def mark_read(request):
+    if (err := _auth(request)):
+        return err
+    body = _body(request) or {}
+    ids = [int(x) for x in body.get('ids') or [] if str(x).isdigit()]
+    qs = Item.objects.filter(user=request.user, id__in=ids)
+    n = qs.update(read_at=timezone.now() if body.get('read', True) else None)
+    return JsonResponse({'updated': n})
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def star(request, pk):
+    if (err := _auth(request)):
+        return err
+    body = _body(request) or {}
+    n = Item.objects.filter(user=request.user, pk=pk).update(starred=bool(body.get('starred', True)))
+    return JsonResponse({'updated': n})
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def settings(request):
+    if (err := _auth(request)):
+        return err
+    body = _body(request) or {}
+    s = refresh.settings_for(request.user)
+    days = body.get('news_keep_days')
+    if isinstance(days, int) and 1 <= days <= 30:
+        s.news_keep_days = days
+        s.save(update_fields=['news_keep_days'])
+    return JsonResponse({'news_keep_days': s.news_keep_days})

podcasts/views.py

@@ -322,13 +322,13 @@ def _body_hash(content):
 PERMANENT_REDIRECTS = (301, 308)
 
 
-def _fetch_feed(url, etag='', last_modified='', body_hash=''):
+def _fetch_feed(url, etag='', last_modified='', body_hash='', user_agent='diora podcast refresh'):
     """Download and parse a feed (with a timeout, unlike feedparser's own
     fetch). With etag/last_modified it is a conditional GET; with body_hash a
     body like the last one counts as unchanged too (a 304), unparsed —
     parsing is what takes the time (seconds for a big feed). Returns a
     FeedFetch; raises FeedFetchError when there is nothing usable."""
-    headers = {'User-Agent': 'diora podcast refresh'}
+    headers = {'User-Agent': user_agent}
     if etag:
         headers['If-None-Match'] = etag
     if last_modified:

requirements.txt

@@ -19,3 +19,5 @@ packaging==26.3
 piper-tts==1.7.0
 # ffmpeg (a static binary in the wheel) finds the pauses for skipping silence in the browser.
 imageio-ffmpeg==0.6.0
+# The feed reader takes the article out of a news page (main text without menus, ads, comments).
+trafilatura==2.3.0