diff --git a/diora/settings.py b/diora/settings.py index c4d3f6a..5bd586c 100644 --- a/diora/settings.py +++ b/diora/settings.py @@ -31,6 +31,7 @@ INSTALLED_APPS = [ 'books', 'gpodder', 'tts', + 'feeds', ] EBOOK_MAX_BYTES = 50 * 1024 * 1024 # 50 MB diff --git a/diora/urls.py b/diora/urls.py index 283cf2c..886ee26 100644 --- a/diora/urls.py +++ b/diora/urls.py @@ -11,6 +11,7 @@ urlpatterns = [ path('accounts/', include('accounts.urls')), path('podcasts/', include('podcasts.urls')), path('books/', include('books.urls')), + path('feeds/', include('feeds.urls')), path('tts/', include('tts.urls')), path('api/2/', include('gpodder.urls')), path('api/sync/', sync_snapshot, name='api_sync'), diff --git a/feeds/__init__.py b/feeds/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/feeds/apps.py b/feeds/apps.py new file mode 100644 index 0000000..29a474b --- /dev/null +++ b/feeds/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class FeedsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'feeds' diff --git a/feeds/cluster.py b/feeds/cluster.py new file mode 100644 index 0000000..b1dd7a8 --- /dev/null +++ b/feeds/cluster.py @@ -0,0 +1,149 @@ +"""Bundling news items about the same event into stories. + +Each item is described by the words of its title (counted twice) and summary, +without stop words and with a crude German/English suffix cut. A new item joins +the most similar story of the last WINDOW hours (cosine of tf-idf vectors, +document frequencies over those stories) if that similarity reaches THRESHOLD +and they share at least MIN_SHARED words; otherwise it starts a story of its own. +A story many sources report on is a top story. +""" + +import math +import re +from collections import Counter +from datetime import timedelta + +from .models import Item, Story + +WINDOW = timedelta(hours=36) +THRESHOLD = 0.25 +MIN_SHARED = 2 + +STOP = set(''' +aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher +da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese +diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für +gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer +in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen +nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind +so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter +weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier +heute gestern morgen laut sagt sagte erst mal neu alt video live ticker liveblog news meldung bericht +the and for with from that this have has are was were will would been into about after over their they +his her its our you your not but all can new says said more than when what who how why off out +tot tote toten getötet sterben stirbt gestorben menschen mensch leute millionen milliarden prozent euro +deutschland deutsche deutschen land länder stadt welt woche wochen monat monate tag tage tagen uhr +frau frauen mann männer kinder erste ersten zweite neuer neues großen große groß kleine wegen gibt geben +'''.split()) + +_WORD = re.compile(r'[a-zäöüß][a-zäöüß0-9\-]+') +_SUFFIXES = ('ungen', 'ung', 'en', 'er', 'es', 'e', 's', 'n') + + +def _stem(w): + if len(w) > 5: + for s in _SUFFIXES: + if w.endswith(s) and len(w) - len(s) >= 4: + return w[:-len(s)] + return w + + +def words(text): + out = [] + for w in _WORD.findall((text or '').lower()): + w = w.strip('-') + if len(w) < 3 or w in STOP: + continue + out.append(_stem(w)) + return out + + +def item_terms(title, summary): + return Counter(words(title) * 2 + words(summary[:300])) + + +def _cosine(a, b, idf): + dot = sum(a[w] * b[w] * idf.get(w, 1.0) ** 2 for w in a.keys() & b.keys()) + if not dot: + return 0.0 + na = math.sqrt(sum((n * idf.get(w, 1.0)) ** 2 for w, n in a.items())) + nb = math.sqrt(sum((n * idf.get(w, 1.0)) ** 2 for w, n in b.items())) + return dot / (na * nb) + + +class Clusterer: + """Assigns one user's new news items to stories (load once, add items, done). + + An item is compared with every item already in a story: the story's score + is the mean of the best match and the average match, so one stray shared + word can't pull an item in, and a story doesn't grow into a magnet the way + a pooled word list would. Items of the same source join only when very + close (a source rarely reports one event twice, but often two similar ones).""" + + SAME_SOURCE = 0.5 + + def __init__(self, user, now): + self.user = user + self.stories = list(Story.objects.filter(user=user, last_at__gte=now - WINDOW)) + self.members = {s.pk: [] for s in self.stories} # story id -> [(source id, terms)] + for i in Item.objects.filter(story__in=self.stories, gone=False).only('story_id', 'source_id', 'title', 'summary'): + self.members[i.story_id].append((i.source_id, item_terms(i.title, i.summary))) + self._df = Counter() + for ms in self.members.values(): + for _, t in ms: + self._df.update(t.keys()) + self._n = sum(len(ms) for ms in self.members.values()) + + def _idf(self): + n = self._n + 1 + return {w: math.log((n + 1) / (d + 1)) + 1 for w, d in self._df.items()} + + def score(self, t, source_id, members, idf): + sims = [] + for sid, mt in members: + if len(t.keys() & mt.keys()) < MIN_SHARED: + sims.append(0.0) + continue + sim = _cosine(t, mt, idf) + if sid == source_id and sim < self.SAME_SOURCE: + sim = 0.0 + sims.append(sim) + if not sims or max(sims) == 0: + return 0.0 + return (max(sims) + sum(sims) / len(sims)) / 2 + + def add(self, item): + t = item_terms(item.title, item.summary) + best, best_sim = None, 0.0 + if t: + idf = self._idf() + for s in self.stories: + if abs((s.last_at - item.published).total_seconds()) > WINDOW.total_seconds(): + continue + sim = self.score(t, item.source_id, self.members[s.pk], idf) + if sim > best_sim: + best, best_sim = s, sim + if best is not None and best_sim >= THRESHOLD: + self._join(best, item) + else: + best = Story.objects.create(user=self.user, lead=item, first_at=item.published, + last_at=item.published, size=1, sources=1) + self.stories.append(best) + self.members[best.pk] = [] + item.story = best + item.save(update_fields=['story']) + self.members[best.pk].append((item.source_id, t)) + self._df.update(t.keys()) + self._n += 1 + return best + + def _join(self, s, item): + item.story = s + item.save(update_fields=['story']) + s.size = Item.objects.filter(story=s).count() + s.sources = Item.objects.filter(story=s).values('source').distinct().count() + s.first_at = min(s.first_at, item.published) + s.last_at = max(s.last_at, item.published) + if s.lead is None or (not s.lead.image_url and item.image_url): + s.lead = item + s.save(update_fields=['size', 'sources', 'first_at', 'last_at', 'lead']) diff --git a/feeds/fetch.py b/feeds/fetch.py new file mode 100644 index 0000000..b535224 --- /dev/null +++ b/feeds/fetch.py @@ -0,0 +1,231 @@ +"""Reading feeds: entries into items, the whole article from its page, finding a +site's feed. Downloading the feed itself is podcasts.views._fetch_feed (ETag, +body hash, redirects), shared with the podcast refresh.""" + +import calendar +import hashlib +import html +import logging +import re +from datetime import datetime, timezone as dt_timezone +from html.parser import HTMLParser +from urllib.parse import urljoin, urlparse + +import requests +from django.utils import timezone + +from podcasts.shownotes import _Sanitizer, html_to_text + +log = logging.getLogger(__name__) + +USER_AGENT = 'Mozilla/5.0 (compatible; diora feed reader; +https://diora.creamfresh.xyz)' +PAGE_TIMEOUT = 15 +# Shorter text than this in the feed counts as a teaser: the article is fetched ("auto"). +TEASER_CHARS = 700 +SUMMARY_CHARS = 400 + + +class _AbsoluteSanitizer(_Sanitizer): + """The podcast sanitizer, with links and images made absolute first + (article HTML is full of relative ones, which the allowlist would drop).""" + + def __init__(self, base): + super().__init__() + self.base = base + + def _start(self, tag, attrs, closed): + if tag == 'img': + # Lazy-loaded images keep the real address in data-src, src is a placeholder. + a = dict(attrs) + real = a.get('data-src') or a.get('data-original') + if real: + attrs = [(n, v) for n, v in attrs if n not in ('src', 'data-src', 'data-original')] + [('src', real)] + fixed = [] + for name, value in attrs: + if name in ('href', 'src') and value and not value.strip().startswith(('data:', 'javascript:', '#')): + value = urljoin(self.base, value.strip()) + fixed.append((name, value)) + super()._start(tag, fixed, closed) + + +def clean_html(text, base=''): + if not text: + return '' + s = _AbsoluteSanitizer(base) + s.feed(text) + return s.result().strip() + + +def plain(text): + return re.sub(r'\s+', ' ', html_to_text(text or '')).strip() + + +def _short(text, n=SUMMARY_CHARS): + if len(text) <= n: + return text + cut = text[:n].rsplit(' ', 1)[0] + return cut + ' …' + + +def _when(entry): + """The entry's time (published, else updated), never in the future.""" + now = timezone.now() + for key in ('published_parsed', 'updated_parsed', 'created_parsed'): + t = entry.get(key) + if t: + try: + when = datetime.fromtimestamp(calendar.timegm(t), dt_timezone.utc) + except (OverflowError, ValueError, TypeError): + continue + return min(when, now) + return now + + +class _FirstImage(HTMLParser): + def __init__(self): + super().__init__() + self.src = '' + + def handle_starttag(self, tag, attrs): + if tag == 'img' and not self.src: + a = dict(attrs) + w, h = a.get('width', ''), a.get('height', '') + if w in ('1', '0') or h in ('1', '0'): + return # tracking pixel + self.src = a.get('src') or '' + + +def _image(entry, content_html, base): + for m in entry.get('media_thumbnail') or []: + if m.get('url'): + return m['url'] + for m in entry.get('media_content') or []: + if m.get('url') and (m.get('medium') == 'image' or (m.get('type') or '').startswith('image/')): + return m['url'] + for enc in entry.get('enclosures') or []: + if (enc.get('type') or '').startswith('image/') and enc.get('href'): + return enc['href'] + p = _FirstImage() + try: + p.feed(content_html or '') + except Exception: + pass + return urljoin(base, p.src) if p.src else '' + + +def guid_hash(entry): + key = entry.get('id') or entry.get('link') or f"{entry.get('title', '')}|{entry.get('published', '')}" + return hashlib.sha256(key.encode('utf-8', 'replace')).hexdigest() + + +def parse_entry(entry, feed_url): + """A feedparser entry -> the fields of an Item (without source/user).""" + link = entry.get('link') or '' + base = link or feed_url + contents = [c.get('value') or '' for c in entry.get('content') or []] + content_html = max(contents, key=len) if contents else '' + summary_html = entry.get('summary') or '' + if len(plain(summary_html)) > len(plain(content_html)): + content_html = summary_html + content = clean_html(content_html, base) + text = plain(content) + summary = plain(summary_html) or text + title = plain(html.unescape(entry.get('title') or '')) or _short(text, 80) or 'Ohne Titel' + return { + 'guid_hash': guid_hash(entry), + 'url': link[:2000], + 'title': title[:500], + 'author': plain(entry.get('author') or '')[:300], + 'published': _when(entry), + 'summary': _short(summary), + 'content': content, + 'image_url': _image(entry, content_html, base)[:2000], + } + + +def is_teaser(content_html): + return len(plain(content_html)) < TEASER_CHARS + + +def _drop_title(html_text): + """trafilatura repeats the headline as the first

; the reader shows it already.""" + return re.sub(r'^\s*(]*>\s*)?

.*?

', lambda m: m.group(1) or '', html_text, count=1, flags=re.S) + + +def fetch_article(url): + """The article on `url` as (clean HTML, image URL), or None. trafilatura + finds the main text and leaves out navigation, ads, comments.""" + import trafilatura # heavy import, only where articles are fetched + + try: + r = requests.get(url, timeout=PAGE_TIMEOUT, headers={'User-Agent': USER_AGENT}) + except requests.RequestException as e: + log.info('article %s: %s', url, type(e).__name__) + return None + if r.status_code >= 400 or 'html' not in r.headers.get('Content-Type', 'text/html'): + return None + page = r.text + try: + body = trafilatura.extract(page, url=r.url, output_format='html', include_images=True, + include_links=True, include_tables=True, favor_precision=True) + except Exception: + log.exception('article %s: extraction failed', url) + return None + if not body: + return None + body = re.sub(r'^\s*\s*|\s*\s*$', '', body.strip()) + content = clean_html(_drop_title(body), r.url) + if len(plain(content)) < 200: + return None + image = '' + try: + meta = trafilatura.extract_metadata(page, default_url=r.url) + image = (meta.image if meta else '') or '' + except Exception: + pass + return content, image + + +# ---- finding a site's feed + +FEED_TYPES = ('application/rss+xml', 'application/atom+xml', 'application/feed+json', 'application/xml', 'text/xml') + + +class _FeedLinks(HTMLParser): + def __init__(self): + super().__init__() + self.feeds = [] + self.icon = '' + + def handle_starttag(self, tag, attrs): + if tag != 'link': + return + a = {k.lower(): (v or '') for k, v in attrs} + rel = a.get('rel', '').lower().split() + if 'alternate' in rel and a.get('type', '').lower() in FEED_TYPES and a.get('href'): + self.feeds.append((a['href'], a.get('title', ''))) + if ('icon' in rel or 'apple-touch-icon' in rel) and a.get('href') and not self.icon: + self.icon = a['href'] + + +def discover(page_html, base): + """[(feed url, title)] a page links to, and its icon URL.""" + p = _FeedLinks() + try: + p.feed(page_html) + except Exception: + pass + seen, out = set(), [] + for href, title in p.feeds: + u = urljoin(base, href) + if u not in seen: + seen.add(u) + out.append((u, title)) + return out, (urljoin(base, p.icon) if p.icon else '') + + +def site_icon(site_url): + if not site_url: + return '' + u = urlparse(site_url) + return f'{u.scheme}://{u.netloc}/favicon.ico' if u.scheme and u.netloc else '' diff --git a/feeds/management/__init__.py b/feeds/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/feeds/management/commands/__init__.py b/feeds/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/feeds/management/commands/refresh_newsfeeds.py b/feeds/management/commands/refresh_newsfeeds.py new file mode 100644 index 0000000..7705e0a --- /dev/null +++ b/feeds/management/commands/refresh_newsfeeds.py @@ -0,0 +1,25 @@ +from django.core.management.base import BaseCommand +from django.utils import timezone + +from feeds.models import Source +from feeds.refresh import refresh +from podcasts.views import _in_backoff_q + + +class Command(BaseCommand): + help = 'Refresh the feed reader\'s sources (news and blogs)' + + def add_arguments(self, parser): + parser.add_argument('--source-id', type=int, help='Refresh only this source (even when backing off)') + + def handle(self, *args, **options): + qs = Source.objects.select_related('user').order_by('last_refreshed_at') + if options['source_id']: + qs = qs.filter(pk=options['source_id']) + else: + qs = qs.exclude(_in_backoff_q(timezone.now())) + if not qs.exists(): + self.stdout.write('No sources to refresh.') + return + refreshed, failed, new = refresh(qs) + self.stdout.write(f'{refreshed} sources refreshed, {failed} failed, +{new} items') diff --git a/feeds/migrations/0001_initial.py b/feeds/migrations/0001_initial.py new file mode 100644 index 0000000..810fbc8 --- /dev/null +++ b/feeds/migrations/0001_initial.py @@ -0,0 +1,111 @@ +# Generated by Django 6.1 on 2026-10-04 17:54 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='FeedSettings', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('news_seen_at', models.DateTimeField(blank=True, null=True)), + ('news_keep_days', models.PositiveIntegerField(default=2)), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='feed_settings', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Source', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url', models.URLField(max_length=1000)), + ('title', models.CharField(max_length=300)), + ('site_url', models.URLField(blank=True, max_length=1000)), + ('icon_url', models.URLField(blank=True, max_length=1000)), + ('kind', models.CharField(choices=[('news', 'Nachrichten'), ('blog', 'Blog')], default='news', max_length=10)), + ('category', models.CharField(blank=True, max_length=100)), + ('fulltext', models.CharField(choices=[('auto', 'Automatisch'), ('always', 'Immer'), ('never', 'Nie')], default='auto', max_length=10)), + ('added_at', models.DateTimeField(auto_now_add=True)), + ('last_refreshed_at', models.DateTimeField(blank=True, null=True)), + ('http_etag', models.CharField(blank=True, max_length=500)), + ('http_last_modified', models.CharField(blank=True, max_length=100)), + ('http_body_hash', models.CharField(blank=True, max_length=64)), + ('last_error', models.CharField(blank=True, max_length=500)), + ('last_error_at', models.DateTimeField(blank=True, null=True)), + ('error_count', models.PositiveIntegerField(default=0)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_sources', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['title'], + 'unique_together': {('user', 'url')}, + }, + ), + migrations.CreateModel( + name='Item', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('kind', models.CharField(choices=[('news', 'Nachrichten'), ('blog', 'Blog')], max_length=10)), + ('guid_hash', models.CharField(max_length=64)), + ('url', models.URLField(blank=True, max_length=2000)), + ('title', models.CharField(max_length=500)), + ('author', models.CharField(blank=True, max_length=300)), + ('published', models.DateTimeField(db_index=True)), + ('fetched_at', models.DateTimeField(auto_now_add=True)), + ('summary', models.TextField(blank=True)), + ('content', models.TextField(blank=True)), + ('content_full', models.BooleanField(default=False)), + ('fulltext_failed', models.BooleanField(default=False)), + ('image_url', models.URLField(blank=True, max_length=2000)), + ('read_at', models.DateTimeField(blank=True, null=True)), + ('starred', models.BooleanField(default=False)), + ('gone', models.BooleanField(default=False)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_items', to=settings.AUTH_USER_MODEL)), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='feeds.source')), + ], + options={ + 'ordering': ['-published'], + }, + ), + migrations.CreateModel( + name='Story', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('first_at', models.DateTimeField()), + ('last_at', models.DateTimeField(db_index=True)), + ('size', models.PositiveIntegerField(default=1)), + ('sources', models.PositiveIntegerField(default=1)), + ('terms', models.TextField(blank=True)), + ('lead', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='feeds.item')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_stories', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-last_at'], + }, + ), + migrations.AddField( + model_name='item', + name='story', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='items', to='feeds.story'), + ), + migrations.AddIndex( + model_name='item', + index=models.Index(fields=['user', 'kind', 'published'], name='feeds_item_user_id_1a8bce_idx'), + ), + migrations.AddIndex( + model_name='item', + index=models.Index(fields=['user', 'kind', 'read_at'], name='feeds_item_user_id_0007dd_idx'), + ), + migrations.AlterUniqueTogether( + name='item', + unique_together={('source', 'guid_hash')}, + ), + ] diff --git a/feeds/migrations/__init__.py b/feeds/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/feeds/models.py b/feeds/models.py new file mode 100644 index 0000000..911f4e6 --- /dev/null +++ b/feeds/models.py @@ -0,0 +1,106 @@ +from django.contrib.auth.models import User +from django.db import models + + +class Source(models.Model): + """A subscribed feed. Every source is either news or a blog, and that shapes + everything after it: news gets bundled into stories, shows up in the + "Seit letztem Mal" briefing and expires after a few days; blog articles stay + until read and are listed per blog.""" + NEWS = 'news' + BLOG = 'blog' + KINDS = [(NEWS, 'Nachrichten'), (BLOG, 'Blog')] + + # Whether the whole article is fetched from its page: "auto" when the feed + # only carries a teaser. + FULLTEXT_AUTO = 'auto' + FULLTEXT_ALWAYS = 'always' + FULLTEXT_NEVER = 'never' + FULLTEXTS = [(FULLTEXT_AUTO, 'Automatisch'), (FULLTEXT_ALWAYS, 'Immer'), (FULLTEXT_NEVER, 'Nie')] + + user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_sources') + url = models.URLField(max_length=1000) + title = models.CharField(max_length=300) + site_url = models.URLField(max_length=1000, blank=True) + icon_url = models.URLField(max_length=1000, blank=True) + kind = models.CharField(max_length=10, choices=KINDS, default=NEWS) + # News: the section it belongs to (Politik, Tech …), groups the briefing. + category = models.CharField(max_length=100, blank=True) + fulltext = models.CharField(max_length=10, choices=FULLTEXTS, default=FULLTEXT_AUTO) + added_at = models.DateTimeField(auto_now_add=True) + + # Refresh bookkeeping, as for podcast feeds (see podcasts.views._fetch_feed). + last_refreshed_at = models.DateTimeField(null=True, blank=True) + http_etag = models.CharField(max_length=500, blank=True) + http_last_modified = models.CharField(max_length=100, blank=True) + http_body_hash = models.CharField(max_length=64, blank=True) + last_error = models.CharField(max_length=500, blank=True) + last_error_at = models.DateTimeField(null=True, blank=True) + error_count = models.PositiveIntegerField(default=0) + + class Meta: + unique_together = ('user', 'url') + ordering = ['title'] + + def __str__(self): + return f'{self.title} ({self.kind}, user={self.user_id})' + + +class Story(models.Model): + """News items about the same event from one or more sources (see cluster.py).""" + user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_stories') + # The item the story is shown with (the first one, or the longest text). + lead = models.ForeignKey('Item', null=True, blank=True, on_delete=models.SET_NULL, related_name='+') + first_at = models.DateTimeField() + last_at = models.DateTimeField(db_index=True) + size = models.PositiveIntegerField(default=1) # items in it + sources = models.PositiveIntegerField(default=1) # distinct sources among them + # Words that describe it (space-separated, for matching new items). + terms = models.TextField(blank=True) + + class Meta: + ordering = ['-last_at'] + + +class Item(models.Model): + source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name='items') + user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_items') + kind = models.CharField(max_length=10, choices=Source.KINDS) # the source's, at the time + guid_hash = models.CharField(max_length=64) # sha256 of the entry's id (or link) + url = models.URLField(max_length=2000, blank=True) + title = models.CharField(max_length=500) + author = models.CharField(max_length=300, blank=True) + published = models.DateTimeField(db_index=True) + fetched_at = models.DateTimeField(auto_now_add=True) + summary = models.TextField(blank=True) # plain text, short + content = models.TextField(blank=True) # sanitized HTML + # The content came from the article's page, not the feed. + content_full = models.BooleanField(default=False) + # Fetching the page failed or gave nothing: don't try on every open. + fulltext_failed = models.BooleanField(default=False) + image_url = models.URLField(max_length=2000, blank=True) + read_at = models.DateTimeField(null=True, blank=True) + starred = models.BooleanField(default=False) + story = models.ForeignKey(Story, null=True, blank=True, on_delete=models.SET_NULL, related_name='items') + # Expired news: the row stays a while without its text, so the entry isn't + # taken in again as new while the feed still carries it (refresh.expire). + gone = models.BooleanField(default=False) + + class Meta: + unique_together = ('source', 'guid_hash') + indexes = [ + models.Index(fields=['user', 'kind', 'published']), + models.Index(fields=['user', 'kind', 'read_at']), + ] + ordering = ['-published'] + + def __str__(self): + return self.title + + +class FeedSettings(models.Model): + user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='feed_settings') + # The last time the news briefing was seen; what came after is "Seit letztem Mal". + news_seen_at = models.DateTimeField(null=True, blank=True) + # News older than this goes (unless starred). + news_keep_days = models.PositiveIntegerField(default=2) diff --git a/feeds/refresh.py b/feeds/refresh.py new file mode 100644 index 0000000..aa13999 --- /dev/null +++ b/feeds/refresh.py @@ -0,0 +1,222 @@ +"""Refreshing sources: new entries become items, teasers get their article, +news is bundled into stories, old news expires.""" + +import logging +import time +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait +from datetime import timedelta + +from django.db import IntegrityError, transaction +from django.utils import timezone + +from podcasts.views import FeedFetchError, _clear_error, _error_text, _fetch_feed + +from . import fetch +from .cluster import Clusterer +from .models import FeedSettings, Item, Source, Story + +log = logging.getLogger(__name__) + +FEED_WORKERS = 12 +ARTICLE_WORKERS = 8 +ARTICLES_PER_RUN = 80 # articles fetched per refresh at most; the rest when opened +BLOG_KEEP = 1000 # items kept per blog +BLOG_FIRST_UNREAD_DAYS = 14 # a new blog's older articles start out read +GONE_KEEP_DAYS = 30 # expired news rows kept (without text) against re-adding + + +def settings_for(user): + s, _ = FeedSettings.objects.get_or_create(user=user) + return s + + +def _apply(source, result, now, keep_days): + """Write one download; returns the new items.""" + if result.status == 304: + source.last_refreshed_at = now + _clear_error(source) + source.save(update_fields=['last_refreshed_at', 'last_error', 'last_error_at', 'error_count', + 'http_etag', 'http_last_modified']) + return [] + parsed = result.parsed + feed = parsed.feed + if feed.get('title') and (not source.title or source.title == source.url): + source.title = fetch.plain(feed.title)[:300] + if feed.get('link'): + source.site_url = feed.link[:1000] + if not source.icon_url: + icon = (feed.get('image') or {}).get('href') or feed.get('icon') or fetch.site_icon(source.site_url) + source.icon_url = (icon or '')[:1000] + if result.permanent_url and result.permanent_url != source.url and \ + not Source.objects.filter(user=source.user, url=result.permanent_url).exists(): + source.url = result.permanent_url[:1000] + source.http_etag = result.etag + source.http_last_modified = result.last_modified + source.http_body_hash = result.body_hash + source.last_refreshed_at = now + _clear_error(source) + first = not source.items.exists() + source.save() + + known = set(source.items.values_list('guid_hash', flat=True)) + oldest_kept = None + if source.kind == Source.BLOG and len(known) >= BLOG_KEEP: + oldest_kept = source.items.order_by('published').values_list('published', flat=True).first() + news_cutoff = now - timedelta(days=keep_days) + new = [] + for entry in parsed.entries: + fields = fetch.parse_entry(entry, source.url) + if fields['guid_hash'] in known: + continue + known.add(fields['guid_hash']) + if source.kind == Source.NEWS and fields['published'] < news_cutoff: + continue + if oldest_kept and fields['published'] < oldest_kept: + continue + item = Item(source=source, user=source.user, kind=source.kind, **fields) + if source.kind == Source.BLOG and first and fields['published'] < now - timedelta(days=BLOG_FIRST_UNREAD_DAYS): + item.read_at = now + new.append(item) + if new: + try: + with transaction.atomic(): + Item.objects.bulk_create(new) + except IntegrityError: + # Two entries of one feed with the same id: one by one, skipping the doubles. + kept = [] + for item in new: + try: + with transaction.atomic(): + item.save() + kept.append(item) + except IntegrityError: + pass + new = kept + return new + + +def needs_article(item, source): + if not item.url or item.content_full or item.fulltext_failed: + return False + if source.fulltext == Source.FULLTEXT_NEVER: + return False + return source.fulltext == Source.FULLTEXT_ALWAYS or fetch.is_teaser(item.content) + + +def load_article(item): + """Fetch the article for one item now (when opened); saves it.""" + got = fetch.fetch_article(item.url) + _store_article(item, got) + return item + + +def _store_article(item, got): + if got is None: + item.fulltext_failed = True + item.save(update_fields=['fulltext_failed']) + return + content, image = got + if len(fetch.plain(content)) <= len(fetch.plain(item.content)): + item.fulltext_failed = True # the feed had more + item.save(update_fields=['fulltext_failed']) + return + item.content = content + item.content_full = True + if not item.image_url and image: + item.image_url = image[:2000] + item.save(update_fields=['content', 'content_full', 'image_url']) + + +def _articles(items, deadline): + todo = [i for i in items if needs_article(i, i.source)][:ARTICLES_PER_RUN] + if not todo: + return + pool = ThreadPoolExecutor(ARTICLE_WORKERS) + futures = {pool.submit(fetch.fetch_article, i.url): i for i in todo} + pending = set(futures) + try: + while pending: + left = None if deadline is None else deadline - time.monotonic() + if left is not None and left <= 0: + break + done, pending = wait(pending, timeout=left, return_when=FIRST_COMPLETED) + for f in done: + try: + _store_article(futures[f], f.result()) + except Exception: + log.exception('article for item %s', futures[f].pk) + finally: + pool.shutdown(wait=False, cancel_futures=True) + + +def expire(user, now=None): + """News past the user's keep time loses its text (rows stay GONE_KEEP_DAYS + as markers), stories without visible items go.""" + now = now or timezone.now() + keep = settings_for(user).news_keep_days + Item.objects.filter(user=user, kind=Source.NEWS, gone=False, starred=False, + published__lt=now - timedelta(days=keep)).update( + gone=True, content='', summary='', story=None) + Item.objects.filter(user=user, gone=True, published__lt=now - timedelta(days=GONE_KEEP_DAYS)).delete() + Story.objects.filter(user=user).exclude(items__gone=False).delete() + # Blogs: the oldest read articles beyond BLOG_KEEP per blog. + for source in Source.objects.filter(user=user, kind=Source.BLOG): + ids = list(source.items.order_by('-published').values_list('id', flat=True)[BLOG_KEEP:]) + if ids: + Item.objects.filter(id__in=ids, starred=False).exclude(read_at=None).delete() + + +def refresh(sources, deadline=None): + """Refresh `sources` (downloads in parallel, writes one by one). + Returns (refreshed, failed, new items).""" + sources = list(sources) + now = timezone.now() + keep = {} + refreshed = failed = 0 + new_items = [] + pool = ThreadPoolExecutor(FEED_WORKERS) + try: + futures = { + pool.submit(_fetch_feed, s.url, s.http_etag, s.http_last_modified, s.http_body_hash, + user_agent=fetch.USER_AGENT): s + for s in sources + } + try: + timeout = None if deadline is None else max(0.0, deadline - time.monotonic()) + for f in as_completed(futures, timeout=timeout): + s = futures[f] + try: + result = f.result() + if s.user_id not in keep: + keep[s.user_id] = settings_for(s.user).news_keep_days + new_items += _apply(s, result, now, keep[s.user_id]) + refreshed += 1 + except Exception as e: + if not isinstance(e, FeedFetchError): + log.exception('feed source %s', s.pk) + s.last_error = _error_text(e) + s.last_error_at = now + s.error_count += 1 + s.save(update_fields=['last_error', 'last_error_at', 'error_count']) + failed += 1 + except TimeoutError: + pass + finally: + # Past the deadline: don't wait for the slow ones (they stay as they are). + pool.shutdown(wait=False, cancel_futures=True) + + for item in new_items: + item.source = next(s for s in sources if s.pk == item.source_id) + _articles(new_items, deadline) + + by_user = {} + for item in new_items: + if item.kind == Source.NEWS: + by_user.setdefault(item.user_id, []).append(item) + for user_id, items in by_user.items(): + c = Clusterer(items[0].user, now) + for item in sorted(items, key=lambda i: i.published): + c.add(item) + for user_id in {s.user_id for s in sources}: + expire(next(s.user for s in sources if s.user_id == user_id), now) + return refreshed, failed, len(new_items) diff --git a/feeds/tests.py b/feeds/tests.py new file mode 100644 index 0000000..ee4ba97 --- /dev/null +++ b/feeds/tests.py @@ -0,0 +1,179 @@ +import json +from datetime import timedelta +from unittest import mock + +import feedparser +from django.contrib.auth.models import User +from django.test import TestCase +from django.utils import timezone + +from podcasts.views import FeedFetch + +from . import cluster, fetch, refresh +from .models import FeedSettings, Item, Source, Story + + +def rss(items, title='Testfeed', link='https://example.org/'): + body = ''.join( + f'{i["guid"]}{i["title"]}{i.get("link", "https://example.org/" + i["guid"])}' + f'{i.get("date", timezone.now().strftime("%a, %d %b %Y %H:%M:%S +0000"))}' + f'' + for i in items) + return f'{title}{link}{body}'.encode() + + +def fetched(content, url='https://example.org/feed'): + return FeedFetch(200, feedparser.parse(content), 'etag', '', url, url, 'hash') + + +LONG = '

' + 'Ein langer Absatz mit sehr viel Text. ' * 40 + '

' + + +class FetchTest(TestCase): + def test_relative_links_become_absolute_and_scripts_go(self): + out = fetch.clean_html('

Hi x' + '

', 'https://ex.org/post/1') + self.assertIn('href="https://ex.org/a"', out) + self.assertIn('src="https://ex.org/post/i.jpg"', out) + self.assertIn('src="https://ex.org/real.jpg"', out) + self.assertNotIn('placeholder', out) + self.assertNotIn('bad()', out) + + def test_parse_entry(self): + e = feedparser.parse(rss([{'guid': 'g1', 'title': 'A & B', 'text': '

Kurz

'}])).entries[0] + f = fetch.parse_entry(e, 'https://example.org/feed') + self.assertEqual(f['title'], 'A & B') + self.assertEqual(f['image_url'], 'https://example.org/p.jpg') + self.assertTrue(fetch.is_teaser(f['content'])) + + def test_drop_repeated_headline(self): + self.assertEqual(fetch._drop_title('

Titel

Text

'), '

Text

') + + def test_discover(self): + feeds, icon = fetch.discover('' + '', 'https://ex.org/') + self.assertEqual(feeds, [('https://ex.org/feed', 'F')]) + self.assertEqual(icon, 'https://ex.org/fav.png') + + +class ClusterTest(TestCase): + def setUp(self): + self.u = User.objects.create_user('u', password='pw12345678') + self.a = Source.objects.create(user=self.u, url='https://a/', title='A') + self.b = Source.objects.create(user=self.u, url='https://b/', title='B') + self.c = Source.objects.create(user=self.u, url='https://c/', title='C') + + def item(self, source, title, summary=''): + return Item.objects.create(source=source, user=self.u, kind='news', guid_hash=title[:60], + title=title, summary=summary, published=timezone.now()) + + def test_same_event_bundled_other_apart(self): + c = cluster.Clusterer(self.u, timezone.now()) + s1 = c.add(self.item(self.a, 'Oktoberfest endet mit neuem Besucherrekord in München')) + s2 = c.add(self.item(self.b, 'Rekord-Wiesn: 7,4 Millionen Besucher auf dem Oktoberfest in München')) + s3 = c.add(self.item(self.c, 'Bahn führt Familienticket dauerhaft ein')) + self.assertEqual(s1, s2) + self.assertNotEqual(s1, s3) + s1.refresh_from_db() + self.assertEqual((s1.size, s1.sources), (2, 2)) + + def test_same_source_needs_to_be_very_close(self): + c = cluster.Clusterer(self.u, timezone.now()) + s1 = c.add(self.item(self.a, 'Nations League: Pressekonferenz vor Irland gegen Israel abgebrochen')) + s2 = c.add(self.item(self.a, 'Nations League: England schlägt Kroatien deutlich')) + self.assertNotEqual(s1, s2) + + +class ApiTest(TestCase): + def setUp(self): + self.u = User.objects.create_user('tester', password='pw12345678') + self.client.force_login(self.u) + + def post(self, url, data=None): + return self.client.post(url, json.dumps(data or {}), content_type='application/json') + + def add(self, kind, feed, url='https://example.org/feed'): + with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed, url)), \ + mock.patch('feeds.fetch.fetch_article', return_value=(LONG, 'https://example.org/img.jpg')): + r = self.post('/feeds/sources/add/', {'url': url, 'kind': kind, 'category': 'Politik'}) + self.assertEqual(r.status_code, 200) + return r.json()['source'] + + def test_requires_login(self): + self.client.logout() + self.assertEqual(self.client.get('/feeds/state/').status_code, 401) + + def test_add_news_fetches_articles_and_bundles(self): + s = self.add('news', rss([{'guid': 'a', 'title': 'Merz besucht Kiew und sagt Hilfe zu', 'text': 'Kurz.'}, + {'guid': 'b', 'title': 'Bahn Familienticket dauerhaft', 'text': 'Kurz.'}])) + self.assertEqual(s['title'], 'Testfeed') + self.assertEqual(s['unread'], 2) + items = Item.objects.filter(user=self.u) + self.assertTrue(all(i.content_full for i in items)) + self.assertEqual(Story.objects.filter(user=self.u).count(), 2) + news = self.client.get('/feeds/news/').json() + self.assertEqual(len(news['stories']), 2) + self.assertEqual(news['stories'][0]['category'], 'Politik') + b = self.client.get('/feeds/briefing/').json() + self.assertEqual(b['total'], 2) + self.assertEqual(b['sections'][0]['category'], 'Politik') + self.post('/feeds/briefing/seen/') + self.assertFalse(Item.objects.filter(user=self.u, read_at=None).exists()) + self.assertEqual(self.client.get('/feeds/briefing/').json()['total'], 0) + + def test_blog_keeps_full_feed_text_and_lists(self): + self.add('blog', rss([{'guid': 'p1', 'title': 'Ein Artikel', 'text': LONG}])) + item = Item.objects.get(user=self.u) + self.assertFalse(item.content_full) # the feed had the whole text + self.assertIsNone(item.story) + blogs = self.client.get('/feeds/blogs/?unread=1').json() + self.assertEqual([i['title'] for i in blogs['items']], ['Ein Artikel']) + detail = self.client.get(f'/feeds/items/{item.id}/').json()['item'] + self.assertIn('langer Absatz', detail['content']) + self.post('/feeds/items/read/', {'ids': [item.id]}) + self.assertEqual(self.client.get('/feeds/blogs/?unread=1').json()['items'], []) + self.post(f'/feeds/items/{item.id}/star/', {'starred': True}) + self.assertEqual(len(self.client.get('/feeds/starred/').json()['items']), 1) + + def test_old_news_expires_and_is_not_taken_in_again(self): + old = (timezone.now() - timedelta(days=3)).strftime('%a, %d %b %Y %H:%M:%S +0000') + feed = rss([{'guid': 'new', 'title': 'Heute passiert', 'text': LONG}, + {'guid': 'old', 'title': 'Vorgestern passiert', 'text': LONG, 'date': old}]) + src = self.add('news', feed) + self.assertEqual(Item.objects.filter(user=self.u).count(), 1) # too old to take in + Item.objects.filter(user=self.u).update(published=timezone.now() - timedelta(days=3)) + refresh.expire(self.u) + self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0) + self.assertEqual(Story.objects.filter(user=self.u).count(), 0) + source = Source.objects.get(pk=src['id']) + with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed)): + refresh.refresh([source]) + self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0) + + def test_switching_kind_moves_items(self): + src = self.add('news', rss([{'guid': 'a', 'title': 'Etwas', 'text': LONG}])) + self.post(f'/feeds/sources/{src["id"]}/', {'kind': 'blog'}) + item = Item.objects.get(user=self.u) + self.assertEqual((item.kind, item.story), ('blog', None)) + self.assertEqual(Story.objects.filter(user=self.u).count(), 0) + + def test_other_users_items_invisible(self): + self.add('blog', rss([{'guid': 'p1', 'title': 'Privat', 'text': LONG}])) + item = Item.objects.get(user=self.u) + other = User.objects.create_user('o', password='pw12345678') + self.client.force_login(other) + self.assertEqual(self.client.get(f'/feeds/items/{item.id}/').status_code, 404) + self.assertEqual(self.client.get('/feeds/blogs/').json()['items'], []) + self.post('/feeds/items/read/', {'ids': [item.id]}) + item.refresh_from_db() + self.assertIsNone(item.read_at) + + def test_opml_export(self): + self.add('blog', rss([]), url='https://blog.example/feed') + r = self.client.get('/feeds/export.opml') + self.assertIn(b'xmlUrl="https://blog.example/feed"', r.content) + self.assertIn(b'text="Blogs"', r.content) + + def test_settings(self): + self.post('/feeds/settings/', {'news_keep_days': 5}) + self.assertEqual(FeedSettings.objects.get(user=self.u).news_keep_days, 5) diff --git a/feeds/urls.py b/feeds/urls.py new file mode 100644 index 0000000..aa3d3f0 --- /dev/null +++ b/feeds/urls.py @@ -0,0 +1,24 @@ +from django.urls import path + +from . import views + +urlpatterns = [ + path('state/', views.state, name='feeds_state'), + path('discover/', views.discover, name='feeds_discover'), + path('sources/add/', views.add_source, name='feeds_add_source'), + path('sources/bulk/', views.add_sources, name='feeds_add_sources'), + path('sources//', views.edit_source, name='feeds_edit_source'), + path('sources//delete/', views.delete_source, name='feeds_delete_source'), + path('export.opml', views.export_opml, name='feeds_export_opml'), + path('refresh/', views.refresh_now, name='feeds_refresh'), + path('briefing/', views.briefing, name='feeds_briefing'), + path('briefing/seen/', views.briefing_seen, name='feeds_briefing_seen'), + path('news/', views.news, name='feeds_news'), + path('stories//', views.story, name='feeds_story'), + path('blogs/', views.blogs, name='feeds_blogs'), + path('starred/', views.starred, name='feeds_starred'), + path('items/read/', views.mark_read, name='feeds_mark_read'), + path('items//', views.item, name='feeds_item'), + path('items//star/', views.star, name='feeds_star'), + path('settings/', views.settings, name='feeds_settings'), +] diff --git a/feeds/views.py b/feeds/views.py new file mode 100644 index 0000000..5e90f11 --- /dev/null +++ b/feeds/views.py @@ -0,0 +1,445 @@ +"""The feed reader's JSON API (session or Bearer token, like the podcast API). + +Sources: GET /feeds/state/ sources, unread counts, settings + POST /feeds/discover/ {url} -> the feeds a page or URL offers + POST /feeds/sources/add/ {url, kind, category, title?} + POST /feeds/sources/bulk/ {sources: [{url, title, kind, category}]} (OPML import) + POST /feeds/sources// {title, kind, category, fulltext} + POST /feeds/sources//delete/ + GET /feeds/export.opml + POST /feeds/refresh/ refresh the account's sources now +News: GET /feeds/briefing/ what's new since the briefing was last seen + POST /feeds/briefing/seen/ seen: everything up to now counts as read + GET /feeds/news/?before=&category= stories, newest first + GET /feeds/stories// one story with all its items +Blogs: GET /feeds/blogs/?source=&unread=1&before= +Items: GET /feeds/items// the article (fetches the page if the feed had a teaser) + POST /feeds/items/read/ {ids, read} + POST /feeds/items//star/ {starred} + GET /feeds/starred/ +Settings: POST /feeds/settings/ {news_keep_days} +""" + +import json +import time +from datetime import timedelta +from xml.sax.saxutils import quoteattr + +import feedparser +import requests +from django.db.models import Count +from django.http import HttpResponse, JsonResponse +from django.utils import timezone +from django.utils.dateparse import parse_datetime +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods + +from . import fetch, refresh +from .models import Item, Source, Story + +PAGE = 40 +BRIEFING_TOP = 8 +BRIEFING_PER_CATEGORY = 5 +REFRESH_WAIT = 40 # seconds an explicit refresh may take +CATEGORY_NONE = 'Allgemein' + + +def _auth(request): + if not request.user.is_authenticated: + return JsonResponse({'error': 'authentication required'}, status=401) + return None + + +def _body(request): + try: + return json.loads(request.body or b'{}') + except (ValueError, UnicodeDecodeError): + return None + + +def _iso(dt): + return dt.isoformat() if dt else None + + +def _source_data(s, unread=0): + return { + 'id': s.id, 'url': s.url, 'title': s.title, 'site_url': s.site_url, 'icon_url': s.icon_url, + 'kind': s.kind, 'category': s.category, 'fulltext': s.fulltext, 'unread': unread, + 'last_refreshed_at': _iso(s.last_refreshed_at), 'last_error': s.last_error, 'error_count': s.error_count, + } + + +def _item_data(i, content=False): + d = { + 'id': i.id, 'source': i.source_id, 'source_title': i.source.title, 'kind': i.kind, + 'url': i.url, 'title': i.title, 'author': i.author, 'published': _iso(i.published), + 'summary': i.summary, 'image_url': i.image_url, 'read': i.read_at is not None, + 'starred': i.starred, 'story': i.story_id, + } + if content: + d['content'] = i.content + d['content_full'] = i.content_full + return d + + +def _story_data(s, items): + items = sorted(items, key=lambda i: i.published) + lead = next((i for i in items if i.id == s.lead_id), items[0] if items else None) + return { + 'id': s.id, 'first_at': _iso(s.first_at), 'last_at': _iso(s.last_at), + 'size': len(items), 'sources': len({i.source_id for i in items}), + 'category': (lead.source.category if lead else '') or CATEGORY_NONE, + 'unread': sum(1 for i in items if i.read_at is None), + 'lead': _item_data(lead) if lead else None, + 'items': [_item_data(i) for i in items], + } + + +def _stories(user, qs): + """Story dicts for a Story queryset (items fetched in one go).""" + stories = list(qs) + items = {} + for i in Item.objects.filter(story__in=stories, gone=False).select_related('source'): + items.setdefault(i.story_id, []).append(i) + return [_story_data(s, items[s.id]) for s in stories if s.id in items] + + +# ---- sources + +@csrf_exempt +@require_http_methods(['GET']) +def state(request): + if (err := _auth(request)): + return err + unread = dict(Item.objects.filter(user=request.user, gone=False, read_at=None) + .values_list('source').annotate(n=Count('id'))) + sources = list(Source.objects.filter(user=request.user)) + s = refresh.settings_for(request.user) + since = s.news_seen_at or (timezone.now() - timedelta(days=1)) + new_news = Story.objects.filter(user=request.user, last_at__gt=since, + items__gone=False, items__read_at=None).distinct().count() + return JsonResponse({ + 'sources': [_source_data(x, unread.get(x.id, 0)) for x in sources], + 'categories': sorted({x.category for x in sources if x.kind == Source.NEWS and x.category}), + 'news_new': new_news, + 'blogs_unread': sum(n for sid, n in unread.items() if any(x.id == sid and x.kind == Source.BLOG for x in sources)), + 'settings': {'news_keep_days': s.news_keep_days, 'news_seen_at': _iso(s.news_seen_at)}, + }) + + +def _probe(url): + """(feed url, title, entries) if `url` is a feed, else ([candidate feeds], icon).""" + r = requests.get(url, timeout=fetch.PAGE_TIMEOUT, headers={'User-Agent': fetch.USER_AGENT}) + r.raise_for_status() + ctype = r.headers.get('Content-Type', '') + if 'html' in ctype and b'= 25 or teaser >= 3 else Source.BLOG, + }) + return JsonResponse({'feeds': out}) + + +def _kind(v): + return v if v in (Source.NEWS, Source.BLOG) else None + + +@csrf_exempt +@require_http_methods(['POST']) +def add_source(request): + if (err := _auth(request)): + return err + body = _body(request) or {} + url = (body.get('url') or '').strip() + kind = _kind(body.get('kind')) + if not url.startswith(('http://', 'https://')) or not kind: + return JsonResponse({'error': 'Adresse und Art (Nachrichten/Blog) nötig'}, status=400) + source, created = Source.objects.get_or_create( + user=request.user, url=url[:1000], + defaults={'title': (body.get('title') or url)[:300], 'kind': kind, + 'category': (body.get('category') or '').strip()[:100]}) + if created: + refresh.refresh([source], deadline=time.monotonic() + 25) + source.refresh_from_db() + return JsonResponse({'source': _source_data(source, source.items.filter(read_at=None, gone=False).count()), + 'created': created}) + + +@csrf_exempt +@require_http_methods(['POST']) +def add_sources(request): + """Many at once (an OPML file the page read); refreshed by the next run.""" + if (err := _auth(request)): + return err + body = _body(request) or {} + added = 0 + for x in body.get('sources') or []: + url = (x.get('url') or '').strip() + kind = _kind(x.get('kind')) + if not url.startswith(('http://', 'https://')) or not kind: + continue + _, created = Source.objects.get_or_create( + user=request.user, url=url[:1000], + defaults={'title': (x.get('title') or url)[:300], 'kind': kind, + 'category': (x.get('category') or '').strip()[:100]}) + added += created + sources = Source.objects.filter(user=request.user, last_refreshed_at=None) + refresh.refresh(sources, deadline=time.monotonic() + REFRESH_WAIT) + return JsonResponse({'added': added}) + + +@csrf_exempt +@require_http_methods(['POST']) +def edit_source(request, pk): + if (err := _auth(request)): + return err + source = Source.objects.filter(user=request.user, pk=pk).first() + body = _body(request) + if source is None or body is None: + return JsonResponse({'error': 'not found'}, status=404) + if 'title' in body and (body['title'] or '').strip(): + source.title = body['title'].strip()[:300] + if 'category' in body: + source.category = (body['category'] or '').strip()[:100] + if body.get('fulltext') in dict(Source.FULLTEXTS): + source.fulltext = body['fulltext'] + new_kind = _kind(body.get('kind')) + if new_kind and new_kind != source.kind: + source.kind = new_kind + # Its items follow; as blog articles they leave their stories. + source.items.update(kind=new_kind, **({'story': None} if new_kind == Source.BLOG else {})) + Story.objects.filter(user=request.user).exclude(items__gone=False).delete() + source.save() + return JsonResponse({'source': _source_data(source)}) + + +@csrf_exempt +@require_http_methods(['POST']) +def delete_source(request, pk): + if (err := _auth(request)): + return err + Source.objects.filter(user=request.user, pk=pk).delete() + Story.objects.filter(user=request.user).exclude(items__gone=False).delete() + return JsonResponse({'ok': True}) + + +@csrf_exempt +@require_http_methods(['GET']) +def export_opml(request): + if (err := _auth(request)): + return err + groups = {} + for s in Source.objects.filter(user=request.user).order_by('kind', 'category', 'title'): + label = ('Nachrichten' + (f' – {s.category}' if s.category else '')) if s.kind == Source.NEWS else 'Blogs' + groups.setdefault(label, []).append(s) + lines = ['', '', + 'diora Feeds', ''] + for label, sources in groups.items(): + lines.append(f'') + for s in sources: + lines.append(f'') + lines.append('') + lines += ['', ''] + resp = HttpResponse('\n'.join(lines), content_type='text/x-opml; charset=utf-8') + resp['Content-Disposition'] = 'attachment; filename="diora-feeds.opml"' + return resp + + +@csrf_exempt +@require_http_methods(['POST']) +def refresh_now(request): + if (err := _auth(request)): + return err + refreshed, failed, new = refresh.refresh(Source.objects.filter(user=request.user), + deadline=time.monotonic() + REFRESH_WAIT) + return JsonResponse({'refreshed': refreshed, 'failed': failed, 'new': new}) + + +# ---- news + +@csrf_exempt +@require_http_methods(['GET']) +def briefing(request): + """Since the briefing was last seen (else the last day): the top stories + (reported by several sources), then the rest by section, newest first.""" + if (err := _auth(request)): + return err + s = refresh.settings_for(request.user) + since = s.news_seen_at or (timezone.now() - timedelta(days=1)) + qs = Story.objects.filter(user=request.user, last_at__gt=since, + items__gone=False, items__read_at=None).distinct() + stories = _stories(request.user, qs) + stories.sort(key=lambda x: (-x['sources'], -x['size'], x['last_at']), reverse=False) + top = [x for x in stories if x['sources'] >= 2][:BRIEFING_TOP] + top_ids = {x['id'] for x in top} + sections = {} + for x in sorted(stories, key=lambda x: x['last_at'], reverse=True): + if x['id'] not in top_ids: + sections.setdefault(x['category'], []).append(x) + return JsonResponse({ + 'since': _iso(since), 'total': len(stories), 'top': top, + 'sections': [{'category': c, 'count': len(v), 'stories': v[:BRIEFING_PER_CATEGORY]} + for c, v in sorted(sections.items(), key=lambda kv: (kv[0] == CATEGORY_NONE, kv[0]))], + }) + + +@csrf_exempt +@require_http_methods(['POST']) +def briefing_seen(request): + if (err := _auth(request)): + return err + now = timezone.now() + s = refresh.settings_for(request.user) + Item.objects.filter(user=request.user, kind=Source.NEWS, read_at=None, fetched_at__lte=now).update(read_at=now) + s.news_seen_at = now + s.save(update_fields=['news_seen_at']) + return JsonResponse({'news_seen_at': _iso(now)}) + + +@csrf_exempt +@require_http_methods(['GET']) +def news(request): + if (err := _auth(request)): + return err + qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at') + before = parse_datetime(request.GET.get('before') or '') + if before: + qs = qs.filter(last_at__lt=before) + category = request.GET.get('category') + if category: + qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category) + stories = _stories(request.user, qs[:PAGE]) + return JsonResponse({'stories': stories, 'more': len(stories) == PAGE}) + + +@csrf_exempt +@require_http_methods(['GET']) +def story(request, pk): + if (err := _auth(request)): + return err + found = _stories(request.user, Story.objects.filter(user=request.user, pk=pk)) + if not found: + return JsonResponse({'error': 'not found'}, status=404) + return JsonResponse({'story': found[0]}) + + +# ---- blogs, items + +@csrf_exempt +@require_http_methods(['GET']) +def blogs(request): + if (err := _auth(request)): + return err + qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source') + if request.GET.get('source', '').isdigit(): + qs = qs.filter(source_id=int(request.GET['source'])) + if request.GET.get('unread') == '1': + qs = qs.filter(read_at=None) + before = parse_datetime(request.GET.get('before') or '') + if before: + qs = qs.filter(published__lt=before) + items = list(qs.order_by('-published')[:PAGE]) + return JsonResponse({'items': [_item_data(i) for i in items], 'more': len(items) == PAGE}) + + +@csrf_exempt +@require_http_methods(['GET']) +def starred(request): + if (err := _auth(request)): + return err + items = Item.objects.filter(user=request.user, starred=True).select_related('source').order_by('-published') + return JsonResponse({'items': [_item_data(i) for i in items[:200]]}) + + +@csrf_exempt +@require_http_methods(['GET']) +def item(request, pk): + if (err := _auth(request)): + return err + i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source').first() + if i is None: + return JsonResponse({'error': 'not found'}, status=404) + if refresh.needs_article(i, i.source): + refresh.load_article(i) + return JsonResponse({'item': _item_data(i, content=True)}) + + +@csrf_exempt +@require_http_methods(['POST']) +def mark_read(request): + if (err := _auth(request)): + return err + body = _body(request) or {} + ids = [int(x) for x in body.get('ids') or [] if str(x).isdigit()] + qs = Item.objects.filter(user=request.user, id__in=ids) + n = qs.update(read_at=timezone.now() if body.get('read', True) else None) + return JsonResponse({'updated': n}) + + +@csrf_exempt +@require_http_methods(['POST']) +def star(request, pk): + if (err := _auth(request)): + return err + body = _body(request) or {} + n = Item.objects.filter(user=request.user, pk=pk).update(starred=bool(body.get('starred', True))) + return JsonResponse({'updated': n}) + + +@csrf_exempt +@require_http_methods(['POST']) +def settings(request): + if (err := _auth(request)): + return err + body = _body(request) or {} + s = refresh.settings_for(request.user) + days = body.get('news_keep_days') + if isinstance(days, int) and 1 <= days <= 30: + s.news_keep_days = days + s.save(update_fields=['news_keep_days']) + return JsonResponse({'news_keep_days': s.news_keep_days}) diff --git a/podcasts/views.py b/podcasts/views.py index d2f1c9d..50f3f8e 100644 --- a/podcasts/views.py +++ b/podcasts/views.py @@ -322,13 +322,13 @@ def _body_hash(content): PERMANENT_REDIRECTS = (301, 308) -def _fetch_feed(url, etag='', last_modified='', body_hash=''): +def _fetch_feed(url, etag='', last_modified='', body_hash='', user_agent='diora podcast refresh'): """Download and parse a feed (with a timeout, unlike feedparser's own fetch). With etag/last_modified it is a conditional GET; with body_hash a body like the last one counts as unchanged too (a 304), unparsed — parsing is what takes the time (seconds for a big feed). Returns a FeedFetch; raises FeedFetchError when there is nothing usable.""" - headers = {'User-Agent': 'diora podcast refresh'} + headers = {'User-Agent': user_agent} if etag: headers['If-None-Match'] = etag if last_modified: diff --git a/requirements.txt b/requirements.txt index 424fcbe..c08aad3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,5 @@ packaging==26.3 piper-tts==1.7.0 # ffmpeg (a static binary in the wheel) finds the pauses for skipping silence in the browser. imageio-ffmpeg==0.6.0 +# The feed reader takes the article out of a news page (main text without menus, ads, comments). +trafilatura==2.3.0