diff --git a/feeds/fetch.py b/feeds/fetch.py
index b535224..4b23c75 100644
--- a/feeds/fetch.py
+++ b/feeds/fetch.py
@@ -224,6 +224,16 @@ def discover(page_html, base):
return out, (urljoin(base, p.icon) if p.icon else '')
+def short_title(title):
+ """A feed's name without the slogan: "tagesschau.de - die erste Adresse für …"
+ -> "tagesschau.de", "Aktuell - FAZ.NET" -> "FAZ.NET", "DIE ZEIT | Nachrichten, …" -> "DIE ZEIT"."""
+ parts = [p.strip() for p in re.split(r'\s+[-–—|:]\s+', title or '') if p.strip()]
+ if len(parts) < 2:
+ return (title or '').strip()
+ branded = [p for p in parts if re.search(r'\w\.[a-z]{2,}$', p, re.I)]
+ return (branded or parts)[0]
+
+
def site_icon(site_url):
if not site_url:
return ''
diff --git a/feeds/refresh.py b/feeds/refresh.py
index aa13999..d82796f 100644
--- a/feeds/refresh.py
+++ b/feeds/refresh.py
@@ -3,6 +3,7 @@ news is bundled into stories, old news expires."""
import logging
import time
+from collections import Counter
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
from datetime import timedelta
@@ -41,7 +42,7 @@ def _apply(source, result, now, keep_days):
parsed = result.parsed
feed = parsed.feed
if feed.get('title') and (not source.title or source.title == source.url):
- source.title = fetch.plain(feed.title)[:300]
+ source.title = fetch.short_title(fetch.plain(feed.title))[:300]
if feed.get('link'):
source.site_url = feed.link[:1000]
if not source.icon_url:
@@ -64,8 +65,14 @@ def _apply(source, result, now, keep_days):
oldest_kept = source.items.order_by('published').values_list('published', flat=True).first()
news_cutoff = now - timedelta(days=keep_days)
new = []
- for entry in parsed.entries:
- fields = fetch.parse_entry(entry, source.url)
+ entries = [fetch.parse_entry(e, source.url) for e in parsed.entries]
+ # Some feeds (Deutschlandfunk) give every entry the time the feed was made:
+ # then the dates say nothing, and the time an entry first shows up counts.
+ stamps = Counter(f['published'] for f in entries)
+ bogus_dates = len(entries) >= 5 and stamps.most_common(1)[0][1] >= 0.8 * len(entries)
+ for fields in entries:
+ if bogus_dates:
+ fields['published'] = now
if fields['guid_hash'] in known:
continue
known.add(fields['guid_hash'])
diff --git a/feeds/tests.py b/feeds/tests.py
index ee4ba97..3f0dbf8 100644
--- a/feeds/tests.py
+++ b/feeds/tests.py
@@ -49,6 +49,13 @@ class FetchTest(TestCase):
def test_drop_repeated_headline(self):
self.assertEqual(fetch._drop_title('
Titel
Text
'), '
Text
')
+ def test_short_title(self):
+ self.assertEqual(fetch.short_title('tagesschau.de - die erste Adresse für Nachrichten'), 'tagesschau.de')
+ self.assertEqual(fetch.short_title('Aktuell - FAZ.NET'), 'FAZ.NET')
+ self.assertEqual(fetch.short_title('DIE ZEIT | Nachrichten, News, Hintergründe'), 'DIE ZEIT')
+ self.assertEqual(fetch.short_title('Golem.de'), 'Golem.de')
+ self.assertEqual(fetch.short_title('Kotlin-Blog'), 'Kotlin-Blog')
+
def test_discover(self):
feeds, icon = fetch.discover(''
'', 'https://ex.org/')
@@ -150,6 +157,12 @@ class ApiTest(TestCase):
refresh.refresh([source])
self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0)
+ def test_feed_wide_timestamp_means_first_seen(self):
+ stamp = (timezone.now() - timedelta(hours=5)).strftime('%a, %d %b %Y %H:%M:%S +0000')
+ self.add('news', rss([{'guid': f'g{n}', 'title': f'Meldung {n}', 'text': LONG, 'date': stamp} for n in range(6)]))
+ for i in Item.objects.filter(user=self.u):
+ self.assertLess((timezone.now() - i.published).total_seconds(), 60)
+
def test_switching_kind_moves_items(self):
src = self.add('news', rss([{'guid': 'a', 'title': 'Etwas', 'text': LONG}]))
self.post(f'/feeds/sources/{src["id"]}/', {'kind': 'blog'})
diff --git a/feeds/views.py b/feeds/views.py
index 5e90f11..5bb5471 100644
--- a/feeds/views.py
+++ b/feeds/views.py
@@ -137,7 +137,7 @@ def _probe(url):
parsed = feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
if parsed.get('bozo') and not parsed.entries and not parsed.feed.get('title'):
return None, ([], '')
- return (r.url, fetch.plain(parsed.feed.get('title') or '') or r.url, parsed), None
+ return (r.url, fetch.short_title(fetch.plain(parsed.feed.get('title') or '')) or r.url, parsed), None
@csrf_exempt
diff --git a/static/js/app.js b/static/js/app.js
index b1c5e61..d12f1a5 100644
--- a/static/js/app.js
+++ b/static/js/app.js
@@ -1179,7 +1179,7 @@ async function editNotes(pk, current) {
// Tabs
// ---------------------------------------------------------------------------
-const TOP_TABS = ['home', 'radio', 'focus', 'podcasts', 'books', 'ambient'];
+const TOP_TABS = ['home', 'radio', 'focus', 'podcasts', 'books', 'feeds', 'ambient'];
const RADIO_SUB_TABS = ['search', 'saved', 'history'];
// ---------------------------------------------------------------------------
@@ -1238,6 +1238,7 @@ function showTab(name) {
if (name === 'books') loadBookList();
if (name === 'ambient') _ambientLoad();
if (name === 'home' && window.__dioraHomeShow) window.__dioraHomeShow(); // static/js/home.js
+ if (name === 'feeds' && window.__dioraFeedsShow) window.__dioraFeedsShow(); // static/js/feeds.js
}
// ---------------------------------------------------------------------------
diff --git a/static/js/feeds.js b/static/js/feeds.js
new file mode 100644
index 0000000..c4d71ad
--- /dev/null
+++ b/static/js/feeds.js
@@ -0,0 +1,850 @@
+/*
+ * The feed reader (the server side is the feeds app, /feeds/…): the "Feeds" tab
+ * with two parts that work differently on purpose —
+ * Nachrichten: stories (the same event from several sources is one card),
+ * the "Seit letztem Mal" briefing on top, old news simply goes away;
+ * Blogs: a plain list of unread articles, per blog if wanted.
+ * Articles open in the page (full text from the server), the browser only on request.
+ * Runs in the web page, the desktop app and both Android apps (in diora Reader,
+ * which has no tab bar, it brings a small Bücher/Feeds bar). Chrome-61-safe:
+ * the e-reader app transpiles it (compat61), so no ?. / ?? here.
+ */
+(function () {
+ 'use strict';
+ const tab = document.getElementById('tab-feeds');
+ if (!tab) return;
+ const READER_APP = document.documentElement.classList.contains('nui-reader-app');
+
+ const st = {
+ view: 'news', // news | blogs | manage
+ state: null, // /feeds/state/
+ briefing: null,
+ stories: [], storiesMore: false,
+ category: '',
+ items: [], itemsMore: false,
+ blogSource: '', blogUnread: true,
+ found: null, // discover results
+ opml: null, // parsed OPML import
+ editing: null, // source id being edited
+ loading: false,
+ error: '',
+ };
+ try {
+ const saved = localStorage.getItem('diora_feeds_view');
+ if (saved === 'news' || saved === 'blogs') st.view = saved;
+ } catch (e) {}
+
+ const esc = s => String(s == null ? '' : s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"');
+
+ async function api(path, body) {
+ const opts = {credentials: 'same-origin', headers: {}};
+ if (body !== undefined) {
+ opts.method = 'POST';
+ opts.headers['Content-Type'] = 'application/json';
+ opts.body = JSON.stringify(body);
+ }
+ const r = await fetch(path, opts);
+ let data = null;
+ try { data = await r.json(); } catch (e) {}
+ if (!r.ok) throw new Error((data && data.error) || ('Fehler ' + r.status));
+ return data;
+ }
+
+ // ---- time
+
+ function when(iso) {
+ if (!iso) return '';
+ const d = new Date(iso);
+ const mins = Math.round((Date.now() - d.getTime()) / 60000);
+ if (mins < 1) return 'gerade eben';
+ if (mins < 60) return 'vor ' + mins + ' Min.';
+ if (mins < 6 * 60) return 'vor ' + Math.round(mins / 60) + ' Std.';
+ const hm = ('0' + d.getHours()).slice(-2) + ':' + ('0' + d.getMinutes()).slice(-2);
+ const day = dayLabel(d);
+ return (day === 'Heute' ? '' : day + ', ') + hm;
+ }
+
+ function dayLabel(d) {
+ const today = new Date(); today.setHours(0, 0, 0, 0);
+ const that = new Date(d.getTime()); that.setHours(0, 0, 0, 0);
+ const diff = Math.round((today - that) / 86400000);
+ if (diff === 0) return 'Heute';
+ if (diff === 1) return 'Gestern';
+ return d.toLocaleDateString('de-DE', {weekday: 'short', day: 'numeric', month: 'short'});
+ }
+
+ // ---- loading
+
+ async function loadState() {
+ st.state = await api('/feeds/state/');
+ }
+
+ async function loadNews(more) {
+ const q = [];
+ if (more && st.stories.length) q.push('before=' + encodeURIComponent(st.stories[st.stories.length - 1].last_at));
+ if (st.category) q.push('category=' + encodeURIComponent(st.category));
+ const [news, briefing] = await Promise.all([
+ api('/feeds/news/' + (q.length ? '?' + q.join('&') : '')),
+ more ? Promise.resolve(st.briefing) : api('/feeds/briefing/'),
+ ]);
+ st.stories = more ? st.stories.concat(news.stories) : news.stories;
+ st.storiesMore = news.more;
+ st.briefing = briefing;
+ }
+
+ async function loadBlogs(more) {
+ const q = [];
+ if (st.blogSource) q.push('source=' + st.blogSource);
+ if (st.blogUnread) q.push('unread=1');
+ if (more && st.items.length) q.push('before=' + encodeURIComponent(st.items[st.items.length - 1].published));
+ const data = await api('/feeds/blogs/' + (q.length ? '?' + q.join('&') : ''));
+ st.items = more ? st.items.concat(data.items) : data.items;
+ st.itemsMore = data.more;
+ }
+
+ async function load(more) {
+ st.loading = true;
+ st.error = '';
+ render();
+ try {
+ if (!more) await loadState();
+ if (st.view === 'news') await loadNews(more);
+ else if (st.view === 'blogs') await loadBlogs(more);
+ } catch (e) {
+ st.error = e.message === 'Failed to fetch' ? 'Keine Verbindung.' : e.message;
+ }
+ st.loading = false;
+ render();
+ }
+
+ // ---- rendering
+
+ function sources(kind) {
+ return ((st.state && st.state.sources) || []).filter(s => !kind || s.kind === kind);
+ }
+
+ function render() {
+ const s = st.state;
+ const newsN = s ? s.news_new : 0;
+ const blogN = s ? s.blogs_unread : 0;
+ let body;
+ if (st.view === 'manage') body = renderManage();
+ else if (!s && st.loading) body = '
Lädt …
';
+ else if (st.view === 'news') body = renderNews();
+ else body = renderBlogs();
+ tab.innerHTML =
+ '
' +
+ '
' +
+ (st.view === 'manage'
+ ? '
Feeds verwalten
'
+ : '
' +
+ '' +
+ '' +
+ '
' +
+ '
' +
+ '' +
+ '' +
+ '
') +
+ '
' +
+ (st.error ? '
' + esc(st.error) + '
' : '') +
+ body + '
';
+ }
+
+ function storyMeta(x) {
+ const lead = x.lead || {};
+ const others = x.sources - 1;
+ return esc(lead.source_title) + (others > 0 ? ' +' + others + '' : '') +
+ ' · ' + esc(when(x.last_at));
+ }
+
+ function storyRow(x) {
+ const lead = x.lead || {};
+ return '';
+ }
+
+ function renderBriefing() {
+ const b = st.briefing;
+ if (!b || !b.total) return '';
+ let h = '' +
+ '