diff --git a/feeds/fetch.py b/feeds/fetch.py
index 4b23c75..6963692 100644
--- a/feeds/fetch.py
+++ b/feeds/fetch.py
@@ -68,7 +68,7 @@ def _short(text, n=SUMMARY_CHARS):
def _when(entry):
- """The entry's time (published, else updated), never in the future."""
+ """The entry's time (published, else updated), never in the future, or None."""
now = timezone.now()
for key in ('published_parsed', 'updated_parsed', 'created_parsed'):
t = entry.get(key)
@@ -78,7 +78,7 @@ def _when(entry):
except (OverflowError, ValueError, TypeError):
continue
return min(when, now)
- return now
+ return None
class _FirstImage(HTMLParser):
@@ -131,12 +131,14 @@ def parse_entry(entry, feed_url):
text = plain(content)
summary = plain(summary_html) or text
title = plain(html.unescape(entry.get('title') or '')) or _short(text, 80) or 'Ohne Titel'
+ when = _when(entry)
return {
'guid_hash': guid_hash(entry),
'url': link[:2000],
'title': title[:500],
'author': plain(entry.get('author') or '')[:300],
- 'published': _when(entry),
+ 'published': when or timezone.now(),
+ 'dated': when is not None, # not an Item field: refresh._apply uses and drops it
'summary': _short(summary),
'content': content,
'image_url': _image(entry, content_html, base)[:2000],
diff --git a/feeds/refresh.py b/feeds/refresh.py
index d82796f..ee2cf48 100644
--- a/feeds/refresh.py
+++ b/feeds/refresh.py
@@ -22,7 +22,8 @@ FEED_WORKERS = 12
ARTICLE_WORKERS = 8
ARTICLES_PER_RUN = 80 # articles fetched per refresh at most; the rest when opened
BLOG_KEEP = 1000 # items kept per blog
-BLOG_FIRST_UNREAD_DAYS = 14 # a new blog's older articles start out read
+BLOG_FIRST_UNREAD_DAYS = 7 # a new blog's older articles start out read
+UNDATED_FIRST_UNREAD = 3 # of a new feed without dates, only the first few
GONE_KEEP_DAYS = 30 # expired news rows kept (without text) against re-adding
@@ -68,11 +69,16 @@ def _apply(source, result, now, keep_days):
entries = [fetch.parse_entry(e, source.url) for e in parsed.entries]
# Some feeds (Deutschlandfunk) give every entry the time the feed was made:
# then the dates say nothing, and the time an entry first shows up counts.
- stamps = Counter(f['published'] for f in entries)
- bogus_dates = len(entries) >= 5 and stamps.most_common(1)[0][1] >= 0.8 * len(entries)
- for fields in entries:
- if bogus_dates:
- fields['published'] = now
+ stamps = Counter(f['published'] for f in entries if f['dated'])
+ bogus_dates = len(entries) >= 5 and bool(stamps) and stamps.most_common(1)[0][1] >= 0.8 * len(entries)
+ for n, fields in enumerate(entries):
+ dated = fields.pop('dated')
+ undated = bogus_dates or not dated
+ if undated:
+ # First time: the feed's order stands for the time (newest first), so the
+ # list doesn't fill up with "just now"; afterwards, when it showed up.
+ fields['published'] = now - timedelta(minutes=n) if first else now
+ fields['undated'] = first and n >= UNDATED_FIRST_UNREAD
if fields['guid_hash'] in known:
continue
known.add(fields['guid_hash'])
@@ -80,8 +86,10 @@ def _apply(source, result, now, keep_days):
continue
if oldest_kept and fields['published'] < oldest_kept:
continue
+ old_undated = fields.pop('undated', False)
item = Item(source=source, user=source.user, kind=source.kind, **fields)
- if source.kind == Source.BLOG and first and fields['published'] < now - timedelta(days=BLOG_FIRST_UNREAD_DAYS):
+ if source.kind == Source.BLOG and first and (
+ old_undated or fields['published'] < now - timedelta(days=BLOG_FIRST_UNREAD_DAYS)):
item.read_at = now
new.append(item)
if new:
diff --git a/feeds/tests.py b/feeds/tests.py
index 3f0dbf8..e482caf 100644
--- a/feeds/tests.py
+++ b/feeds/tests.py
@@ -142,6 +142,16 @@ class ApiTest(TestCase):
self.post(f'/feeds/items/{item.id}/star/', {'starred': True})
self.assertEqual(len(self.client.get('/feeds/starred/').json()['items']), 1)
+ def test_blog_folders(self):
+ with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'x', 'title': 'Essay', 'text': LONG}]))):
+ self.post('/feeds/sources/add/', {'url': 'https://a.example/feed', 'kind': 'blog', 'category': 'Philosophie'})
+ with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'y', 'title': 'Code', 'text': LONG}]))):
+ self.post('/feeds/sources/add/', {'url': 'https://b.example/feed', 'kind': 'blog', 'category': ''})
+ self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Philosophie'])
+ self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=Philosophie').json()['items']], ['Essay'])
+ self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=').json()['items']], ['Code'])
+ self.assertIn('text="Blogs – Philosophie"', self.client.get('/feeds/export.opml').content.decode())
+
def test_old_news_expires_and_is_not_taken_in_again(self):
old = (timezone.now() - timedelta(days=3)).strftime('%a, %d %b %Y %H:%M:%S +0000')
feed = rss([{'guid': 'new', 'title': 'Heute passiert', 'text': LONG},
@@ -161,7 +171,16 @@ class ApiTest(TestCase):
stamp = (timezone.now() - timedelta(hours=5)).strftime('%a, %d %b %Y %H:%M:%S +0000')
self.add('news', rss([{'guid': f'g{n}', 'title': f'Meldung {n}', 'text': LONG, 'date': stamp} for n in range(6)]))
for i in Item.objects.filter(user=self.u):
- self.assertLess((timezone.now() - i.published).total_seconds(), 60)
+ self.assertLess((timezone.now() - i.published).total_seconds(), 600) # first fetch: feed order, minutes apart
+
+ def test_new_blog_without_dates_keeps_order_and_few_unread(self):
+ feed = (b'Undatiert'
+ + b''.join(b'u%dPost %dx' % (n, n)
+ for n in range(6)) + b'')
+ self.add('blog', feed)
+ items = list(Item.objects.filter(user=self.u).order_by('-published'))
+ self.assertEqual([i.title for i in items], [f'Post {n}' for n in range(6)])
+ self.assertEqual(sum(1 for i in items if i.read_at is None), 3)
def test_switching_kind_moves_items(self):
src = self.add('news', rss([{'guid': 'a', 'title': 'Etwas', 'text': LONG}]))
@@ -185,7 +204,7 @@ class ApiTest(TestCase):
self.add('blog', rss([]), url='https://blog.example/feed')
r = self.client.get('/feeds/export.opml')
self.assertIn(b'xmlUrl="https://blog.example/feed"', r.content)
- self.assertIn(b'text="Blogs"', r.content)
+ self.assertIn('text="Blogs – Politik"', r.content.decode())
def test_settings(self):
self.post('/feeds/settings/', {'news_keep_days': 5})
diff --git a/feeds/views.py b/feeds/views.py
index 5bb5471..31fccf3 100644
--- a/feeds/views.py
+++ b/feeds/views.py
@@ -12,7 +12,7 @@ News: GET /feeds/briefing/ what's new since the briefing was
POST /feeds/briefing/seen/ seen: everything up to now counts as read
GET /feeds/news/?before=&category= stories, newest first
GET /feeds/stories// one story with all its items
-Blogs: GET /feeds/blogs/?source=&unread=1&before=
+Blogs: GET /feeds/blogs/?folder=&source=&unread=1&before= (a blog's category is its folder)
Items: GET /feeds/items// the article (fetches the page if the feed had a teaser)
POST /feeds/items/read/ {ids, read}
POST /feeds/items//star/ {starred}
@@ -121,6 +121,7 @@ def state(request):
return JsonResponse({
'sources': [_source_data(x, unread.get(x.id, 0)) for x in sources],
'categories': sorted({x.category for x in sources if x.kind == Source.NEWS and x.category}),
+ 'folders': sorted({x.category for x in sources if x.kind == Source.BLOG and x.category}),
'news_new': new_news,
'blogs_unread': sum(n for sid, n in unread.items() if any(x.id == sid and x.kind == Source.BLOG for x in sources)),
'settings': {'news_keep_days': s.news_keep_days, 'news_seen_at': _iso(s.news_seen_at)},
@@ -274,7 +275,8 @@ def export_opml(request):
return err
groups = {}
for s in Source.objects.filter(user=request.user).order_by('kind', 'category', 'title'):
- label = ('Nachrichten' + (f' – {s.category}' if s.category else '')) if s.kind == Source.NEWS else 'Blogs'
+ kind = 'Nachrichten' if s.kind == Source.NEWS else 'Blogs'
+ label = kind + (f' – {s.category}' if s.category else '')
groups.setdefault(label, []).append(s)
lines = ['', '',
'diora Feeds', '']
@@ -378,6 +380,8 @@ def blogs(request):
qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source')
if request.GET.get('source', '').isdigit():
qs = qs.filter(source_id=int(request.GET['source']))
+ elif 'folder' in request.GET:
+ qs = qs.filter(source__category=request.GET['folder'])
if request.GET.get('unread') == '1':
qs = qs.filter(read_at=None)
before = parse_datetime(request.GET.get('before') or '')
diff --git a/static/js/feeds.js b/static/js/feeds.js
index c4d71ad..2d85db2 100644
--- a/static/js/feeds.js
+++ b/static/js/feeds.js
@@ -23,6 +23,7 @@
category: '',
items: [], itemsMore: false,
blogSource: '', blogUnread: true,
+ blogFolder: null, // null = all blogs, '' = blogs without a folder, else a folder name
found: null, // discover results
opml: null, // parsed OPML import
editing: null, // source id being edited
@@ -96,6 +97,7 @@
async function loadBlogs(more) {
const q = [];
if (st.blogSource) q.push('source=' + st.blogSource);
+ else if (st.blogFolder !== null) q.push('folder=' + encodeURIComponent(st.blogFolder));
if (st.blogUnread) q.push('unread=1');
if (more && st.items.length) q.push('before=' + encodeURIComponent(st.items[st.items.length - 1].published));
const data = await api('/feeds/blogs/' + (q.length ? '?' + q.join('&') : ''));
@@ -227,9 +229,23 @@
return '