diff --git a/feeds/fetch.py b/feeds/fetch.py index 4b23c75..6963692 100644 --- a/feeds/fetch.py +++ b/feeds/fetch.py @@ -68,7 +68,7 @@ def _short(text, n=SUMMARY_CHARS): def _when(entry): - """The entry's time (published, else updated), never in the future.""" + """The entry's time (published, else updated), never in the future, or None.""" now = timezone.now() for key in ('published_parsed', 'updated_parsed', 'created_parsed'): t = entry.get(key) @@ -78,7 +78,7 @@ def _when(entry): except (OverflowError, ValueError, TypeError): continue return min(when, now) - return now + return None class _FirstImage(HTMLParser): @@ -131,12 +131,14 @@ def parse_entry(entry, feed_url): text = plain(content) summary = plain(summary_html) or text title = plain(html.unescape(entry.get('title') or '')) or _short(text, 80) or 'Ohne Titel' + when = _when(entry) return { 'guid_hash': guid_hash(entry), 'url': link[:2000], 'title': title[:500], 'author': plain(entry.get('author') or '')[:300], - 'published': _when(entry), + 'published': when or timezone.now(), + 'dated': when is not None, # not an Item field: refresh._apply uses and drops it 'summary': _short(summary), 'content': content, 'image_url': _image(entry, content_html, base)[:2000], diff --git a/feeds/refresh.py b/feeds/refresh.py index d82796f..ee2cf48 100644 --- a/feeds/refresh.py +++ b/feeds/refresh.py @@ -22,7 +22,8 @@ FEED_WORKERS = 12 ARTICLE_WORKERS = 8 ARTICLES_PER_RUN = 80 # articles fetched per refresh at most; the rest when opened BLOG_KEEP = 1000 # items kept per blog -BLOG_FIRST_UNREAD_DAYS = 14 # a new blog's older articles start out read +BLOG_FIRST_UNREAD_DAYS = 7 # a new blog's older articles start out read +UNDATED_FIRST_UNREAD = 3 # of a new feed without dates, only the first few GONE_KEEP_DAYS = 30 # expired news rows kept (without text) against re-adding @@ -68,11 +69,16 @@ def _apply(source, result, now, keep_days): entries = [fetch.parse_entry(e, source.url) for e in parsed.entries] # Some feeds (Deutschlandfunk) give every entry the time the feed was made: # then the dates say nothing, and the time an entry first shows up counts. - stamps = Counter(f['published'] for f in entries) - bogus_dates = len(entries) >= 5 and stamps.most_common(1)[0][1] >= 0.8 * len(entries) - for fields in entries: - if bogus_dates: - fields['published'] = now + stamps = Counter(f['published'] for f in entries if f['dated']) + bogus_dates = len(entries) >= 5 and bool(stamps) and stamps.most_common(1)[0][1] >= 0.8 * len(entries) + for n, fields in enumerate(entries): + dated = fields.pop('dated') + undated = bogus_dates or not dated + if undated: + # First time: the feed's order stands for the time (newest first), so the + # list doesn't fill up with "just now"; afterwards, when it showed up. + fields['published'] = now - timedelta(minutes=n) if first else now + fields['undated'] = first and n >= UNDATED_FIRST_UNREAD if fields['guid_hash'] in known: continue known.add(fields['guid_hash']) @@ -80,8 +86,10 @@ def _apply(source, result, now, keep_days): continue if oldest_kept and fields['published'] < oldest_kept: continue + old_undated = fields.pop('undated', False) item = Item(source=source, user=source.user, kind=source.kind, **fields) - if source.kind == Source.BLOG and first and fields['published'] < now - timedelta(days=BLOG_FIRST_UNREAD_DAYS): + if source.kind == Source.BLOG and first and ( + old_undated or fields['published'] < now - timedelta(days=BLOG_FIRST_UNREAD_DAYS)): item.read_at = now new.append(item) if new: diff --git a/feeds/tests.py b/feeds/tests.py index 3f0dbf8..e482caf 100644 --- a/feeds/tests.py +++ b/feeds/tests.py @@ -142,6 +142,16 @@ class ApiTest(TestCase): self.post(f'/feeds/items/{item.id}/star/', {'starred': True}) self.assertEqual(len(self.client.get('/feeds/starred/').json()['items']), 1) + def test_blog_folders(self): + with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'x', 'title': 'Essay', 'text': LONG}]))): + self.post('/feeds/sources/add/', {'url': 'https://a.example/feed', 'kind': 'blog', 'category': 'Philosophie'}) + with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'y', 'title': 'Code', 'text': LONG}]))): + self.post('/feeds/sources/add/', {'url': 'https://b.example/feed', 'kind': 'blog', 'category': ''}) + self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Philosophie']) + self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=Philosophie').json()['items']], ['Essay']) + self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=').json()['items']], ['Code']) + self.assertIn('text="Blogs – Philosophie"', self.client.get('/feeds/export.opml').content.decode()) + def test_old_news_expires_and_is_not_taken_in_again(self): old = (timezone.now() - timedelta(days=3)).strftime('%a, %d %b %Y %H:%M:%S +0000') feed = rss([{'guid': 'new', 'title': 'Heute passiert', 'text': LONG}, @@ -161,7 +171,16 @@ class ApiTest(TestCase): stamp = (timezone.now() - timedelta(hours=5)).strftime('%a, %d %b %Y %H:%M:%S +0000') self.add('news', rss([{'guid': f'g{n}', 'title': f'Meldung {n}', 'text': LONG, 'date': stamp} for n in range(6)])) for i in Item.objects.filter(user=self.u): - self.assertLess((timezone.now() - i.published).total_seconds(), 60) + self.assertLess((timezone.now() - i.published).total_seconds(), 600) # first fetch: feed order, minutes apart + + def test_new_blog_without_dates_keeps_order_and_few_unread(self): + feed = (b'Undatiert' + + b''.join(b'u%dPost %dx' % (n, n) + for n in range(6)) + b'') + self.add('blog', feed) + items = list(Item.objects.filter(user=self.u).order_by('-published')) + self.assertEqual([i.title for i in items], [f'Post {n}' for n in range(6)]) + self.assertEqual(sum(1 for i in items if i.read_at is None), 3) def test_switching_kind_moves_items(self): src = self.add('news', rss([{'guid': 'a', 'title': 'Etwas', 'text': LONG}])) @@ -185,7 +204,7 @@ class ApiTest(TestCase): self.add('blog', rss([]), url='https://blog.example/feed') r = self.client.get('/feeds/export.opml') self.assertIn(b'xmlUrl="https://blog.example/feed"', r.content) - self.assertIn(b'text="Blogs"', r.content) + self.assertIn('text="Blogs – Politik"', r.content.decode()) def test_settings(self): self.post('/feeds/settings/', {'news_keep_days': 5}) diff --git a/feeds/views.py b/feeds/views.py index 5bb5471..31fccf3 100644 --- a/feeds/views.py +++ b/feeds/views.py @@ -12,7 +12,7 @@ News: GET /feeds/briefing/ what's new since the briefing was POST /feeds/briefing/seen/ seen: everything up to now counts as read GET /feeds/news/?before=&category= stories, newest first GET /feeds/stories// one story with all its items -Blogs: GET /feeds/blogs/?source=&unread=1&before= +Blogs: GET /feeds/blogs/?folder=&source=&unread=1&before= (a blog's category is its folder) Items: GET /feeds/items// the article (fetches the page if the feed had a teaser) POST /feeds/items/read/ {ids, read} POST /feeds/items//star/ {starred} @@ -121,6 +121,7 @@ def state(request): return JsonResponse({ 'sources': [_source_data(x, unread.get(x.id, 0)) for x in sources], 'categories': sorted({x.category for x in sources if x.kind == Source.NEWS and x.category}), + 'folders': sorted({x.category for x in sources if x.kind == Source.BLOG and x.category}), 'news_new': new_news, 'blogs_unread': sum(n for sid, n in unread.items() if any(x.id == sid and x.kind == Source.BLOG for x in sources)), 'settings': {'news_keep_days': s.news_keep_days, 'news_seen_at': _iso(s.news_seen_at)}, @@ -274,7 +275,8 @@ def export_opml(request): return err groups = {} for s in Source.objects.filter(user=request.user).order_by('kind', 'category', 'title'): - label = ('Nachrichten' + (f' – {s.category}' if s.category else '')) if s.kind == Source.NEWS else 'Blogs' + kind = 'Nachrichten' if s.kind == Source.NEWS else 'Blogs' + label = kind + (f' – {s.category}' if s.category else '') groups.setdefault(label, []).append(s) lines = ['', '', 'diora Feeds', ''] @@ -378,6 +380,8 @@ def blogs(request): qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source') if request.GET.get('source', '').isdigit(): qs = qs.filter(source_id=int(request.GET['source'])) + elif 'folder' in request.GET: + qs = qs.filter(source__category=request.GET['folder']) if request.GET.get('unread') == '1': qs = qs.filter(read_at=None) before = parse_datetime(request.GET.get('before') or '') diff --git a/static/js/feeds.js b/static/js/feeds.js index c4d71ad..2d85db2 100644 --- a/static/js/feeds.js +++ b/static/js/feeds.js @@ -23,6 +23,7 @@ category: '', items: [], itemsMore: false, blogSource: '', blogUnread: true, + blogFolder: null, // null = all blogs, '' = blogs without a folder, else a folder name found: null, // discover results opml: null, // parsed OPML import editing: null, // source id being edited @@ -96,6 +97,7 @@ async function loadBlogs(more) { const q = []; if (st.blogSource) q.push('source=' + st.blogSource); + else if (st.blogFolder !== null) q.push('folder=' + encodeURIComponent(st.blogFolder)); if (st.blogUnread) q.push('unread=1'); if (more && st.items.length) q.push('before=' + encodeURIComponent(st.items[st.items.length - 1].published)); const data = await api('/feeds/blogs/' + (q.length ? '?' + q.join('&') : '')); @@ -227,9 +229,23 @@ return '

Noch keine Blogs abonniert.

' + '
'; } - let h = '
' + - '' + + shown.map(b => '').join('') + '' + '' + (st.items.some(i => !i.read) ? '' : '') + @@ -252,8 +268,8 @@ } function categoryInput(value) { - const cats = (st.state && st.state.categories) || []; - return '' + + const cats = ((st.state && st.state.categories) || []).concat((st.state && st.state.folders) || []); + return '' + '' + cats.map(c => ''; } @@ -270,7 +286,7 @@ (f.samples.length ? '
    ' + f.samples.slice(0, 3).map(t => '
  • ' + esc(t) + '
  • ').join('') + '
' : '') + (f.subscribed ? '

Schon abonniert.

' : '
' + - '
' + + '
' + '') + '').join(''); } @@ -283,12 +299,12 @@ list.map(x => x.id === st.editing ? editForm(x) : '').join('') + ''; h += group('Nachrichten', news.sort((a, b) => (a.category || '').localeCompare(b.category || '') || a.title.localeCompare(b.title))); - h += group('Blogs', blogs); + h += group('Blogs', blogs.sort((a, b) => (a.category || '~').localeCompare(b.category || '~') || a.title.localeCompare(b.title))); h += '

Import & Einstellungen

' + ' ' + @@ -307,7 +323,7 @@ '
' + esc(x.url) + '
' + (x.last_error ? '

Letzter Fehler: ' + esc(x.last_error) + '

' : '') + '
' + - '' + + '' + '