Feeds: Ordner für Blogs, Blogs ohne Datum in Feed-Reihenfolge

- Die Kategorie eines Blogs ist sein Ordner: Ordner-Chips mit Ungelesen-Zahl in
  der Blog-Ansicht, /feeds/blogs/?folder=, Ordner im OPML-Export und -Import
- Neue Feeds ohne (brauchbare) Daten: die Reihenfolge des Feeds gilt als Zeit,
  nur die ersten 3 ungelesen; ein neuer Blog bringt 7 statt 14 Tage ungelesen
- sw v91

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

feeds/fetch.py

@@ -68,7 +68,7 @@ def _short(text, n=SUMMARY_CHARS):
 
 
 def _when(entry):
-    """The entry's time (published, else updated), never in the future."""
+    """The entry's time (published, else updated), never in the future, or None."""
     now = timezone.now()
     for key in ('published_parsed', 'updated_parsed', 'created_parsed'):
         t = entry.get(key)
@@ -78,7 +78,7 @@ def _when(entry):
             except (OverflowError, ValueError, TypeError):
                 continue
             return min(when, now)
-    return now
+    return None
 
 
 class _FirstImage(HTMLParser):
@@ -131,12 +131,14 @@ def parse_entry(entry, feed_url):
     text = plain(content)
     summary = plain(summary_html) or text
     title = plain(html.unescape(entry.get('title') or '')) or _short(text, 80) or 'Ohne Titel'
+    when = _when(entry)
     return {
         'guid_hash': guid_hash(entry),
         'url': link[:2000],
         'title': title[:500],
         'author': plain(entry.get('author') or '')[:300],
-        'published': _when(entry),
+        'published': when or timezone.now(),
+        'dated': when is not None,  # not an Item field: refresh._apply uses and drops it
         'summary': _short(summary),
         'content': content,
         'image_url': _image(entry, content_html, base)[:2000],

feeds/refresh.py

@@ -22,7 +22,8 @@ FEED_WORKERS = 12
 ARTICLE_WORKERS = 8
 ARTICLES_PER_RUN = 80        # articles fetched per refresh at most; the rest when opened
 BLOG_KEEP = 1000             # items kept per blog
-BLOG_FIRST_UNREAD_DAYS = 14  # a new blog's older articles start out read
+BLOG_FIRST_UNREAD_DAYS = 7   # a new blog's older articles start out read
+UNDATED_FIRST_UNREAD = 3     # of a new feed without dates, only the first few
 GONE_KEEP_DAYS = 30          # expired news rows kept (without text) against re-adding
 
 
@@ -68,11 +69,16 @@ def _apply(source, result, now, keep_days):
     entries = [fetch.parse_entry(e, source.url) for e in parsed.entries]
     # Some feeds (Deutschlandfunk) give every entry the time the feed was made:
     # then the dates say nothing, and the time an entry first shows up counts.
-    stamps = Counter(f['published'] for f in entries)
-    bogus_dates = len(entries) >= 5 and stamps.most_common(1)[0][1] >= 0.8 * len(entries)
-    for fields in entries:
-        if bogus_dates:
-            fields['published'] = now
+    stamps = Counter(f['published'] for f in entries if f['dated'])
+    bogus_dates = len(entries) >= 5 and bool(stamps) and stamps.most_common(1)[0][1] >= 0.8 * len(entries)
+    for n, fields in enumerate(entries):
+        dated = fields.pop('dated')
+        undated = bogus_dates or not dated
+        if undated:
+            # First time: the feed's order stands for the time (newest first), so the
+            # list doesn't fill up with "just now"; afterwards, when it showed up.
+            fields['published'] = now - timedelta(minutes=n) if first else now
+            fields['undated'] = first and n >= UNDATED_FIRST_UNREAD
         if fields['guid_hash'] in known:
             continue
         known.add(fields['guid_hash'])
@@ -80,8 +86,10 @@ def _apply(source, result, now, keep_days):
             continue
         if oldest_kept and fields['published'] < oldest_kept:
             continue
+        old_undated = fields.pop('undated', False)
         item = Item(source=source, user=source.user, kind=source.kind, **fields)
-        if source.kind == Source.BLOG and first and fields['published'] < now - timedelta(days=BLOG_FIRST_UNREAD_DAYS):
+        if source.kind == Source.BLOG and first and (
+                old_undated or fields['published'] < now - timedelta(days=BLOG_FIRST_UNREAD_DAYS)):
             item.read_at = now
         new.append(item)
     if new:

feeds/tests.py

@@ -142,6 +142,16 @@ class ApiTest(TestCase):
         self.post(f'/feeds/items/{item.id}/star/', {'starred': True})
         self.assertEqual(len(self.client.get('/feeds/starred/').json()['items']), 1)
 
+    def test_blog_folders(self):
+        with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'x', 'title': 'Essay', 'text': LONG}]))):
+            self.post('/feeds/sources/add/', {'url': 'https://a.example/feed', 'kind': 'blog', 'category': 'Philosophie'})
+        with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'y', 'title': 'Code', 'text': LONG}]))):
+            self.post('/feeds/sources/add/', {'url': 'https://b.example/feed', 'kind': 'blog', 'category': ''})
+        self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Philosophie'])
+        self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=Philosophie').json()['items']], ['Essay'])
+        self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=').json()['items']], ['Code'])
+        self.assertIn('text="Blogs – Philosophie"', self.client.get('/feeds/export.opml').content.decode())
+
     def test_old_news_expires_and_is_not_taken_in_again(self):
         old = (timezone.now() - timedelta(days=3)).strftime('%a, %d %b %Y %H:%M:%S +0000')
         feed = rss([{'guid': 'new', 'title': 'Heute passiert', 'text': LONG},
@@ -161,7 +171,16 @@ class ApiTest(TestCase):
         stamp = (timezone.now() - timedelta(hours=5)).strftime('%a, %d %b %Y %H:%M:%S +0000')
         self.add('news', rss([{'guid': f'g{n}', 'title': f'Meldung {n}', 'text': LONG, 'date': stamp} for n in range(6)]))
         for i in Item.objects.filter(user=self.u):
-            self.assertLess((timezone.now() - i.published).total_seconds(), 60)
+            self.assertLess((timezone.now() - i.published).total_seconds(), 600)  # first fetch: feed order, minutes apart
+
+    def test_new_blog_without_dates_keeps_order_and_few_unread(self):
+        feed = (b'<?xml version="1.0"?><rss version="2.0"><channel><title>Undatiert</title>'
+                + b''.join(b'<item><guid>u%d</guid><title>Post %d</title><description>x</description></item>' % (n, n)
+                           for n in range(6)) + b'</channel></rss>')
+        self.add('blog', feed)
+        items = list(Item.objects.filter(user=self.u).order_by('-published'))
+        self.assertEqual([i.title for i in items], [f'Post {n}' for n in range(6)])
+        self.assertEqual(sum(1 for i in items if i.read_at is None), 3)
 
     def test_switching_kind_moves_items(self):
         src = self.add('news', rss([{'guid': 'a', 'title': 'Etwas', 'text': LONG}]))
@@ -185,7 +204,7 @@ class ApiTest(TestCase):
         self.add('blog', rss([]), url='https://blog.example/feed')
         r = self.client.get('/feeds/export.opml')
         self.assertIn(b'xmlUrl="https://blog.example/feed"', r.content)
-        self.assertIn(b'text="Blogs"', r.content)
+        self.assertIn('text="Blogs – Politik"', r.content.decode())
 
     def test_settings(self):
         self.post('/feeds/settings/', {'news_keep_days': 5})

feeds/views.py

@@ -12,7 +12,7 @@ News:      GET  /feeds/briefing/              what's new since the briefing was
            POST /feeds/briefing/seen/         seen: everything up to now counts as read
            GET  /feeds/news/?before=&category=  stories, newest first
            GET  /feeds/stories/<id>/          one story with all its items
-Blogs:     GET  /feeds/blogs/?source=&unread=1&before=
+Blogs:     GET  /feeds/blogs/?folder=&source=&unread=1&before=   (a blog's category is its folder)
 Items:     GET  /feeds/items/<id>/            the article (fetches the page if the feed had a teaser)
            POST /feeds/items/read/            {ids, read}
            POST /feeds/items/<id>/star/       {starred}
@@ -121,6 +121,7 @@ def state(request):
     return JsonResponse({
         'sources': [_source_data(x, unread.get(x.id, 0)) for x in sources],
         'categories': sorted({x.category for x in sources if x.kind == Source.NEWS and x.category}),
+        'folders': sorted({x.category for x in sources if x.kind == Source.BLOG and x.category}),
         'news_new': new_news,
         'blogs_unread': sum(n for sid, n in unread.items() if any(x.id == sid and x.kind == Source.BLOG for x in sources)),
         'settings': {'news_keep_days': s.news_keep_days, 'news_seen_at': _iso(s.news_seen_at)},
@@ -274,7 +275,8 @@ def export_opml(request):
         return err
     groups = {}
     for s in Source.objects.filter(user=request.user).order_by('kind', 'category', 'title'):
-        label = ('Nachrichten' + (f' – {s.category}' if s.category else '')) if s.kind == Source.NEWS else 'Blogs'
+        kind = 'Nachrichten' if s.kind == Source.NEWS else 'Blogs'
+        label = kind + (f' – {s.category}' if s.category else '')
         groups.setdefault(label, []).append(s)
     lines = ['<?xml version="1.0" encoding="UTF-8"?>', '<opml version="2.0">',
              '<head><title>diora Feeds</title></head>', '<body>']
@@ -378,6 +380,8 @@ def blogs(request):
     qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source')
     if request.GET.get('source', '').isdigit():
         qs = qs.filter(source_id=int(request.GET['source']))
+    elif 'folder' in request.GET:
+        qs = qs.filter(source__category=request.GET['folder'])
     if request.GET.get('unread') == '1':
         qs = qs.filter(read_at=None)
     before = parse_datetime(request.GET.get('before') or '')

static/js/feeds.js

@@ -23,6 +23,7 @@
     category: '',
     items: [], itemsMore: false,
     blogSource: '', blogUnread: true,
+    blogFolder: null,      // null = all blogs, '' = blogs without a folder, else a folder name
     found: null,           // discover results
     opml: null,            // parsed OPML import
     editing: null,         // source id being edited
@@ -96,6 +97,7 @@
   async function loadBlogs(more) {
     const q = [];
     if (st.blogSource) q.push('source=' + st.blogSource);
+    else if (st.blogFolder !== null) q.push('folder=' + encodeURIComponent(st.blogFolder));
     if (st.blogUnread) q.push('unread=1');
     if (more && st.items.length) q.push('before=' + encodeURIComponent(st.items[st.items.length - 1].published));
     const data = await api('/feeds/blogs/' + (q.length ? '?' + q.join('&') : ''));
@@ -227,9 +229,23 @@
       return '<div class="fd-empty"><p>Noch keine Blogs abonniert.</p>' +
         '<button type="button" class="btn btn-primary" data-act="manage">Feed hinzufügen</button></div>';
     }
-    let h = '<div class="fd-filter">' +
-      '<select class="search-input fd-select" data-act="blog-source"><option value="">Alle Blogs</option>' +
-      blogs.map(b => '<option value="' + b.id + '"' + (String(b.id) === String(st.blogSource) ? ' selected' : '') + '>' +
+    // Folders (a blog's category): chips with their unread count.
+    const folders = (st.state.folders || []);
+    const unreadIn = f => blogs.filter(b => (b.category || '') === f).reduce((n, b) => n + b.unread, 0);
+    const total = blogs.reduce((n, b) => n + b.unread, 0);
+    let h = '';
+    if (folders.length) {
+      const chip = (value, label, n) => '<button type="button" class="fd-chip' + (st.blogFolder === value ? ' on' : '') +
+        '" data-folder="' + (value === null ? '*' : esc(value)) + '">' + esc(label) + (n ? ' <small>' + n + '</small>' : '') + '</button>';
+      h += '<div class="fd-chips">' + chip(null, 'Alle', total) +
+        folders.map(f => chip(f, f, unreadIn(f))).join('') +
+        (blogs.some(b => !b.category) ? chip('', 'Ohne Ordner', unreadIn('')) : '') + '</div>';
+    }
+    const shown = st.blogFolder === null ? blogs : blogs.filter(b => (b.category || '') === st.blogFolder);
+    h += '<div class="fd-filter">' +
+      '<select class="search-input fd-select" data-act="blog-source"><option value="">' +
+      (st.blogFolder === null ? 'Alle Blogs' : 'Alle in ' + esc(st.blogFolder || 'Ohne Ordner')) + '</option>' +
+      shown.map(b => '<option value="' + b.id + '"' + (String(b.id) === String(st.blogSource) ? ' selected' : '') + '>' +
         esc(b.title) + (b.unread ? ' (' + b.unread + ')' : '') + '</option>').join('') + '</select>' +
       '<label class="fd-check"><input type="checkbox" data-act="blog-unread"' + (st.blogUnread ? ' checked' : '') + '> Nur ungelesene</label>' +
       (st.items.some(i => !i.read) ? '<button type="button" class="btn fd-allread" data-act="blogs-read">Alle gelesen</button>' : '') +
@@ -252,8 +268,8 @@
   }
 
   function categoryInput(value) {
-    const cats = (st.state && st.state.categories) || [];
-    return '<input class="search-input" name="category" list="fd-cats" placeholder="Ressort, z. B. Politik" value="' + esc(value || '') + '">' +
+    const cats = ((st.state && st.state.categories) || []).concat((st.state && st.state.folders) || []);
+    return '<input class="search-input" name="category" list="fd-cats" placeholder="z. B. Politik / Philosophie" value="' + esc(value || '') + '">' +
       '<datalist id="fd-cats">' + cats.map(c => '<option value="' + esc(c) + '">').join('') + '</datalist>';
   }
 
@@ -270,7 +286,7 @@
         (f.samples.length ? '<ul class="fd-samples">' + f.samples.slice(0, 3).map(t => '<li>' + esc(t) + '</li>').join('') + '</ul>' : '') +
         (f.subscribed ? '<p class="fd-note">Schon abonniert.</p>' :
           '<div class="fd-grid"><label>Art ' + kindSelect('kind', f.suggested_kind) + '</label>' +
-          '<label class="fd-cat-field">Ressort (Nachrichten) ' + categoryInput('') + '</label></div>' +
+          '<label class="fd-cat-field">' + (f.suggested_kind === 'news' ? 'Ressort' : 'Ordner') + ' ' + categoryInput('') + '</label></div>' +
           '<button class="btn btn-primary" type="submit">Abonnieren</button>') +
         '</form>').join('');
     }
@@ -283,12 +299,12 @@
       list.map(x => x.id === st.editing ? editForm(x) :
         '<button type="button" class="fd-src" data-edit="' + x.id + '">' +
         '<span class="fd-src-title">' + esc(x.title) + '</span>' +
-        '<span class="fd-row-meta">' + (x.kind === 'news' ? esc(x.category || 'Allgemein') + ' · ' : '') +
+        '<span class="fd-row-meta">' + (x.kind === 'news' ? esc(x.category || 'Allgemein') + ' · ' : (x.category ? esc(x.category) + ' · ' : '')) +
         (x.unread ? x.unread + ' ungelesen' : '') +
         (x.error_count >= 3 ? ' <span class="fd-warn" title="' + esc(x.last_error) + '">⚠ Abruf schlägt fehl</span>' : '') +
         '</span></button>').join('') + '</section>';
     h += group('Nachrichten', news.sort((a, b) => (a.category || '').localeCompare(b.category || '') || a.title.localeCompare(b.title)));
-    h += group('Blogs', blogs);
+    h += group('Blogs', blogs.sort((a, b) => (a.category || '~').localeCompare(b.category || '~') || a.title.localeCompare(b.title)));
 
     h += '<section class="fd-box"><h3>Import &amp; Einstellungen</h3>' +
       '<label class="btn fd-file">OPML importieren …<input type="file" accept=".opml,.xml,text/xml,text/x-opml" data-act="opml" hidden></label> ' +
@@ -307,7 +323,7 @@
       '<div class="fd-note fd-url">' + esc(x.url) + '</div>' +
       (x.last_error ? '<p class="fd-warn">Letzter Fehler: ' + esc(x.last_error) + '</p>' : '') +
       '<div class="fd-grid"><label>Art ' + kindSelect('kind', x.kind) + '</label>' +
-      '<label class="fd-cat-field">Ressort ' + categoryInput(x.category) + '</label>' +
+      '<label class="fd-cat-field">' + (x.kind === 'news' ? 'Ressort' : 'Ordner') + ' ' + categoryInput(x.category) + '</label>' +
       '<label>Ganzer Artikel <select class="search-input" name="fulltext">' +
       [['auto', 'Automatisch'], ['always', 'Immer holen'], ['never', 'Nur Feed-Text']].map(o =>
         '<option value="' + o[0] + '"' + (x.fulltext === o[0] ? ' selected' : '') + '>' + o[1] + '</option>').join('') +
@@ -514,7 +530,7 @@
   // ---- events
 
   tab.addEventListener('click', async e => {
-    const t = e.target.closest('[data-view],[data-act],[data-story],[data-item],[data-cat],[data-edit]');
+    const t = e.target.closest('[data-view],[data-act],[data-story],[data-item],[data-cat],[data-edit],[data-folder]');
     if (!t || t.tagName === 'SELECT' || (t.tagName === 'INPUT' && t.type !== 'button')) return;
     if (t.dataset.view) {
       st.view = t.dataset.view;
@@ -522,6 +538,12 @@
       load();
       return;
     }
+    if (t.dataset.folder !== undefined) {
+      st.blogFolder = t.dataset.folder === '*' ? null : t.dataset.folder;
+      st.blogSource = '';
+      window.scrollTo(0, 0);
+      return load();
+    }
     if (t.dataset.story) return openStory(parseInt(t.dataset.story, 10));
     if (t.dataset.item) {
       readerList = st.items;
@@ -592,7 +614,7 @@
       const list = [];
       st.opml.forEach(g => {
         if (g.kind === 'skip') return;
-        g.feeds.forEach(f => list.push({url: f.url, title: f.title, kind: g.kind, category: g.kind === 'news' ? g.name : ''}));
+        g.feeds.forEach(f => list.push({url: f.url, title: f.title, kind: g.kind, category: g.name}));
       });
       t.disabled = true;
       t.textContent = 'Wird übernommen …';
@@ -653,7 +675,7 @@
         const found = st.found[parseInt(f.dataset.n, 10)];
         if (btn) btn.textContent = 'Lädt Artikel …';
         await api('/feeds/sources/add/', {url: found.url, title: found.title, kind: f.kind.value,
-          category: f.kind.value === 'news' ? f.category.value : ''});
+          category: f.category.value});
         found.subscribed = true;
         await loadState();
       } else if (f.dataset.form === 'edit') {
@@ -669,12 +691,12 @@
     render();
   });
 
-  // The Ressort field only means something for news.
+  // The same field is the Ressort of news and the folder of a blog.
   tab.addEventListener('input', e => {
     if (e.target.name !== 'kind') return;
     const form = e.target.form;
     const cat = form && form.querySelector('.fd-cat-field');
-    if (cat) cat.style.display = e.target.value === 'news' ? '' : 'none';
+    if (cat && cat.firstChild) cat.firstChild.nodeValue = (e.target.value === 'news' ? 'Ressort' : 'Ordner') + ' ';
   });
 
   // ---- showing the tab
@@ -790,6 +812,7 @@
     '#fd-article .fd-other-title { font-weight: 600; line-height: 1.35; }',
     '.fd-chip { padding: 6px 12px; border: 1px solid var(--border, #2a2a2a); border-radius: 16px; background: none; color: inherit; font: inherit; font-size: 13px; cursor: pointer; }',
     '.fd-chip.on { border-color: var(--accent, #e63946); color: var(--accent, #e63946); }',
+    '.fd-chip small { color: var(--nui-muted, #9a9a9a); font-size: 11px; }',
     '#tab-feeds .fd-day { margin-top: 16px; }',
     '#tab-feeds .fd-card { display: flex; gap: 12px; width: 100%; padding: 12px 0; background: none; border: none; border-bottom: 1px solid var(--border, #2a2a2a); color: inherit; font: inherit; text-align: left; cursor: pointer; }',
     '#tab-feeds .fd-thumb { width: 92px; height: 66px; flex: none; object-fit: cover; border-radius: 4px; background: #1a1a1a; order: 2; }',

static/js/sw.js

@@ -2,7 +2,7 @@
  * diora service worker — caches the app shell for offline use.
  */
 
-const CACHE = 'diora-v90';
+const CACHE = 'diora-v91';
 const SHELL = [
   '/static/css/app.css',
   '/static/js/app.js',