Video-Podcasts: Folgen mit Video-Enclosure werden aufgenommen (abgespielt wird der Ton)

Audio bleibt bevorzugt, wenn eine Folge beides hat; Endungen werden am
Pfad erkannt (Query-Strings stören nicht mehr). refresh_feeds --full
liest alle Feeds ohne ETag neu ein; dabei gefundene Folgen, die vor dem
letzten Abruf erschienen sind, kommen in die Liste, nicht nach „Neu“.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
mrwnslz 2026-10-02 11:58:24 +02:00
parent 35ea42b6c2
commit 5b2d25e2b2
3 changed files with 107 additions and 15 deletions

View file

@ -11,6 +11,11 @@ class Command(BaseCommand):
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID (even when backing off)') parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID (even when backing off)')
parser.add_argument('--limit', type=int, default=0, help='Max number of feeds to refresh') parser.add_argument('--limit', type=int, default=0, help='Max number of feeds to refresh')
parser.add_argument(
'--full', action='store_true',
help='Download every feed in full (ignore ETag/Last-Modified) and take in episodes skipped before '
'(e.g. video ones); those out before the last refresh go into the list, not into "Neu"',
)
def handle(self, *args, **options): def handle(self, *args, **options):
qs = PodcastFeed.objects.all().order_by('last_refreshed_at') qs = PodcastFeed.objects.all().order_by('last_refreshed_at')
@ -30,5 +35,5 @@ class Command(BaseCommand):
return return
# Parallel downloads, one per distinct feed URL (see _refresh_feeds). # Parallel downloads, one per distinct feed URL (see _refresh_feeds).
refreshed, failed, new = _refresh_feeds(qs) refreshed, failed, new = _refresh_feeds(qs, full=options['full'])
self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes') self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes')

View file

@ -223,6 +223,62 @@ class FirstFetchTests(TestCase):
self.assertEqual([e['title'] for e in inbox], ['e7', 'e6', 'e5', 'e4']) self.assertEqual([e['title'] for e in inbox], ['e7', 'e6', 'e5', 'e4'])
class VideoEnclosureTests(TestCase):
"""Video podcasts are taken in (the players play their sound only)."""
def setUp(self):
self.user = User.objects.create_user(username='alice', password='pw12345678')
self.client.force_login(self.user)
self.feed = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/f', title='Feed')
@staticmethod
def rss(*items):
entries = ''.join(
f'<item><guid>{guid}</guid><title>{guid}</title><pubDate>{date}</pubDate>{enclosures}</item>'
for guid, date, enclosures in items
)
return f'<?xml version="1.0"?><rss version="2.0"><channel><title>Feed</title>{entries}</channel></rss>'
def refresh(self, rss, **kw):
with patch('podcasts.views._fetch_feed', lambda url, **_: _ok(rss)):
return _refresh_feed(self.feed, **kw)
def test_video_only_and_audio_preferred(self):
self.refresh(self.rss(
('v', 'Mon, 05 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/v.mp4" type="video/mp4" length="1"/>'),
('both', 'Tue, 06 Jan 2026 10:00:00 GMT',
'<enclosure url="https://example.org/b.mp4" type="video/mp4" length="1"/>'
'<enclosure url="https://example.org/b.m4a" type="audio/mp4" length="1"/>'),
('untyped', 'Wed, 07 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/u.m4v?x=1" length="1"/>'),
('none', 'Thu, 08 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/n.pdf" type="application/pdf"/>'),
))
urls = dict(PodcastEpisode.objects.filter(feed=self.feed).values_list('guid', 'audio_url'))
self.assertEqual(urls, {
'v': 'https://example.org/v.mp4',
'both': 'https://example.org/b.m4a',
'untyped': 'https://example.org/u.m4v?x=1',
})
def test_full_reread_keeps_old_found_episodes_out_of_the_inbox(self):
audio = '<enclosure url="https://example.org/a.mp3" type="audio/mpeg" length="1"/>'
video = '<enclosure url="https://example.org/{}.mp4" type="video/mp4" length="1"/>'
self.refresh(self.rss(('a', 'Mon, 05 Jan 2026 10:00:00 GMT', audio)))
self.feed.refresh_from_db()
later = (self.feed.last_refreshed_at + timedelta(days=1)).strftime('%a, %d %b %Y %H:%M:%S GMT')
self.refresh(self.rss(
('a', 'Mon, 05 Jan 2026 10:00:00 GMT', audio),
('old-video', 'Tue, 06 Jan 2026 10:00:00 GMT', video.format('o')),
('new-video', later, video.format('n')),
), full=True)
inbox = [e['title'] for e in self.client.get('/podcasts/inbox/').json()['episodes']]
self.assertIn('new-video', inbox)
self.assertNotIn('old-video', inbox)
old = PodcastEpisode.objects.get(feed=self.feed, guid='old-video')
progress = EpisodeProgress.objects.get(user=self.user, episode=old)
self.assertTrue(progress.dismissed)
self.assertFalse(progress.played)
class RefreshAllTests(TestCase): class RefreshAllTests(TestCase):
"""One request refreshes all feeds on the server; one request returns the state.""" """One request refreshes all feeds on the server; one request returns the state."""

View file

@ -408,10 +408,17 @@ def _in_backoff_q(now=None):
return q return q
def _refresh_feeds(feeds, chunk=32): # File endings of an enclosure without a usable type.
AUDIO_EXTENSIONS = ('.mp3', '.m4a', '.m4b', '.ogg', '.oga', '.opus', '.aac', '.flac', '.wav')
VIDEO_EXTENSIONS = ('.mp4', '.m4v', '.mov', '.webm')
def _refresh_feeds(feeds, chunk=32, full=False):
"""Refresh many feeds at once: the downloads run in parallel and each """Refresh many feeds at once: the downloads run in parallel and each
feed URL is fetched once, even when several users subscribed to it; the feed URL is fetched once, even when several users subscribed to it; the
database writes then happen one by one. Returns (refreshed, failed, new).""" database writes then happen one by one. Returns (refreshed, failed, new).
`full`: download every feed again (no 304) and take in episodes the
reader skipped before (see _apply_feed)."""
feeds = list(feeds) feeds = list(feeds)
refreshed = failed = new = 0 refreshed = failed = new = 0
for start in range(0, len(feeds), chunk): for start in range(0, len(feeds), chunk):
@ -423,7 +430,7 @@ def _refresh_feeds(feeds, chunk=32):
def fetch(url): def fetch(url):
try: try:
return url, _fetch_feed(url, **_validators(by_url[url])), None return url, _fetch_feed(url, **({} if full else _validators(by_url[url]))), None
except Exception as e: # network, HTTP or parser trouble: report and go on except Exception as e: # network, HTTP or parser trouble: report and go on
return url, None, e return url, None, e
@ -440,7 +447,7 @@ def _refresh_feeds(feeds, chunk=32):
_record_error(feed, err) _record_error(feed, err)
continue continue
try: try:
new += _refresh_feed(feed, fetched) new += _refresh_feed(feed, fetched, full=full)
refreshed += 1 refreshed += 1
except Exception as e: except Exception as e:
failed += 1 failed += 1
@ -448,7 +455,7 @@ def _refresh_feeds(feeds, chunk=32):
return refreshed, failed, new return refreshed, failed, new
def _refresh_feed(feed_obj, fetched=None): def _refresh_feed(feed_obj, fetched=None, full=False):
"""Fetch rss_url (unless `fetched`, a FeedFetch, is given) and upsert """Fetch rss_url (unless `fetched`, a FeedFetch, is given) and upsert
episodes. Returns count of new episodes. A failure is stored on the feed episodes. Returns count of new episodes. A failure is stored on the feed
(last_error, error_count) and raised again.""" (last_error, error_count) and raised again."""
@ -459,7 +466,7 @@ def _refresh_feed(feed_obj, fetched=None):
_record_error(feed_obj, e) _record_error(feed_obj, e)
raise raise
try: try:
return _apply_feed(feed_obj, fetched) return _apply_feed(feed_obj, fetched, full=full)
except Exception as e: except Exception as e:
_record_error(feed_obj, e) _record_error(feed_obj, e)
raise raise
@ -479,8 +486,27 @@ def _move_to_permanent_url(feed_obj, new_url):
feed_obj.rss_url = new_url feed_obj.rss_url = new_url
def _apply_feed(feed_obj, fetched): def _entry_media_url(entry):
"""The episode's file: its audio enclosure, else a video one (the players
play only its sound). '' when there is neither."""
video = ''
for enc in entry.get('enclosures', []):
url = enc.get('url', '')
mime = (enc.get('type') or '').lower()
path = urllib.parse.urlsplit(url).path.lower()
if mime.startswith('audio/') or path.endswith(AUDIO_EXTENSIONS):
return url
if not video and (mime.startswith('video/') or path.endswith(VIDEO_EXTENSIONS)):
video = url
return video
def _apply_feed(feed_obj, fetched, full=False):
"""Store a fetched feed. `full` (a re-read after the reader learned to
take more episodes, e.g. video ones): episodes it finds that came out
before the last refresh go into the podcast's list, not into "Neu"."""
now = timezone.now() now = timezone.now()
previous_refresh = feed_obj.last_refreshed_at
if fetched.status == 304: if fetched.status == 304:
# Unchanged since the last full download: nothing to parse or write. # Unchanged since the last full download: nothing to parse or write.
feed_obj.last_refreshed_at = now feed_obj.last_refreshed_at = now
@ -524,13 +550,7 @@ def _apply_feed(feed_obj, fetched):
changed_episodes = {} changed_episodes = {}
for entry in entries: for entry in entries:
# Find audio enclosure audio_url = _entry_media_url(entry)
audio_url = ''
for enc in entry.get('enclosures', []):
mime = enc.get('type', '')
if mime.startswith('audio/') or enc.get('url', '').endswith(('.mp3', '.m4a', '.ogg', '.opus', '.aac')):
audio_url = enc.get('url', '')
break
if not audio_url: if not audio_url:
continue continue
@ -598,6 +618,17 @@ def _apply_feed(feed_obj, fetched):
if changed_episodes: if changed_episodes:
PodcastEpisode.objects.bulk_update(list(changed_episodes.values()), EXTRA_FIELDS, batch_size=100) PodcastEpisode.objects.bulk_update(list(changed_episodes.values()), EXTRA_FIELDS, batch_size=100)
if full and not first_fetch and new_episodes_list:
# Found only now, but out before the last refresh: not news.
late = [e for e in new_episodes_list if e.pub_date is None or e.pub_date < previous_refresh]
if late:
EpisodeProgress.objects.bulk_create(
[EpisodeProgress(user=feed_obj.user, episode=e, dismissed=True) for e in late],
ignore_conflicts=True,
)
gone = {e.id for e in late}
new_episodes_list = [e for e in new_episodes_list if e.id not in gone]
if first_fetch and len(new_episodes_list) > NEW_ON_SUBSCRIBE: if first_fetch and len(new_episodes_list) > NEW_ON_SUBSCRIBE:
# Only the newest episodes count as new; the rest stay in the # Only the newest episodes count as new; the rest stay in the
# podcast's episode list, out of the inbox (dismissed, not played). # podcast's episode list, out of the inbox (dismissed, not played).