Video-Podcasts: Folgen mit Video-Enclosure werden aufgenommen (abgespielt wird der Ton)
Audio bleibt bevorzugt, wenn eine Folge beides hat; Endungen werden am Pfad erkannt (Query-Strings stören nicht mehr). refresh_feeds --full liest alle Feeds ohne ETag neu ein; dabei gefundene Folgen, die vor dem letzten Abruf erschienen sind, kommen in die Liste, nicht nach „Neu“. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
35ea42b6c2
commit
5b2d25e2b2
3 changed files with 107 additions and 15 deletions
|
|
@ -11,6 +11,11 @@ class Command(BaseCommand):
|
||||||
def add_arguments(self, parser):
|
def add_arguments(self, parser):
|
||||||
parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID (even when backing off)')
|
parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID (even when backing off)')
|
||||||
parser.add_argument('--limit', type=int, default=0, help='Max number of feeds to refresh')
|
parser.add_argument('--limit', type=int, default=0, help='Max number of feeds to refresh')
|
||||||
|
parser.add_argument(
|
||||||
|
'--full', action='store_true',
|
||||||
|
help='Download every feed in full (ignore ETag/Last-Modified) and take in episodes skipped before '
|
||||||
|
'(e.g. video ones); those out before the last refresh go into the list, not into "Neu"',
|
||||||
|
)
|
||||||
|
|
||||||
def handle(self, *args, **options):
|
def handle(self, *args, **options):
|
||||||
qs = PodcastFeed.objects.all().order_by('last_refreshed_at')
|
qs = PodcastFeed.objects.all().order_by('last_refreshed_at')
|
||||||
|
|
@ -30,5 +35,5 @@ class Command(BaseCommand):
|
||||||
return
|
return
|
||||||
|
|
||||||
# Parallel downloads, one per distinct feed URL (see _refresh_feeds).
|
# Parallel downloads, one per distinct feed URL (see _refresh_feeds).
|
||||||
refreshed, failed, new = _refresh_feeds(qs)
|
refreshed, failed, new = _refresh_feeds(qs, full=options['full'])
|
||||||
self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes')
|
self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes')
|
||||||
|
|
|
||||||
|
|
@ -223,6 +223,62 @@ class FirstFetchTests(TestCase):
|
||||||
self.assertEqual([e['title'] for e in inbox], ['e7', 'e6', 'e5', 'e4'])
|
self.assertEqual([e['title'] for e in inbox], ['e7', 'e6', 'e5', 'e4'])
|
||||||
|
|
||||||
|
|
||||||
|
class VideoEnclosureTests(TestCase):
|
||||||
|
"""Video podcasts are taken in (the players play their sound only)."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(username='alice', password='pw12345678')
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
self.feed = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/f', title='Feed')
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def rss(*items):
|
||||||
|
entries = ''.join(
|
||||||
|
f'<item><guid>{guid}</guid><title>{guid}</title><pubDate>{date}</pubDate>{enclosures}</item>'
|
||||||
|
for guid, date, enclosures in items
|
||||||
|
)
|
||||||
|
return f'<?xml version="1.0"?><rss version="2.0"><channel><title>Feed</title>{entries}</channel></rss>'
|
||||||
|
|
||||||
|
def refresh(self, rss, **kw):
|
||||||
|
with patch('podcasts.views._fetch_feed', lambda url, **_: _ok(rss)):
|
||||||
|
return _refresh_feed(self.feed, **kw)
|
||||||
|
|
||||||
|
def test_video_only_and_audio_preferred(self):
|
||||||
|
self.refresh(self.rss(
|
||||||
|
('v', 'Mon, 05 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/v.mp4" type="video/mp4" length="1"/>'),
|
||||||
|
('both', 'Tue, 06 Jan 2026 10:00:00 GMT',
|
||||||
|
'<enclosure url="https://example.org/b.mp4" type="video/mp4" length="1"/>'
|
||||||
|
'<enclosure url="https://example.org/b.m4a" type="audio/mp4" length="1"/>'),
|
||||||
|
('untyped', 'Wed, 07 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/u.m4v?x=1" length="1"/>'),
|
||||||
|
('none', 'Thu, 08 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/n.pdf" type="application/pdf"/>'),
|
||||||
|
))
|
||||||
|
urls = dict(PodcastEpisode.objects.filter(feed=self.feed).values_list('guid', 'audio_url'))
|
||||||
|
self.assertEqual(urls, {
|
||||||
|
'v': 'https://example.org/v.mp4',
|
||||||
|
'both': 'https://example.org/b.m4a',
|
||||||
|
'untyped': 'https://example.org/u.m4v?x=1',
|
||||||
|
})
|
||||||
|
|
||||||
|
def test_full_reread_keeps_old_found_episodes_out_of_the_inbox(self):
|
||||||
|
audio = '<enclosure url="https://example.org/a.mp3" type="audio/mpeg" length="1"/>'
|
||||||
|
video = '<enclosure url="https://example.org/{}.mp4" type="video/mp4" length="1"/>'
|
||||||
|
self.refresh(self.rss(('a', 'Mon, 05 Jan 2026 10:00:00 GMT', audio)))
|
||||||
|
self.feed.refresh_from_db()
|
||||||
|
later = (self.feed.last_refreshed_at + timedelta(days=1)).strftime('%a, %d %b %Y %H:%M:%S GMT')
|
||||||
|
self.refresh(self.rss(
|
||||||
|
('a', 'Mon, 05 Jan 2026 10:00:00 GMT', audio),
|
||||||
|
('old-video', 'Tue, 06 Jan 2026 10:00:00 GMT', video.format('o')),
|
||||||
|
('new-video', later, video.format('n')),
|
||||||
|
), full=True)
|
||||||
|
inbox = [e['title'] for e in self.client.get('/podcasts/inbox/').json()['episodes']]
|
||||||
|
self.assertIn('new-video', inbox)
|
||||||
|
self.assertNotIn('old-video', inbox)
|
||||||
|
old = PodcastEpisode.objects.get(feed=self.feed, guid='old-video')
|
||||||
|
progress = EpisodeProgress.objects.get(user=self.user, episode=old)
|
||||||
|
self.assertTrue(progress.dismissed)
|
||||||
|
self.assertFalse(progress.played)
|
||||||
|
|
||||||
|
|
||||||
class RefreshAllTests(TestCase):
|
class RefreshAllTests(TestCase):
|
||||||
"""One request refreshes all feeds on the server; one request returns the state."""
|
"""One request refreshes all feeds on the server; one request returns the state."""
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -408,10 +408,17 @@ def _in_backoff_q(now=None):
|
||||||
return q
|
return q
|
||||||
|
|
||||||
|
|
||||||
def _refresh_feeds(feeds, chunk=32):
|
# File endings of an enclosure without a usable type.
|
||||||
|
AUDIO_EXTENSIONS = ('.mp3', '.m4a', '.m4b', '.ogg', '.oga', '.opus', '.aac', '.flac', '.wav')
|
||||||
|
VIDEO_EXTENSIONS = ('.mp4', '.m4v', '.mov', '.webm')
|
||||||
|
|
||||||
|
|
||||||
|
def _refresh_feeds(feeds, chunk=32, full=False):
|
||||||
"""Refresh many feeds at once: the downloads run in parallel and each
|
"""Refresh many feeds at once: the downloads run in parallel and each
|
||||||
feed URL is fetched once, even when several users subscribed to it; the
|
feed URL is fetched once, even when several users subscribed to it; the
|
||||||
database writes then happen one by one. Returns (refreshed, failed, new)."""
|
database writes then happen one by one. Returns (refreshed, failed, new).
|
||||||
|
`full`: download every feed again (no 304) and take in episodes the
|
||||||
|
reader skipped before (see _apply_feed)."""
|
||||||
feeds = list(feeds)
|
feeds = list(feeds)
|
||||||
refreshed = failed = new = 0
|
refreshed = failed = new = 0
|
||||||
for start in range(0, len(feeds), chunk):
|
for start in range(0, len(feeds), chunk):
|
||||||
|
|
@ -423,7 +430,7 @@ def _refresh_feeds(feeds, chunk=32):
|
||||||
|
|
||||||
def fetch(url):
|
def fetch(url):
|
||||||
try:
|
try:
|
||||||
return url, _fetch_feed(url, **_validators(by_url[url])), None
|
return url, _fetch_feed(url, **({} if full else _validators(by_url[url]))), None
|
||||||
except Exception as e: # network, HTTP or parser trouble: report and go on
|
except Exception as e: # network, HTTP or parser trouble: report and go on
|
||||||
return url, None, e
|
return url, None, e
|
||||||
|
|
||||||
|
|
@ -440,7 +447,7 @@ def _refresh_feeds(feeds, chunk=32):
|
||||||
_record_error(feed, err)
|
_record_error(feed, err)
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
new += _refresh_feed(feed, fetched)
|
new += _refresh_feed(feed, fetched, full=full)
|
||||||
refreshed += 1
|
refreshed += 1
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
failed += 1
|
failed += 1
|
||||||
|
|
@ -448,7 +455,7 @@ def _refresh_feeds(feeds, chunk=32):
|
||||||
return refreshed, failed, new
|
return refreshed, failed, new
|
||||||
|
|
||||||
|
|
||||||
def _refresh_feed(feed_obj, fetched=None):
|
def _refresh_feed(feed_obj, fetched=None, full=False):
|
||||||
"""Fetch rss_url (unless `fetched`, a FeedFetch, is given) and upsert
|
"""Fetch rss_url (unless `fetched`, a FeedFetch, is given) and upsert
|
||||||
episodes. Returns count of new episodes. A failure is stored on the feed
|
episodes. Returns count of new episodes. A failure is stored on the feed
|
||||||
(last_error, error_count) and raised again."""
|
(last_error, error_count) and raised again."""
|
||||||
|
|
@ -459,7 +466,7 @@ def _refresh_feed(feed_obj, fetched=None):
|
||||||
_record_error(feed_obj, e)
|
_record_error(feed_obj, e)
|
||||||
raise
|
raise
|
||||||
try:
|
try:
|
||||||
return _apply_feed(feed_obj, fetched)
|
return _apply_feed(feed_obj, fetched, full=full)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_record_error(feed_obj, e)
|
_record_error(feed_obj, e)
|
||||||
raise
|
raise
|
||||||
|
|
@ -479,8 +486,27 @@ def _move_to_permanent_url(feed_obj, new_url):
|
||||||
feed_obj.rss_url = new_url
|
feed_obj.rss_url = new_url
|
||||||
|
|
||||||
|
|
||||||
def _apply_feed(feed_obj, fetched):
|
def _entry_media_url(entry):
|
||||||
|
"""The episode's file: its audio enclosure, else a video one (the players
|
||||||
|
play only its sound). '' when there is neither."""
|
||||||
|
video = ''
|
||||||
|
for enc in entry.get('enclosures', []):
|
||||||
|
url = enc.get('url', '')
|
||||||
|
mime = (enc.get('type') or '').lower()
|
||||||
|
path = urllib.parse.urlsplit(url).path.lower()
|
||||||
|
if mime.startswith('audio/') or path.endswith(AUDIO_EXTENSIONS):
|
||||||
|
return url
|
||||||
|
if not video and (mime.startswith('video/') or path.endswith(VIDEO_EXTENSIONS)):
|
||||||
|
video = url
|
||||||
|
return video
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_feed(feed_obj, fetched, full=False):
|
||||||
|
"""Store a fetched feed. `full` (a re-read after the reader learned to
|
||||||
|
take more episodes, e.g. video ones): episodes it finds that came out
|
||||||
|
before the last refresh go into the podcast's list, not into "Neu"."""
|
||||||
now = timezone.now()
|
now = timezone.now()
|
||||||
|
previous_refresh = feed_obj.last_refreshed_at
|
||||||
if fetched.status == 304:
|
if fetched.status == 304:
|
||||||
# Unchanged since the last full download: nothing to parse or write.
|
# Unchanged since the last full download: nothing to parse or write.
|
||||||
feed_obj.last_refreshed_at = now
|
feed_obj.last_refreshed_at = now
|
||||||
|
|
@ -524,13 +550,7 @@ def _apply_feed(feed_obj, fetched):
|
||||||
changed_episodes = {}
|
changed_episodes = {}
|
||||||
|
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
# Find audio enclosure
|
audio_url = _entry_media_url(entry)
|
||||||
audio_url = ''
|
|
||||||
for enc in entry.get('enclosures', []):
|
|
||||||
mime = enc.get('type', '')
|
|
||||||
if mime.startswith('audio/') or enc.get('url', '').endswith(('.mp3', '.m4a', '.ogg', '.opus', '.aac')):
|
|
||||||
audio_url = enc.get('url', '')
|
|
||||||
break
|
|
||||||
if not audio_url:
|
if not audio_url:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|
@ -598,6 +618,17 @@ def _apply_feed(feed_obj, fetched):
|
||||||
if changed_episodes:
|
if changed_episodes:
|
||||||
PodcastEpisode.objects.bulk_update(list(changed_episodes.values()), EXTRA_FIELDS, batch_size=100)
|
PodcastEpisode.objects.bulk_update(list(changed_episodes.values()), EXTRA_FIELDS, batch_size=100)
|
||||||
|
|
||||||
|
if full and not first_fetch and new_episodes_list:
|
||||||
|
# Found only now, but out before the last refresh: not news.
|
||||||
|
late = [e for e in new_episodes_list if e.pub_date is None or e.pub_date < previous_refresh]
|
||||||
|
if late:
|
||||||
|
EpisodeProgress.objects.bulk_create(
|
||||||
|
[EpisodeProgress(user=feed_obj.user, episode=e, dismissed=True) for e in late],
|
||||||
|
ignore_conflicts=True,
|
||||||
|
)
|
||||||
|
gone = {e.id for e in late}
|
||||||
|
new_episodes_list = [e for e in new_episodes_list if e.id not in gone]
|
||||||
|
|
||||||
if first_fetch and len(new_episodes_list) > NEW_ON_SUBSCRIBE:
|
if first_fetch and len(new_episodes_list) > NEW_ON_SUBSCRIBE:
|
||||||
# Only the newest episodes count as new; the rest stay in the
|
# Only the newest episodes count as new; the rest stay in the
|
||||||
# podcast's episode list, out of the inbox (dismissed, not played).
|
# podcast's episode list, out of the inbox (dismissed, not played).
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue