Video-Podcasts: Folgen mit Video-Enclosure werden aufgenommen (abgespielt wird der Ton)
Audio bleibt bevorzugt, wenn eine Folge beides hat; Endungen werden am Pfad erkannt (Query-Strings stören nicht mehr). refresh_feeds --full liest alle Feeds ohne ETag neu ein; dabei gefundene Folgen, die vor dem letzten Abruf erschienen sind, kommen in die Liste, nicht nach „Neu“. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
35ea42b6c2
commit
5b2d25e2b2
3 changed files with 107 additions and 15 deletions
|
|
@ -11,6 +11,11 @@ class Command(BaseCommand):
|
|||
def add_arguments(self, parser):
|
||||
parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID (even when backing off)')
|
||||
parser.add_argument('--limit', type=int, default=0, help='Max number of feeds to refresh')
|
||||
parser.add_argument(
|
||||
'--full', action='store_true',
|
||||
help='Download every feed in full (ignore ETag/Last-Modified) and take in episodes skipped before '
|
||||
'(e.g. video ones); those out before the last refresh go into the list, not into "Neu"',
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
qs = PodcastFeed.objects.all().order_by('last_refreshed_at')
|
||||
|
|
@ -30,5 +35,5 @@ class Command(BaseCommand):
|
|||
return
|
||||
|
||||
# Parallel downloads, one per distinct feed URL (see _refresh_feeds).
|
||||
refreshed, failed, new = _refresh_feeds(qs)
|
||||
refreshed, failed, new = _refresh_feeds(qs, full=options['full'])
|
||||
self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes')
|
||||
|
|
|
|||
|
|
@ -223,6 +223,62 @@ class FirstFetchTests(TestCase):
|
|||
self.assertEqual([e['title'] for e in inbox], ['e7', 'e6', 'e5', 'e4'])
|
||||
|
||||
|
||||
class VideoEnclosureTests(TestCase):
|
||||
"""Video podcasts are taken in (the players play their sound only)."""
|
||||
|
||||
def setUp(self):
|
||||
self.user = User.objects.create_user(username='alice', password='pw12345678')
|
||||
self.client.force_login(self.user)
|
||||
self.feed = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/f', title='Feed')
|
||||
|
||||
@staticmethod
|
||||
def rss(*items):
|
||||
entries = ''.join(
|
||||
f'<item><guid>{guid}</guid><title>{guid}</title><pubDate>{date}</pubDate>{enclosures}</item>'
|
||||
for guid, date, enclosures in items
|
||||
)
|
||||
return f'<?xml version="1.0"?><rss version="2.0"><channel><title>Feed</title>{entries}</channel></rss>'
|
||||
|
||||
def refresh(self, rss, **kw):
|
||||
with patch('podcasts.views._fetch_feed', lambda url, **_: _ok(rss)):
|
||||
return _refresh_feed(self.feed, **kw)
|
||||
|
||||
def test_video_only_and_audio_preferred(self):
|
||||
self.refresh(self.rss(
|
||||
('v', 'Mon, 05 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/v.mp4" type="video/mp4" length="1"/>'),
|
||||
('both', 'Tue, 06 Jan 2026 10:00:00 GMT',
|
||||
'<enclosure url="https://example.org/b.mp4" type="video/mp4" length="1"/>'
|
||||
'<enclosure url="https://example.org/b.m4a" type="audio/mp4" length="1"/>'),
|
||||
('untyped', 'Wed, 07 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/u.m4v?x=1" length="1"/>'),
|
||||
('none', 'Thu, 08 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/n.pdf" type="application/pdf"/>'),
|
||||
))
|
||||
urls = dict(PodcastEpisode.objects.filter(feed=self.feed).values_list('guid', 'audio_url'))
|
||||
self.assertEqual(urls, {
|
||||
'v': 'https://example.org/v.mp4',
|
||||
'both': 'https://example.org/b.m4a',
|
||||
'untyped': 'https://example.org/u.m4v?x=1',
|
||||
})
|
||||
|
||||
def test_full_reread_keeps_old_found_episodes_out_of_the_inbox(self):
|
||||
audio = '<enclosure url="https://example.org/a.mp3" type="audio/mpeg" length="1"/>'
|
||||
video = '<enclosure url="https://example.org/{}.mp4" type="video/mp4" length="1"/>'
|
||||
self.refresh(self.rss(('a', 'Mon, 05 Jan 2026 10:00:00 GMT', audio)))
|
||||
self.feed.refresh_from_db()
|
||||
later = (self.feed.last_refreshed_at + timedelta(days=1)).strftime('%a, %d %b %Y %H:%M:%S GMT')
|
||||
self.refresh(self.rss(
|
||||
('a', 'Mon, 05 Jan 2026 10:00:00 GMT', audio),
|
||||
('old-video', 'Tue, 06 Jan 2026 10:00:00 GMT', video.format('o')),
|
||||
('new-video', later, video.format('n')),
|
||||
), full=True)
|
||||
inbox = [e['title'] for e in self.client.get('/podcasts/inbox/').json()['episodes']]
|
||||
self.assertIn('new-video', inbox)
|
||||
self.assertNotIn('old-video', inbox)
|
||||
old = PodcastEpisode.objects.get(feed=self.feed, guid='old-video')
|
||||
progress = EpisodeProgress.objects.get(user=self.user, episode=old)
|
||||
self.assertTrue(progress.dismissed)
|
||||
self.assertFalse(progress.played)
|
||||
|
||||
|
||||
class RefreshAllTests(TestCase):
|
||||
"""One request refreshes all feeds on the server; one request returns the state."""
|
||||
|
||||
|
|
|
|||
|
|
@ -408,10 +408,17 @@ def _in_backoff_q(now=None):
|
|||
return q
|
||||
|
||||
|
||||
def _refresh_feeds(feeds, chunk=32):
|
||||
# File endings of an enclosure without a usable type.
|
||||
AUDIO_EXTENSIONS = ('.mp3', '.m4a', '.m4b', '.ogg', '.oga', '.opus', '.aac', '.flac', '.wav')
|
||||
VIDEO_EXTENSIONS = ('.mp4', '.m4v', '.mov', '.webm')
|
||||
|
||||
|
||||
def _refresh_feeds(feeds, chunk=32, full=False):
|
||||
"""Refresh many feeds at once: the downloads run in parallel and each
|
||||
feed URL is fetched once, even when several users subscribed to it; the
|
||||
database writes then happen one by one. Returns (refreshed, failed, new)."""
|
||||
database writes then happen one by one. Returns (refreshed, failed, new).
|
||||
`full`: download every feed again (no 304) and take in episodes the
|
||||
reader skipped before (see _apply_feed)."""
|
||||
feeds = list(feeds)
|
||||
refreshed = failed = new = 0
|
||||
for start in range(0, len(feeds), chunk):
|
||||
|
|
@ -423,7 +430,7 @@ def _refresh_feeds(feeds, chunk=32):
|
|||
|
||||
def fetch(url):
|
||||
try:
|
||||
return url, _fetch_feed(url, **_validators(by_url[url])), None
|
||||
return url, _fetch_feed(url, **({} if full else _validators(by_url[url]))), None
|
||||
except Exception as e: # network, HTTP or parser trouble: report and go on
|
||||
return url, None, e
|
||||
|
||||
|
|
@ -440,7 +447,7 @@ def _refresh_feeds(feeds, chunk=32):
|
|||
_record_error(feed, err)
|
||||
continue
|
||||
try:
|
||||
new += _refresh_feed(feed, fetched)
|
||||
new += _refresh_feed(feed, fetched, full=full)
|
||||
refreshed += 1
|
||||
except Exception as e:
|
||||
failed += 1
|
||||
|
|
@ -448,7 +455,7 @@ def _refresh_feeds(feeds, chunk=32):
|
|||
return refreshed, failed, new
|
||||
|
||||
|
||||
def _refresh_feed(feed_obj, fetched=None):
|
||||
def _refresh_feed(feed_obj, fetched=None, full=False):
|
||||
"""Fetch rss_url (unless `fetched`, a FeedFetch, is given) and upsert
|
||||
episodes. Returns count of new episodes. A failure is stored on the feed
|
||||
(last_error, error_count) and raised again."""
|
||||
|
|
@ -459,7 +466,7 @@ def _refresh_feed(feed_obj, fetched=None):
|
|||
_record_error(feed_obj, e)
|
||||
raise
|
||||
try:
|
||||
return _apply_feed(feed_obj, fetched)
|
||||
return _apply_feed(feed_obj, fetched, full=full)
|
||||
except Exception as e:
|
||||
_record_error(feed_obj, e)
|
||||
raise
|
||||
|
|
@ -479,8 +486,27 @@ def _move_to_permanent_url(feed_obj, new_url):
|
|||
feed_obj.rss_url = new_url
|
||||
|
||||
|
||||
def _apply_feed(feed_obj, fetched):
|
||||
def _entry_media_url(entry):
|
||||
"""The episode's file: its audio enclosure, else a video one (the players
|
||||
play only its sound). '' when there is neither."""
|
||||
video = ''
|
||||
for enc in entry.get('enclosures', []):
|
||||
url = enc.get('url', '')
|
||||
mime = (enc.get('type') or '').lower()
|
||||
path = urllib.parse.urlsplit(url).path.lower()
|
||||
if mime.startswith('audio/') or path.endswith(AUDIO_EXTENSIONS):
|
||||
return url
|
||||
if not video and (mime.startswith('video/') or path.endswith(VIDEO_EXTENSIONS)):
|
||||
video = url
|
||||
return video
|
||||
|
||||
|
||||
def _apply_feed(feed_obj, fetched, full=False):
|
||||
"""Store a fetched feed. `full` (a re-read after the reader learned to
|
||||
take more episodes, e.g. video ones): episodes it finds that came out
|
||||
before the last refresh go into the podcast's list, not into "Neu"."""
|
||||
now = timezone.now()
|
||||
previous_refresh = feed_obj.last_refreshed_at
|
||||
if fetched.status == 304:
|
||||
# Unchanged since the last full download: nothing to parse or write.
|
||||
feed_obj.last_refreshed_at = now
|
||||
|
|
@ -524,13 +550,7 @@ def _apply_feed(feed_obj, fetched):
|
|||
changed_episodes = {}
|
||||
|
||||
for entry in entries:
|
||||
# Find audio enclosure
|
||||
audio_url = ''
|
||||
for enc in entry.get('enclosures', []):
|
||||
mime = enc.get('type', '')
|
||||
if mime.startswith('audio/') or enc.get('url', '').endswith(('.mp3', '.m4a', '.ogg', '.opus', '.aac')):
|
||||
audio_url = enc.get('url', '')
|
||||
break
|
||||
audio_url = _entry_media_url(entry)
|
||||
if not audio_url:
|
||||
continue
|
||||
|
||||
|
|
@ -598,6 +618,17 @@ def _apply_feed(feed_obj, fetched):
|
|||
if changed_episodes:
|
||||
PodcastEpisode.objects.bulk_update(list(changed_episodes.values()), EXTRA_FIELDS, batch_size=100)
|
||||
|
||||
if full and not first_fetch and new_episodes_list:
|
||||
# Found only now, but out before the last refresh: not news.
|
||||
late = [e for e in new_episodes_list if e.pub_date is None or e.pub_date < previous_refresh]
|
||||
if late:
|
||||
EpisodeProgress.objects.bulk_create(
|
||||
[EpisodeProgress(user=feed_obj.user, episode=e, dismissed=True) for e in late],
|
||||
ignore_conflicts=True,
|
||||
)
|
||||
gone = {e.id for e in late}
|
||||
new_episodes_list = [e for e in new_episodes_list if e.id not in gone]
|
||||
|
||||
if first_fetch and len(new_episodes_list) > NEW_ON_SUBSCRIBE:
|
||||
# Only the newest episodes count as new; the rest stay in the
|
||||
# podcast's episode list, out of the inbox (dismissed, not played).
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue