diff --git a/docs/youtube.md b/docs/youtube.md index 3ef30d5..4e1be4c 100644 --- a/docs/youtube.md +++ b/docs/youtube.md @@ -27,6 +27,15 @@ sound**: each app finds the audio stream itself. `{"ready": true, "spans": [], "error": "youtube"}` (skip silence is up to the app, as on Android anyway). +- `POST /podcasts/feeds//move/ {"rss_url": …}` gives an existing subscription a + new address and keeps everything (settings, episodes, places, played and favourite + marks, queue). Moved to a YouTube address, every episode whose file is named after + its video (`…/<11-character id>.mp3`, as podsync-like servers name them) becomes + that video (`guid` `yt:video:`, `audio_url` its `watch?v=` page), so the + channel's feed finds it again instead of adding it twice. Used to move channels + that came through such a server (it downloaded the sound on the server) to the + channel itself. + ## Apps A client that sees a `watch?v=` address resolves it **itself, on the device, diff --git a/podcasts/test_youtube.py b/podcasts/test_youtube.py index 215615c..e52734c 100644 --- a/podcasts/test_youtube.py +++ b/podcasts/test_youtube.py @@ -192,3 +192,68 @@ class YouTubeFeedTests(TestCase): save({'position_seconds': 20, 'duration_seconds': 5}) # known now: not changed again ep.refresh_from_db() self.assertEqual(ep.duration_seconds, 929) + + +class MoveFeedTests(TestCase): + """A channel fed through another server (podsync-like) moved to the channel itself.""" + OLD = 'http://creamfresh.example:8001/feed/abc/rss.xml' + + def setUp(self): + from .models import EpisodeProgress, PodcastQueue + self.user = User.objects.create_user(username='alice', password='pw12345678') + self.client.force_login(self.user) + self.feed = PodcastFeed.objects.create(user=self.user, rss_url=self.OLD, title='Tages Schau', + playback_speed=1.2, http_etag='"x"') + mk = lambda vid, title, guid=None: PodcastEpisode.objects.create( + feed=self.feed, guid=guid or f'old-{vid}', title=title, duration_seconds=600, + audio_url=f'http://creamfresh.example:8001/feed/abc/audio/{vid}.mp3') + self.newest = mk('aaaaaaaaaaa', 'Video eins') + self.older = mk('zzzzzzzzzzz', 'Altes Video') # no longer in the channel's feed + self.trailer = mk('UCs8mbJ-M142ZskR5VR0gBig', 'Kanal - Videos') + EpisodeProgress.objects.create(user=self.user, episode=self.newest, position_seconds=0, played=True) + EpisodeProgress.objects.create(user=self.user, episode=self.older, position_seconds=300, favorite=True) + PodcastQueue.objects.create(user=self.user, episode=self.older, position=0) + + def move(self, url): + def fetch(u, **kw): + self.assertEqual(kw.get('etag', ''), '') # the old address's validators are gone + return FeedFetch(200, feedparser.parse(FEED_XML)) + with patch('podcasts.youtube.requests.get', lambda u, **kw: _Page(PAGE)), \ + patch('podcasts.views._fetch_feed', fetch): + return self.client.post(f'/podcasts/feeds/{self.feed.pk}/move/', json.dumps({'rss_url': url}), + content_type='application/json') + + def test_the_subscription_keeps_everything_and_its_episodes_become_videos(self): + from .models import EpisodeProgress, PodcastQueue + r = self.move('https://www.youtube.com/@tagesschau') + self.assertEqual(r.status_code, 200, r.content) + self.assertEqual(r.json()['videos'], 2) + self.feed.refresh_from_db() + self.assertEqual(self.feed.rss_url, f'{youtube.FEED}?channel_id={CHANNEL}') + self.assertEqual(self.feed.playback_speed, 1.2) + self.assertEqual(self.feed.artwork_url, 'https://yt3.example/pic.jpg') + self.newest.refresh_from_db() + self.older.refresh_from_db() + self.trailer.refresh_from_db() + self.assertEqual((self.newest.guid, self.newest.audio_url), ('yt:video:aaaaaaaaaaa', youtube.WATCH + 'aaaaaaaaaaa')) + self.assertEqual(self.older.audio_url, youtube.WATCH + 'zzzzzzzzzzz') + self.assertTrue(self.trailer.audio_url.endswith('.mp3')) # not a video: left as it was + # Video eins came again with the channel's feed: the same episode, not a second one. + self.assertEqual(PodcastEpisode.objects.filter(title='Video eins').count(), 1) + self.assertEqual(set(PodcastEpisode.objects.values_list('title', flat=True)), + {'Video eins', 'Video zwei', 'Altes Video', 'Kanal - Videos'}) + self.assertTrue(EpisodeProgress.objects.get(episode=self.newest).played) + p = EpisodeProgress.objects.get(episode=self.older) + self.assertEqual((p.position_seconds, p.favorite), (300, True)) + self.assertTrue(PodcastQueue.objects.filter(episode=self.older).exists()) + + def test_an_address_subscribed_already_is_refused(self): + PodcastFeed.objects.create(user=self.user, rss_url=f'{youtube.FEED}?channel_id={CHANNEL}', title='x') + self.assertEqual(self.move('https://www.youtube.com/@tagesschau').status_code, 409) + self.feed.refresh_from_db() + self.assertEqual(self.feed.rss_url, self.OLD) + + def test_only_ones_own_feed(self): + bob = User.objects.create_user(username='bob', password='pw12345678') + self.client.force_login(bob) + self.assertEqual(self.move('https://www.youtube.com/@tagesschau').status_code, 404) diff --git a/podcasts/urls.py b/podcasts/urls.py index 2ec6463..a6ebd46 100644 --- a/podcasts/urls.py +++ b/podcasts/urls.py @@ -13,6 +13,7 @@ urlpatterns = [ path('feeds//remove/', views.remove_feed, name='podcast_remove_feed'), path('feeds//set-auto-queue/', views.set_auto_queue, name='podcast_set_auto_queue'), path('feeds//settings/', views.feed_settings, name='podcast_feed_settings'), + path('feeds//move/', views.move_feed, name='podcast_move_feed'), path('feeds//episodes/', views.feed_episodes, name='podcast_feed_episodes'), path('episodes/', views.episode_list, name='podcast_episode_list'), path('episodes/favorite/', views.set_favorites, name='podcast_set_favorites'), diff --git a/podcasts/views.py b/podcasts/views.py index 45257bb..ce5e208 100644 --- a/podcasts/views.py +++ b/podcasts/views.py @@ -1012,6 +1012,68 @@ FEED_SETTING_PARSERS = { } +@csrf_exempt +@require_http_methods(['POST']) +def move_feed(request, pk): + """Give a subscription a new address, keeping it: its settings, episodes, places, + played and favourite marks, queue entries. For a podcast that moved, or one that was + a YouTube channel fed through some other server (a podsync-like one): moved to the + channel itself (a YouTube address of any kind), each episode whose file was named + after its video becomes that video (docs/youtube.md), so the channel's feed finds it + again instead of adding it twice. {"ok", "rss_url", "videos": episodes made videos, + "new_episodes"}.""" + if not request.user.is_authenticated: + return JsonResponse({'error': 'authentication required'}, status=401) + try: + body = json.loads(request.body) + except (json.JSONDecodeError, ValueError): + return JsonResponse({'error': 'invalid JSON'}, status=400) + feed = PodcastFeed.objects.filter(pk=pk, user=request.user).first() + if feed is None: + return JsonResponse({'error': 'not found'}, status=404) + rss_url = str(body.get('rss_url') or '').strip() + if not rss_url.startswith(('http://', 'https://')) or len(rss_url) > 1000: + return JsonResponse({'error': 'rss_url required'}, status=400) + artwork = '' + if youtube_mod.is_youtube(rss_url) and not youtube_mod.is_feed(rss_url): + try: + found = youtube_mod.resolve(rss_url) + except youtube_mod.NotAChannel as e: + return JsonResponse({'error': f'No YouTube channel there ({e})'}, status=400) + rss_url, artwork = found['feed'], found['artwork'] + if rss_url != feed.rss_url and PodcastFeed.objects.filter(user=request.user, rss_url=rss_url).exists(): + return JsonResponse({'error': 'already subscribed at that address'}, status=409) + + videos = 0 + with transaction.atomic(): + if youtube_mod.is_feed(rss_url): + taken = set(feed.episodes.values_list('guid', flat=True)) + changed = [] + for ep in feed.episodes.only('id', 'guid', 'audio_url'): + vid = youtube_mod.video_id_in(ep.audio_url) + guid = f'yt:video:{vid}' if vid else '' + if not vid or (guid in taken and ep.guid != guid): + continue # not named after a video, or that video is here already + taken.add(guid) + ep.guid, ep.audio_url = guid, youtube_mod.WATCH + vid + changed.append(ep) + PodcastEpisode.objects.bulk_update(changed, ['guid', 'audio_url'], batch_size=200) + videos = len(changed) + feed.rss_url = rss_url + if artwork: + feed.artwork_url = artwork[:1000] + # Whatever the old address answered says nothing about the new one. + feed.http_etag = feed.http_last_modified = feed.http_body_hash = '' + feed.save() + try: + # Videos the old address didn't carry that came out before its last refresh aren't news. + new_episodes = _refresh_feed(feed, full=True) + except Exception as e: # moved all the same: the next refresh tries again + log.warning('feed %s moved, first refresh failed: %s', feed.pk, e) + new_episodes = 0 + return JsonResponse({'ok': True, 'rss_url': feed.rss_url, 'videos': videos, 'new_episodes': new_episodes}) + + @csrf_exempt @require_http_methods(['POST']) def feed_settings(request, pk): diff --git a/podcasts/youtube.py b/podcasts/youtube.py index 15dab4f..e8e53e2 100644 --- a/podcasts/youtube.py +++ b/podcasts/youtube.py @@ -78,6 +78,16 @@ def video_url(entry): return WATCH + vid +def video_id_in(audio_url): + """The video an episode is, judged by its address: a watch page, or a file named after + the video (as servers that turn channels into podcasts name them), '' otherwise.""" + if is_video(audio_url): + return audio_url[len(WATCH):] + path = urllib.parse.urlsplit(audio_url).path + m = re.search(r'/([\w-]{11})\.(?:mp3|m4a|mp4|webm|opus|ogg|aac)$', path) + return m.group(1) if m else '' + + def entry_artwork(entry): thumbs = entry.get('media_thumbnail') or [] return (thumbs[0].get('url') or '') if thumbs else ''