Merge branch 'wt-impdl'
This commit is contained in:
commit
8ba82a0e55
7 changed files with 300 additions and 40 deletions
|
|
@ -2,12 +2,14 @@
|
||||||
|
|
||||||
AntennaPod's "Datenbank exportieren" writes its whole SQLite database. From
|
AntennaPod's "Datenbank exportieren" writes its whole SQLite database. From
|
||||||
it come the subscriptions with their settings, the state of every episode
|
it come the subscriptions with their settings, the state of every episode
|
||||||
that has one (played, new, started, queued, favourite) and the queue. See
|
that has one (played, new, started, queued, favourite), the queue and which
|
||||||
|
episodes were on the phone (the app offers to download those again). See
|
||||||
the import_antennapod command for how to run it.
|
the import_antennapod command for how to run it.
|
||||||
|
|
||||||
The backup is only read. Episodes without any state are not imported: the
|
The backup is only read. Episodes without any state (and not on the phone)
|
||||||
feed's current episodes come with its next refresh anyway. Those that diora
|
are not imported: the feed's current episodes come with its next refresh
|
||||||
already has, though, are taken out of the inbox (AntennaPod: not new).
|
anyway. Those that diora already has, though, are taken out of the inbox
|
||||||
|
(AntennaPod: not new).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
|
@ -37,6 +39,8 @@ AUTO_DOWNLOAD_OFF, AUTO_DOWNLOAD_GLOBAL, AUTO_DOWNLOAD_ON = 0, 1, 2
|
||||||
# FeedItems.read
|
# FeedItems.read
|
||||||
NEW, UNPLAYED, PLAYED = -1, 0, 1
|
NEW, UNPLAYED, PLAYED = -1, 0, 1
|
||||||
SUBSCRIBED = 0 # Feeds.state
|
SUBSCRIBED = 0 # Feeds.state
|
||||||
|
# FeedMedia.filesize above this is garbage (seen: 10^13); taken as unknown.
|
||||||
|
MAX_FILESIZE = 4_000_000_000
|
||||||
|
|
||||||
SETTING_FIELDS = ('playback_speed', 'skip_intro', 'skip_outro', 'volume', 'include_words',
|
SETTING_FIELDS = ('playback_speed', 'skip_intro', 'skip_outro', 'volume', 'include_words',
|
||||||
'exclude_words', 'min_duration', 'new_episodes', 'auto_download')
|
'exclude_words', 'min_duration', 'new_episodes', 'auto_download')
|
||||||
|
|
@ -250,6 +254,11 @@ class Stats:
|
||||||
keep_sorted: str = ''
|
keep_sorted: str = ''
|
||||||
inbox_before: int = 0
|
inbox_before: int = 0
|
||||||
inbox_after: int = 0
|
inbox_after: int = 0
|
||||||
|
downloads_backup: int = 0
|
||||||
|
downloads: int = 0
|
||||||
|
downloads_bytes: int = 0
|
||||||
|
downloads_created: int = 0
|
||||||
|
downloads_missing: int = 0
|
||||||
backup_date: datetime | None = None
|
backup_date: datetime | None = None
|
||||||
seconds: float = 0.0
|
seconds: float = 0.0
|
||||||
|
|
||||||
|
|
@ -325,9 +334,11 @@ class AntennaPodImport:
|
||||||
st.queue_backup = len(queue_items)
|
st.queue_backup = len(queue_items)
|
||||||
st.favorites_backup = len(favorite_items)
|
st.favorites_backup = len(favorite_items)
|
||||||
|
|
||||||
states, not_new, item_episode = self._import_episodes(db, feed_map, set(queue_items), favorite_items)
|
states, not_new, item_episode, downloads = self._import_episodes(
|
||||||
|
db, feed_map, set(queue_items), favorite_items)
|
||||||
self._import_progress(states, not_new)
|
self._import_progress(states, not_new)
|
||||||
self._import_queue(queue_items, item_episode)
|
self._import_queue(queue_items, item_episode)
|
||||||
|
self._import_downloads(settings, downloads, item_episode)
|
||||||
|
|
||||||
st.queue_after = len(_queue_order(user))
|
st.queue_after = len(_queue_order(user))
|
||||||
st.inbox_after = _inbox_count(user)
|
st.inbox_after = _inbox_count(user)
|
||||||
|
|
@ -410,20 +421,25 @@ class AntennaPodImport:
|
||||||
|
|
||||||
def _import_episodes(self, db, feed_map, queued, favorites):
|
def _import_episodes(self, db, feed_map, queued, favorites):
|
||||||
"""Match AntennaPod's episodes to diora's, create the missing ones
|
"""Match AntennaPod's episodes to diora's, create the missing ones
|
||||||
that have a state. Returns ({episode id: ItemState} for those with a
|
that have a state (or were downloaded). Returns ({episode id: ItemState}
|
||||||
state, {episode ids} of matched ones without (not new), {AntennaPod
|
for those with a state, {episode ids} of matched ones without (not new),
|
||||||
item id: episode id})."""
|
{AntennaPod item id: episode id}, [(pub date ms, AntennaPod item id,
|
||||||
|
size)] of the downloaded ones)."""
|
||||||
st = self.stats
|
st = self.stats
|
||||||
backup = self.backup_date
|
backup = self.backup_date
|
||||||
states = {}
|
states = {}
|
||||||
not_new = set()
|
not_new = set()
|
||||||
item_episode = {}
|
item_episode = {}
|
||||||
|
downloads = []
|
||||||
to_create = {} # (diora feed id, guid) -> [ItemState, AntennaPod item id, audio url, duration, pub_date]
|
to_create = {} # (diora feed id, guid) -> [ItemState, AntennaPod item id, audio url, duration, pub_date]
|
||||||
|
|
||||||
|
media = _columns(db, 'FeedMedia')
|
||||||
cur = db.execute(
|
cur = db.execute(
|
||||||
'SELECT i.feed, i.id, i.item_identifier, i.read, i.pubDate, m.download_url, m.duration, '
|
'SELECT i.feed, i.id, i.item_identifier, i.read, i.pubDate, m.download_url, m.duration, '
|
||||||
'm.position, m.last_played_time, m.playback_completion_date '
|
'm.position, m.last_played_time, m.playback_completion_date, '
|
||||||
'FROM FeedItems i LEFT JOIN FeedMedia m ON m.feeditem = i.id ORDER BY i.feed, i.id'
|
+ ('m.downloaded, ' if 'downloaded' in media else '0, ')
|
||||||
|
+ ('m.filesize ' if 'filesize' in media else '0 ')
|
||||||
|
+ 'FROM FeedItems i LEFT JOIN FeedMedia m ON m.feeditem = i.id ORDER BY i.feed, i.id'
|
||||||
)
|
)
|
||||||
|
|
||||||
def rows():
|
def rows():
|
||||||
|
|
@ -436,12 +452,19 @@ class AntennaPodImport:
|
||||||
for ap_feed, items in groupby(rows(), key=lambda r: r[0]):
|
for ap_feed, items in groupby(rows(), key=lambda r: r[0]):
|
||||||
feed_id = feed_map.get(ap_feed)
|
feed_id = feed_map.get(ap_feed)
|
||||||
by_guid = by_url = None
|
by_guid = by_url = None
|
||||||
for (_, item_id, ident, read, pub_ms, audio, duration, position, last_played, completed) in items:
|
for (_, item_id, ident, read, pub_ms, audio, duration, position, last_played, completed,
|
||||||
|
downloaded, size) in items:
|
||||||
position = max(int(position or 0), 0) // 1000
|
position = max(int(position or 0), 0) // 1000
|
||||||
activity = max(filter(None, (_ms(last_played), _ms(completed))), default=None)
|
activity = max(filter(None, (_ms(last_played), _ms(completed))), default=None)
|
||||||
in_queue = item_id in queued
|
in_queue = item_id in queued
|
||||||
favorite = item_id in favorites
|
favorite = item_id in favorites
|
||||||
stateful = read in (NEW, PLAYED) or position > 0 or activity is not None or in_queue or favorite
|
stateful = read in (NEW, PLAYED) or position > 0 or activity is not None or in_queue or favorite
|
||||||
|
# FeedMedia.downloaded: when the file was downloaded (ms; 1 from
|
||||||
|
# older versions), 0 = not on the phone.
|
||||||
|
downloaded = (downloaded or 0) > 0
|
||||||
|
if downloaded:
|
||||||
|
size = int(size or 0)
|
||||||
|
downloads.append((int(pub_ms or 0), item_id, size if 0 < size <= MAX_FILESIZE else 0))
|
||||||
if feed_id is None:
|
if feed_id is None:
|
||||||
st.in_skipped_feeds += stateful
|
st.in_skipped_feeds += stateful
|
||||||
continue
|
continue
|
||||||
|
|
@ -464,7 +487,12 @@ class AntennaPodImport:
|
||||||
not_new.add(ep_id)
|
not_new.add(ep_id)
|
||||||
item_episode[item_id] = ep_id
|
item_episode[item_id] = ep_id
|
||||||
continue
|
continue
|
||||||
|
if not downloaded:
|
||||||
|
continue
|
||||||
|
# On the phone in AntennaPod but not in diora yet (a feed
|
||||||
|
# not fetched so far): created as not new, so the app can
|
||||||
|
# offer to download it again.
|
||||||
|
else:
|
||||||
st.stateful += 1
|
st.stateful += 1
|
||||||
played = read == PLAYED
|
played = read == PLAYED
|
||||||
pub_date = _ms(pub_ms)
|
pub_date = _ms(pub_ms)
|
||||||
|
|
@ -484,7 +512,7 @@ class AntennaPodImport:
|
||||||
item_episode[item_id] = ep_id
|
item_episode[item_id] = ep_id
|
||||||
continue
|
continue
|
||||||
if normalize_url(audio) is None or len(audio) > 1000:
|
if normalize_url(audio) is None or len(audio) > 1000:
|
||||||
st.no_audio += 1
|
st.no_audio += stateful
|
||||||
continue
|
continue
|
||||||
key = (feed_id, guid)
|
key = (feed_id, guid)
|
||||||
if key in to_create:
|
if key in to_create:
|
||||||
|
|
@ -493,12 +521,13 @@ class AntennaPodImport:
|
||||||
pending[1].append(item_id)
|
pending[1].append(item_id)
|
||||||
else:
|
else:
|
||||||
to_create[key] = [state, [item_id], audio, max(int(duration or 0), 0) // 1000, pub_date]
|
to_create[key] = [state, [item_id], audio, max(int(duration or 0), 0) // 1000, pub_date]
|
||||||
|
st.downloads_created += not stateful
|
||||||
|
|
||||||
if to_create:
|
if to_create:
|
||||||
self._create_episodes(db, to_create, states, item_episode)
|
self._create_episodes(db, to_create, states, item_episode)
|
||||||
not_new -= states.keys()
|
not_new -= states.keys()
|
||||||
st.stateless_matched = len(not_new)
|
st.stateless_matched = len(not_new)
|
||||||
return states, not_new, item_episode
|
return states, not_new, item_episode, downloads
|
||||||
|
|
||||||
def _create_episodes(self, db, to_create, states, item_episode):
|
def _create_episodes(self, db, to_create, states, item_episode):
|
||||||
st = self.stats
|
st = self.stats
|
||||||
|
|
@ -522,7 +551,7 @@ class AntennaPodImport:
|
||||||
duration_seconds=duration, pub_date=pub_date, artwork_url=_web_url(image),
|
duration_seconds=duration, pub_date=pub_date, artwork_url=_web_url(image),
|
||||||
))
|
))
|
||||||
PodcastEpisode.objects.bulk_create(batch)
|
PodcastEpisode.objects.bulk_create(batch)
|
||||||
st.episodes_created = len(to_create)
|
st.episodes_created = len(to_create) - st.downloads_created
|
||||||
|
|
||||||
by_feed = {}
|
by_feed = {}
|
||||||
for feed_id, guid in to_create:
|
for feed_id, guid in to_create:
|
||||||
|
|
@ -627,6 +656,29 @@ class AntennaPodImport:
|
||||||
if _keep_sorted(user):
|
if _keep_sorted(user):
|
||||||
st.keep_sorted = PodcastSettings.objects.get(user=user).keep_sorted
|
st.keep_sorted = PodcastSettings.objects.get(user=user).keep_sorted
|
||||||
|
|
||||||
|
# -- downloads ----------------------------------------------------------
|
||||||
|
|
||||||
|
def _import_downloads(self, settings, downloads, item_episode):
|
||||||
|
"""The episodes AntennaPod had on the phone, newest first: the app
|
||||||
|
can't take its files over, so it offers to download them again.
|
||||||
|
Replaces what an earlier import left."""
|
||||||
|
st = self.stats
|
||||||
|
st.downloads_backup = len(downloads)
|
||||||
|
entries, seen = [], set()
|
||||||
|
for _, item_id, size in sorted(downloads, key=lambda d: (-d[0], -d[1])):
|
||||||
|
ep_id = item_episode.get(item_id)
|
||||||
|
if ep_id is None:
|
||||||
|
st.downloads_missing += 1
|
||||||
|
continue
|
||||||
|
if ep_id in seen:
|
||||||
|
continue
|
||||||
|
seen.add(ep_id)
|
||||||
|
entries.append({'episode_id': ep_id, 'size': size})
|
||||||
|
st.downloads = len(entries)
|
||||||
|
st.downloads_bytes = sum(e['size'] for e in entries)
|
||||||
|
settings.import_downloads = entries
|
||||||
|
settings.save(update_fields=['import_downloads'])
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Summary
|
# Summary
|
||||||
|
|
@ -670,6 +722,10 @@ def summary_lines(st, username, path, dry_run, global_auto_download):
|
||||||
f'{st.queue_after} after ({st.queue_appended} diora entries after AntennaPod\'s)'
|
f'{st.queue_after} after ({st.queue_appended} diora entries after AntennaPod\'s)'
|
||||||
+ (f'; kept sorted by "{st.keep_sorted}"' if st.keep_sorted else ''),
|
+ (f'; kept sorted by "{st.keep_sorted}"' if st.keep_sorted else ''),
|
||||||
f'Inbox: {st.inbox_before} before, {st.inbox_after} after',
|
f'Inbox: {st.inbox_before} before, {st.inbox_after} after',
|
||||||
|
f'Downloads: {st.downloads} on the phone in AntennaPod ({st.downloads_bytes / 1_000_000:.0f} MB) '
|
||||||
|
f'-> offered to the app for download'
|
||||||
|
+ (f' ({st.downloads_created} episodes created for them)' if st.downloads_created else '')
|
||||||
|
+ (f'; {st.downloads_missing} not found (feeds not imported, no audio URL)' if st.downloads_missing else ''),
|
||||||
f'Created feeds ({len(st.created_titles)}):',
|
f'Created feeds ({len(st.created_titles)}):',
|
||||||
*(f' - {t}' for t in st.created_titles),
|
*(f' - {t}' for t in st.created_titles),
|
||||||
f'Skipped feeds ({len(st.skipped_feeds)}):',
|
f'Skipped feeds ({len(st.skipped_feeds)}):',
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,8 @@ from podcasts.antennapod import AntennaPodImport, AntennaPodImportError, summary
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
help = (
|
help = (
|
||||||
"Import an AntennaPod database backup (Settings > Database > Export) into one user's podcasts: "
|
"Import an AntennaPod database backup (Settings > Database > Export) into one user's podcasts: "
|
||||||
'subscriptions with their settings, played/new/started episodes, favourites and the queue. '
|
'subscriptions with their settings, played/new/started episodes, favourites and the queue; '
|
||||||
|
'the episodes downloaded in AntennaPod are offered to the app for download. '
|
||||||
'Feeds it creates are fetched by the next refresh; running the import once more after that '
|
'Feeds it creates are fetched by the next refresh; running the import once more after that '
|
||||||
'also takes their older episodes that were not new in AntennaPod out of the inbox. '
|
'also takes their older episodes that were not new in AntennaPod out of the inbox. '
|
||||||
'Running it twice changes nothing more.'
|
'Running it twice changes nothing more.'
|
||||||
|
|
|
||||||
18
podcasts/migrations/0011_settings_import_downloads.py
Normal file
18
podcasts/migrations/0011_settings_import_downloads.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 6.1 on 2026-10-02 09:51
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('podcasts', '0010_feed_auto_download'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='podcastsettings',
|
||||||
|
name='import_downloads',
|
||||||
|
field=models.JSONField(blank=True, default=list),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -236,6 +236,10 @@ class PodcastSettings(models.Model):
|
||||||
keep_sorted = models.CharField(max_length=10, choices=SORT_ORDERS, default='', blank=True)
|
keep_sorted = models.CharField(max_length=10, choices=SORT_ORDERS, default='', blank=True)
|
||||||
# What PodcastFeed.auto_download starts as for a new subscription.
|
# What PodcastFeed.auto_download starts as for a new subscription.
|
||||||
auto_download_default = models.BooleanField(default=False)
|
auto_download_default = models.BooleanField(default=False)
|
||||||
|
# Episodes that were downloaded in AntennaPod when its backup was
|
||||||
|
# imported: [{episode_id, size (bytes, 0 = unknown)}]. The app offers once
|
||||||
|
# to download them again, then clears the list.
|
||||||
|
import_downloads = models.JSONField(default=list, blank=True)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def for_user(cls, user):
|
def for_user(cls, user):
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,7 @@ class Backup:
|
||||||
item_identifier TEXT, image_url TEXT);
|
item_identifier TEXT, image_url TEXT);
|
||||||
CREATE TABLE FeedMedia (id INTEGER PRIMARY KEY AUTOINCREMENT, duration INTEGER, download_url TEXT,
|
CREATE TABLE FeedMedia (id INTEGER PRIMARY KEY AUTOINCREMENT, duration INTEGER, download_url TEXT,
|
||||||
position INTEGER, playback_completion_date INTEGER, feeditem INTEGER, played_duration INTEGER,
|
position INTEGER, playback_completion_date INTEGER, feeditem INTEGER, played_duration INTEGER,
|
||||||
last_played_time INTEGER);
|
last_played_time INTEGER, downloaded INTEGER DEFAULT 0, filesize INTEGER DEFAULT 0);
|
||||||
CREATE TABLE Queue (id INTEGER PRIMARY KEY AUTOINCREMENT, feeditem INTEGER, feed INTEGER);
|
CREATE TABLE Queue (id INTEGER PRIMARY KEY AUTOINCREMENT, feeditem INTEGER, feed INTEGER);
|
||||||
CREATE TABLE Favorites (id INTEGER PRIMARY KEY AUTOINCREMENT, feeditem INTEGER, feed INTEGER);
|
CREATE TABLE Favorites (id INTEGER PRIMARY KEY AUTOINCREMENT, feeditem INTEGER, feed INTEGER);
|
||||||
"""
|
"""
|
||||||
|
|
@ -55,7 +55,7 @@ class Backup:
|
||||||
return cur.lastrowid
|
return cur.lastrowid
|
||||||
|
|
||||||
def item(self, feed, guid, read=0, audio=None, pub=T0, duration=1800, position=0, completed=None,
|
def item(self, feed, guid, read=0, audio=None, pub=T0, duration=1800, position=0, completed=None,
|
||||||
last_played=None, title=None, description='', image_url=None, media=True):
|
last_played=None, title=None, description='', image_url=None, media=True, downloaded=0, size=0):
|
||||||
cur = self.db.execute(
|
cur = self.db.execute(
|
||||||
'INSERT INTO FeedItems (title, pubDate, read, description, feed, item_identifier, image_url) '
|
'INSERT INTO FeedItems (title, pubDate, read, description, feed, item_identifier, image_url) '
|
||||||
'VALUES (?,?,?,?,?,?,?)',
|
'VALUES (?,?,?,?,?,?,?)',
|
||||||
|
|
@ -64,9 +64,10 @@ class Backup:
|
||||||
if media:
|
if media:
|
||||||
self.db.execute(
|
self.db.execute(
|
||||||
'INSERT INTO FeedMedia (duration, download_url, position, playback_completion_date, feeditem, '
|
'INSERT INTO FeedMedia (duration, download_url, position, playback_completion_date, feeditem, '
|
||||||
'played_duration, last_played_time) VALUES (?,?,?,?,?,?,?)',
|
'played_duration, last_played_time, downloaded, filesize) VALUES (?,?,?,?,?,?,?,?,?)',
|
||||||
(duration * 1000, audio or f'https://cdn.example.org/{guid}.mp3', position * 1000,
|
(duration * 1000, audio or f'https://cdn.example.org/{guid}.mp3', position * 1000,
|
||||||
ms(completed) if completed else 0, item_id, 0, ms(last_played) if last_played else 0))
|
ms(completed) if completed else 0, item_id, 0, ms(last_played) if last_played else 0,
|
||||||
|
downloaded, size))
|
||||||
self.db.commit()
|
self.db.commit()
|
||||||
return item_id
|
return item_id
|
||||||
|
|
||||||
|
|
@ -444,6 +445,128 @@ class EpisodeImportTests(ImportTestCase):
|
||||||
self.assertIn('Progress: 0 new, 0 updated, 4 unchanged, 0 kept (diora newer); 0 taken out', out)
|
self.assertIn('Progress: 0 new, 0 updated, 4 unchanged, 0 kept (diora newer); 0 taken out', out)
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadImportTests(ImportTestCase):
|
||||||
|
"""Episodes on the phone in AntennaPod: offered to the app for download."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super().setUp()
|
||||||
|
self.feed = self.diora_feed()
|
||||||
|
self.ap_feed = self.ap.feed('https://example.org/feed')
|
||||||
|
|
||||||
|
def offered(self):
|
||||||
|
return PodcastSettings.for_user(self.user).import_downloads
|
||||||
|
|
||||||
|
def test_downloads_are_offered(self):
|
||||||
|
played = self.episode(self.feed, 'played')
|
||||||
|
known = self.episode(self.feed, 'known')
|
||||||
|
self.ap.item(self.ap_feed, 'played', read=1, pub=T0 + timedelta(days=1), downloaded=1, size=30_000_000)
|
||||||
|
self.ap.item(self.ap_feed, 'known', read=0, pub=T0 + timedelta(days=3), downloaded=ms(T0), size=20_400_000)
|
||||||
|
self.ap.item(self.ap_feed, 'on-phone', read=0, pub=T0 + timedelta(days=2), downloaded=1,
|
||||||
|
size=27_634_898_567_168) # garbage: unknown
|
||||||
|
self.ap.item(self.ap_feed, 'not-downloaded', read=0)
|
||||||
|
self.ap.item(self.ap_feed, 'local', read=0, audio='antennapod_local:x', downloaded=1, size=5)
|
||||||
|
self.ap.item(self.ap.feed('antennapod_local:folder'), 'file', read=0, downloaded=1, size=7)
|
||||||
|
|
||||||
|
before = self.snapshot()
|
||||||
|
out = self.run_import('--dry-run')
|
||||||
|
self.assertEqual(self.snapshot(), before)
|
||||||
|
self.assertFalse(PodcastSettings.objects.filter(user=self.user).exists())
|
||||||
|
self.assertIn('Downloads: 3 on the phone in AntennaPod (50 MB) -> offered to the app for download '
|
||||||
|
'(1 episodes created for them); 2 not found (feeds not imported, no audio URL)\n', out)
|
||||||
|
|
||||||
|
out = self.run_import()
|
||||||
|
created = PodcastEpisode.objects.get(feed=self.feed, guid='on-phone')
|
||||||
|
self.assertEqual(self.offered(), [ # newest first
|
||||||
|
{'episode_id': known.id, 'size': 20_400_000}, {'episode_id': created.id, 'size': 0},
|
||||||
|
{'episode_id': played.id, 'size': 30_000_000}])
|
||||||
|
# Created only to be downloaded; not new in AntennaPod, so not in the inbox.
|
||||||
|
p = self.progress(created)
|
||||||
|
self.assertEqual((p.played, p.dismissed), (False, True))
|
||||||
|
self.assertFalse(PodcastEpisode.objects.filter(guid__in=['not-downloaded', 'local', 'file']).exists())
|
||||||
|
self.assertIn('Episodes: 1 with a state: 1 matched by guid, 0 by URL, 0 created, '
|
||||||
|
'0 skipped (no audio URL), 0 in skipped feeds', out)
|
||||||
|
|
||||||
|
# Another backup later: its downloads replace the list.
|
||||||
|
self.ap.db.execute('UPDATE FeedMedia SET downloaded = 0 WHERE filesize != 30000000')
|
||||||
|
self.ap.db.commit()
|
||||||
|
out = self.run_import()
|
||||||
|
self.assertEqual(self.offered(), [{'episode_id': played.id, 'size': 30_000_000}])
|
||||||
|
self.assertIn('Downloads: 1 on the phone in AntennaPod (30 MB) -> offered to the app for download\n', out)
|
||||||
|
self.ap.db.execute('UPDATE FeedMedia SET downloaded = 0')
|
||||||
|
self.ap.db.commit()
|
||||||
|
out = self.run_import()
|
||||||
|
self.assertEqual(self.offered(), [])
|
||||||
|
self.assertIn('Downloads: 0 on the phone in AntennaPod (0 MB)', out)
|
||||||
|
|
||||||
|
def test_one_entry_per_episode(self):
|
||||||
|
ep = self.episode(self.feed, 'g')
|
||||||
|
self.ap.item(self.ap_feed, 'g', read=1, downloaded=1, size=10)
|
||||||
|
self.ap.item(self.ap.feed('http://www.example.org/feed/'), 'g', read=1, downloaded=1, size=10)
|
||||||
|
self.run_import()
|
||||||
|
self.assertEqual(self.offered(), [{'episode_id': ep.id, 'size': 10}])
|
||||||
|
|
||||||
|
def test_backup_without_download_columns(self):
|
||||||
|
self.ap.db.executescript('ALTER TABLE FeedMedia DROP COLUMN downloaded; '
|
||||||
|
'ALTER TABLE FeedMedia DROP COLUMN filesize;')
|
||||||
|
self.ap.db.execute('INSERT INTO FeedItems (title, read, feed, item_identifier) VALUES (?,?,?,?)',
|
||||||
|
('G', 1, self.ap_feed, 'g'))
|
||||||
|
self.ap.db.commit()
|
||||||
|
PodcastSettings.objects.create(user=self.user, import_downloads=[{'episode_id': 1, 'size': 1}])
|
||||||
|
out = self.run_import()
|
||||||
|
self.assertEqual(self.offered(), [])
|
||||||
|
self.assertIn('Downloads: 0 on the phone', out)
|
||||||
|
|
||||||
|
|
||||||
|
class ImportDownloadsViewTests(TestCase):
|
||||||
|
"""GET import-downloads/, its clear/, and the count in state/."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(username='alice', password='pw12345678')
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
feed = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/a', title='A')
|
||||||
|
self.ep = PodcastEpisode.objects.create(feed=feed, guid='e', title='E', audio_url='https://example.org/e.mp3',
|
||||||
|
duration_seconds=1800, pub_date=T0)
|
||||||
|
EpisodeProgress.objects.create(user=self.user, episode=self.ep, position_seconds=120)
|
||||||
|
no_audio = PodcastEpisode.objects.create(feed=feed, guid='n', title='N', audio_url='')
|
||||||
|
bob = User.objects.create_user(username='bob', password='pw12345678')
|
||||||
|
bobs_feed = PodcastFeed.objects.create(user=bob, rss_url='https://example.org/a', title='A')
|
||||||
|
bobs = PodcastEpisode.objects.create(feed=bobs_feed, guid='b', title='B', audio_url='https://example.org/b.mp3')
|
||||||
|
gone_feed = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/gone', title='Gone')
|
||||||
|
gone = PodcastEpisode.objects.create(feed=gone_feed, guid='g', title='G', audio_url='https://example.org/g.mp3')
|
||||||
|
gone_feed.delete() # unsubscribed since the import
|
||||||
|
PodcastSettings.objects.create(user=self.user, import_downloads=[
|
||||||
|
{'episode_id': i, 'size': 1000 + n} for n, i in enumerate((bobs.id, self.ep.id, no_audio.id, gone.id))])
|
||||||
|
|
||||||
|
def test_offered_episodes(self):
|
||||||
|
data = self.client.get('/podcasts/import-downloads/').json()
|
||||||
|
self.assertEqual(data['bytes'], 1001)
|
||||||
|
[e] = data['episodes']
|
||||||
|
self.assertEqual((e['id'], e['title'], e['feed__title'], e['duration_seconds'], e['pub_date'],
|
||||||
|
e['position_seconds'], e['in_queue'], e['size']),
|
||||||
|
(self.ep.id, 'E', 'A', 1800, T0.isoformat(), 120, False, 1001))
|
||||||
|
# The same keys as an inbox entry of state/, plus the size.
|
||||||
|
state = self.client.get('/podcasts/state/').json()
|
||||||
|
self.assertEqual(set(e), set(state['inbox'][0]) | {'size'})
|
||||||
|
self.assertEqual(state['import_downloads'], {'count': 1, 'bytes': 1001})
|
||||||
|
|
||||||
|
def test_clear(self):
|
||||||
|
self.assertEqual(self.client.get('/podcasts/import-downloads/clear/').status_code, 405)
|
||||||
|
r = self.client.post('/podcasts/import-downloads/clear/', '{}', content_type='application/json')
|
||||||
|
self.assertEqual(r.json(), {'ok': True})
|
||||||
|
self.assertEqual(PodcastSettings.objects.get(user=self.user).import_downloads, [])
|
||||||
|
self.assertEqual(self.client.get('/podcasts/import-downloads/').json(), {'episodes': [], 'bytes': 0})
|
||||||
|
self.assertEqual(self.client.get('/podcasts/state/').json()['import_downloads'], {'count': 0, 'bytes': 0})
|
||||||
|
|
||||||
|
def test_other_user_and_login(self):
|
||||||
|
self.client.force_login(User.objects.get(username='bob'))
|
||||||
|
self.assertEqual(self.client.get('/podcasts/import-downloads/').json(), {'episodes': [], 'bytes': 0})
|
||||||
|
self.assertEqual(self.client.get('/podcasts/state/').json()['import_downloads'], {'count': 0, 'bytes': 0})
|
||||||
|
self.client.logout()
|
||||||
|
self.assertEqual(self.client.get('/podcasts/import-downloads/').status_code, 401)
|
||||||
|
self.assertEqual(self.client.post('/podcasts/import-downloads/clear/').status_code, 401)
|
||||||
|
self.assertEqual(len(PodcastSettings.objects.get(user=self.user).import_downloads), 4)
|
||||||
|
|
||||||
|
|
||||||
class AfterFirstRefreshTests(ImportTestCase):
|
class AfterFirstRefreshTests(ImportTestCase):
|
||||||
"""The way to move: import, let the new feeds be fetched, import again."""
|
"""The way to move: import, let the new feeds be fetched, import again."""
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -28,4 +28,6 @@ urlpatterns = [
|
||||||
path('progress/mark-played/', views.mark_played, name='podcast_mark_played'),
|
path('progress/mark-played/', views.mark_played, name='podcast_mark_played'),
|
||||||
path('progress/dismiss/', views.dismiss_episodes, name='podcast_dismiss_episodes'),
|
path('progress/dismiss/', views.dismiss_episodes, name='podcast_dismiss_episodes'),
|
||||||
path('inbox/', views.inbox, name='podcast_inbox'),
|
path('inbox/', views.inbox, name='podcast_inbox'),
|
||||||
|
path('import-downloads/', views.import_downloads, name='podcast_import_downloads'),
|
||||||
|
path('import-downloads/clear/', views.clear_import_downloads, name='podcast_clear_import_downloads'),
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -1436,7 +1436,8 @@ def refresh_all_feeds(request):
|
||||||
@csrf_exempt
|
@csrf_exempt
|
||||||
def podcast_state(request):
|
def podcast_state(request):
|
||||||
"""Everything a client shows first, in one response: feeds, queue, the
|
"""Everything a client shows first, in one response: feeds, queue, the
|
||||||
first page of the inbox and the settings."""
|
first page of the inbox, the settings and how many AntennaPod downloads
|
||||||
|
wait to be offered (import-downloads/)."""
|
||||||
if not request.user.is_authenticated:
|
if not request.user.is_authenticated:
|
||||||
return JsonResponse({'error': 'authentication required'}, status=401)
|
return JsonResponse({'error': 'authentication required'}, status=401)
|
||||||
try:
|
try:
|
||||||
|
|
@ -1444,12 +1445,14 @@ def podcast_state(request):
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return JsonResponse({'error': 'inbox_limit must be a number'}, status=400)
|
return JsonResponse({'error': 'inbox_limit must be a number'}, status=400)
|
||||||
s = PodcastSettings.for_user(request.user)
|
s = PodcastSettings.for_user(request.user)
|
||||||
|
downloads = _import_downloads(request.user, s.import_downloads)
|
||||||
return JsonResponse({
|
return JsonResponse({
|
||||||
'feeds': _feeds_data(request.user),
|
'feeds': _feeds_data(request.user),
|
||||||
'queue': _queue_data(request.user),
|
'queue': _queue_data(request.user),
|
||||||
'inbox': _inbox_data(request.user, 0, limit, with_description=False),
|
'inbox': _inbox_data(request.user, 0, limit, with_description=False),
|
||||||
'settings': {'enqueue_position': s.enqueue_position, 'auto_queue_position': s.auto_queue_position,
|
'settings': {'enqueue_position': s.enqueue_position, 'auto_queue_position': s.auto_queue_position,
|
||||||
'keep_sorted': s.keep_sorted, 'auto_download_default': s.auto_download_default},
|
'keep_sorted': s.keep_sorted, 'auto_download_default': s.auto_download_default},
|
||||||
|
'import_downloads': {'count': len(downloads), 'bytes': sum(e['size'] for e in downloads)},
|
||||||
'server_time': timezone.now().isoformat(),
|
'server_time': timezone.now().isoformat(),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -1730,29 +1733,17 @@ def inbox(request):
|
||||||
return JsonResponse({'episodes': _inbox_data(request.user, offset, limit), 'offset': offset, 'limit': limit})
|
return JsonResponse({'episodes': _inbox_data(request.user, offset, limit), 'offset': offset, 'limit': limit})
|
||||||
|
|
||||||
|
|
||||||
def _inbox_data(user, offset, limit, with_description=True):
|
def _inbox_fields(with_description=True):
|
||||||
"""Inbox episodes. Shownotes can be large (up to 50 KB each); the apps'
|
|
||||||
state/ leaves them out and loads one episode's at a time."""
|
|
||||||
fields = ['id', 'title', 'audio_url', 'duration_seconds', 'pub_date', 'artwork_url',
|
fields = ['id', 'title', 'audio_url', 'duration_seconds', 'pub_date', 'artwork_url',
|
||||||
'feed__id', 'feed__title', 'feed__artwork_url']
|
'feed__id', 'feed__title', 'feed__artwork_url']
|
||||||
if with_description:
|
if with_description:
|
||||||
fields.insert(2, 'description')
|
fields.insert(2, 'description')
|
||||||
hidden_ids = set(
|
return fields
|
||||||
EpisodeProgress.objects.filter(
|
|
||||||
user=user,
|
|
||||||
).filter(
|
|
||||||
Q(played=True) | Q(dismissed=True)
|
|
||||||
).values_list('episode_id', flat=True)
|
|
||||||
)
|
|
||||||
|
|
||||||
episodes = list(
|
|
||||||
PodcastEpisode.objects.filter(feed__user=user)
|
|
||||||
.exclude(id__in=hidden_ids)
|
|
||||||
.select_related('feed')
|
|
||||||
.order_by('-pub_date')
|
|
||||||
.values(*fields)[offset:offset + limit]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
def _add_user_state(user, episodes):
|
||||||
|
"""Complete inbox-style episode dicts (_inbox_fields) with the user's
|
||||||
|
position and whether they are queued."""
|
||||||
progress_map = {
|
progress_map = {
|
||||||
ep['episode_id']: ep
|
ep['episode_id']: ep
|
||||||
for ep in EpisodeProgress.objects.filter(
|
for ep in EpisodeProgress.objects.filter(
|
||||||
|
|
@ -1773,3 +1764,68 @@ def _inbox_data(user, offset, limit, with_description=True):
|
||||||
ep['position_seconds'] = prog.get('position_seconds', 0)
|
ep['position_seconds'] = prog.get('position_seconds', 0)
|
||||||
ep['in_queue'] = ep['id'] in queued_ids
|
ep['in_queue'] = ep['id'] in queued_ids
|
||||||
return episodes
|
return episodes
|
||||||
|
|
||||||
|
|
||||||
|
def _inbox_data(user, offset, limit, with_description=True):
|
||||||
|
"""Inbox episodes. Shownotes can be large (up to 50 KB each); the apps'
|
||||||
|
state/ leaves them out and loads one episode's at a time."""
|
||||||
|
hidden_ids = set(
|
||||||
|
EpisodeProgress.objects.filter(
|
||||||
|
user=user,
|
||||||
|
).filter(
|
||||||
|
Q(played=True) | Q(dismissed=True)
|
||||||
|
).values_list('episode_id', flat=True)
|
||||||
|
)
|
||||||
|
|
||||||
|
episodes = list(
|
||||||
|
PodcastEpisode.objects.filter(feed__user=user)
|
||||||
|
.exclude(id__in=hidden_ids)
|
||||||
|
.select_related('feed')
|
||||||
|
.order_by('-pub_date')
|
||||||
|
.values(*_inbox_fields(with_description))[offset:offset + limit]
|
||||||
|
)
|
||||||
|
return _add_user_state(user, episodes)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Episodes downloaded in AntennaPod (antennapod.py): offered once for download
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _import_downloads(user, entries, fields=('id',)):
|
||||||
|
"""The entries of PodcastSettings.import_downloads still worth offering
|
||||||
|
(the episode is still there, the user's, with an audio URL), in their
|
||||||
|
order: dicts of `fields` plus 'size' (bytes, 0 = unknown)."""
|
||||||
|
sizes = {}
|
||||||
|
for e in entries or []:
|
||||||
|
if isinstance(e, dict) and isinstance(e.get('episode_id'), int):
|
||||||
|
size = e.get('size')
|
||||||
|
sizes.setdefault(e['episode_id'], size if isinstance(size, int) and size > 0 else 0)
|
||||||
|
ids = list(sizes)
|
||||||
|
found = {}
|
||||||
|
for i in range(0, len(ids), 500):
|
||||||
|
for ep in (PodcastEpisode.objects.filter(id__in=ids[i:i + 500], feed__user=user)
|
||||||
|
.exclude(audio_url='').values(*fields)):
|
||||||
|
found[ep['id']] = ep
|
||||||
|
return [{**found[ep_id], 'size': sizes[ep_id]} for ep_id in ids if ep_id in found]
|
||||||
|
|
||||||
|
|
||||||
|
@csrf_exempt
|
||||||
|
@require_http_methods(['GET'])
|
||||||
|
def import_downloads(request):
|
||||||
|
"""The episodes that were on the phone in AntennaPod, inbox-style plus
|
||||||
|
their size; the app offers once to download them again."""
|
||||||
|
if not request.user.is_authenticated:
|
||||||
|
return JsonResponse({'error': 'authentication required'}, status=401)
|
||||||
|
entries = PodcastSettings.objects.filter(user=request.user).values_list('import_downloads', flat=True).first()
|
||||||
|
episodes = _add_user_state(request.user, _import_downloads(request.user, entries, _inbox_fields(False)))
|
||||||
|
return JsonResponse({'episodes': episodes, 'bytes': sum(e['size'] for e in episodes)})
|
||||||
|
|
||||||
|
|
||||||
|
@csrf_exempt
|
||||||
|
@require_http_methods(['POST'])
|
||||||
|
def clear_import_downloads(request):
|
||||||
|
"""Offered (downloaded or declined): the list is done with."""
|
||||||
|
if not request.user.is_authenticated:
|
||||||
|
return JsonResponse({'error': 'authentication required'}, status=401)
|
||||||
|
PodcastSettings.objects.filter(user=request.user).update(import_downloads=[])
|
||||||
|
return JsonResponse({'ok': True})
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue