Compare commits
4 commits
35ea42b6c2
...
561776556f
| Author | SHA1 | Date | |
|---|---|---|---|
| 561776556f | |||
| 8ba82a0e55 | |||
| fdea47baef | |||
| 5b2d25e2b2 |
9 changed files with 425 additions and 57 deletions
|
|
@ -2,12 +2,14 @@
|
||||||
|
|
||||||
AntennaPod's "Datenbank exportieren" writes its whole SQLite database. From
|
AntennaPod's "Datenbank exportieren" writes its whole SQLite database. From
|
||||||
it come the subscriptions with their settings, the state of every episode
|
it come the subscriptions with their settings, the state of every episode
|
||||||
that has one (played, new, started, queued, favourite) and the queue. See
|
that has one (played, new, started, queued, favourite), the queue and which
|
||||||
|
episodes were on the phone (the app offers to download those again). See
|
||||||
the import_antennapod command for how to run it.
|
the import_antennapod command for how to run it.
|
||||||
|
|
||||||
The backup is only read. Episodes without any state are not imported: the
|
The backup is only read. Episodes without any state (and not on the phone)
|
||||||
feed's current episodes come with its next refresh anyway. Those that diora
|
are not imported: the feed's current episodes come with its next refresh
|
||||||
already has, though, are taken out of the inbox (AntennaPod: not new).
|
anyway. Those that diora already has, though, are taken out of the inbox
|
||||||
|
(AntennaPod: not new).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
|
@ -37,6 +39,8 @@ AUTO_DOWNLOAD_OFF, AUTO_DOWNLOAD_GLOBAL, AUTO_DOWNLOAD_ON = 0, 1, 2
|
||||||
# FeedItems.read
|
# FeedItems.read
|
||||||
NEW, UNPLAYED, PLAYED = -1, 0, 1
|
NEW, UNPLAYED, PLAYED = -1, 0, 1
|
||||||
SUBSCRIBED = 0 # Feeds.state
|
SUBSCRIBED = 0 # Feeds.state
|
||||||
|
# FeedMedia.filesize above this is garbage (seen: 10^13); taken as unknown.
|
||||||
|
MAX_FILESIZE = 4_000_000_000
|
||||||
|
|
||||||
SETTING_FIELDS = ('playback_speed', 'skip_intro', 'skip_outro', 'volume', 'include_words',
|
SETTING_FIELDS = ('playback_speed', 'skip_intro', 'skip_outro', 'volume', 'include_words',
|
||||||
'exclude_words', 'min_duration', 'new_episodes', 'auto_download')
|
'exclude_words', 'min_duration', 'new_episodes', 'auto_download')
|
||||||
|
|
@ -250,6 +254,12 @@ class Stats:
|
||||||
keep_sorted: str = ''
|
keep_sorted: str = ''
|
||||||
inbox_before: int = 0
|
inbox_before: int = 0
|
||||||
inbox_after: int = 0
|
inbox_after: int = 0
|
||||||
|
downloads_backup: int = 0
|
||||||
|
downloads: int = 0
|
||||||
|
downloads_bytes: int = 0
|
||||||
|
downloads_created: int = 0
|
||||||
|
downloads_missing: int = 0
|
||||||
|
downloads_offered: bool = True
|
||||||
backup_date: datetime | None = None
|
backup_date: datetime | None = None
|
||||||
seconds: float = 0.0
|
seconds: float = 0.0
|
||||||
|
|
||||||
|
|
@ -282,11 +292,13 @@ def _columns(db, table):
|
||||||
|
|
||||||
|
|
||||||
class AntennaPodImport:
|
class AntennaPodImport:
|
||||||
def __init__(self, user, path, backup_date, global_auto_download=False):
|
def __init__(self, user, path, backup_date, global_auto_download=False, offer_downloads=True):
|
||||||
self.user = user
|
self.user = user
|
||||||
self.path = path
|
self.path = path
|
||||||
self.backup_date = backup_date
|
self.backup_date = backup_date
|
||||||
self.global_auto_download = global_auto_download
|
self.global_auto_download = global_auto_download
|
||||||
|
# False: the app doesn't offer AntennaPod's downloads (they stay in AntennaPod).
|
||||||
|
self.offer_downloads = offer_downloads
|
||||||
self.stats = Stats(backup_date=backup_date)
|
self.stats = Stats(backup_date=backup_date)
|
||||||
|
|
||||||
def run(self, dry_run=False):
|
def run(self, dry_run=False):
|
||||||
|
|
@ -325,9 +337,11 @@ class AntennaPodImport:
|
||||||
st.queue_backup = len(queue_items)
|
st.queue_backup = len(queue_items)
|
||||||
st.favorites_backup = len(favorite_items)
|
st.favorites_backup = len(favorite_items)
|
||||||
|
|
||||||
states, not_new, item_episode = self._import_episodes(db, feed_map, set(queue_items), favorite_items)
|
states, not_new, item_episode, downloads = self._import_episodes(
|
||||||
|
db, feed_map, set(queue_items), favorite_items)
|
||||||
self._import_progress(states, not_new)
|
self._import_progress(states, not_new)
|
||||||
self._import_queue(queue_items, item_episode)
|
self._import_queue(queue_items, item_episode)
|
||||||
|
self._import_downloads(settings, downloads, item_episode)
|
||||||
|
|
||||||
st.queue_after = len(_queue_order(user))
|
st.queue_after = len(_queue_order(user))
|
||||||
st.inbox_after = _inbox_count(user)
|
st.inbox_after = _inbox_count(user)
|
||||||
|
|
@ -410,20 +424,25 @@ class AntennaPodImport:
|
||||||
|
|
||||||
def _import_episodes(self, db, feed_map, queued, favorites):
|
def _import_episodes(self, db, feed_map, queued, favorites):
|
||||||
"""Match AntennaPod's episodes to diora's, create the missing ones
|
"""Match AntennaPod's episodes to diora's, create the missing ones
|
||||||
that have a state. Returns ({episode id: ItemState} for those with a
|
that have a state (or were downloaded). Returns ({episode id: ItemState}
|
||||||
state, {episode ids} of matched ones without (not new), {AntennaPod
|
for those with a state, {episode ids} of matched ones without (not new),
|
||||||
item id: episode id})."""
|
{AntennaPod item id: episode id}, [(pub date ms, AntennaPod item id,
|
||||||
|
size)] of the downloaded ones)."""
|
||||||
st = self.stats
|
st = self.stats
|
||||||
backup = self.backup_date
|
backup = self.backup_date
|
||||||
states = {}
|
states = {}
|
||||||
not_new = set()
|
not_new = set()
|
||||||
item_episode = {}
|
item_episode = {}
|
||||||
|
downloads = []
|
||||||
to_create = {} # (diora feed id, guid) -> [ItemState, AntennaPod item id, audio url, duration, pub_date]
|
to_create = {} # (diora feed id, guid) -> [ItemState, AntennaPod item id, audio url, duration, pub_date]
|
||||||
|
|
||||||
|
media = _columns(db, 'FeedMedia')
|
||||||
cur = db.execute(
|
cur = db.execute(
|
||||||
'SELECT i.feed, i.id, i.item_identifier, i.read, i.pubDate, m.download_url, m.duration, '
|
'SELECT i.feed, i.id, i.item_identifier, i.read, i.pubDate, m.download_url, m.duration, '
|
||||||
'm.position, m.last_played_time, m.playback_completion_date '
|
'm.position, m.last_played_time, m.playback_completion_date, '
|
||||||
'FROM FeedItems i LEFT JOIN FeedMedia m ON m.feeditem = i.id ORDER BY i.feed, i.id'
|
+ ('m.downloaded, ' if 'downloaded' in media else '0, ')
|
||||||
|
+ ('m.filesize ' if 'filesize' in media else '0 ')
|
||||||
|
+ 'FROM FeedItems i LEFT JOIN FeedMedia m ON m.feeditem = i.id ORDER BY i.feed, i.id'
|
||||||
)
|
)
|
||||||
|
|
||||||
def rows():
|
def rows():
|
||||||
|
|
@ -436,12 +455,19 @@ class AntennaPodImport:
|
||||||
for ap_feed, items in groupby(rows(), key=lambda r: r[0]):
|
for ap_feed, items in groupby(rows(), key=lambda r: r[0]):
|
||||||
feed_id = feed_map.get(ap_feed)
|
feed_id = feed_map.get(ap_feed)
|
||||||
by_guid = by_url = None
|
by_guid = by_url = None
|
||||||
for (_, item_id, ident, read, pub_ms, audio, duration, position, last_played, completed) in items:
|
for (_, item_id, ident, read, pub_ms, audio, duration, position, last_played, completed,
|
||||||
|
downloaded, size) in items:
|
||||||
position = max(int(position or 0), 0) // 1000
|
position = max(int(position or 0), 0) // 1000
|
||||||
activity = max(filter(None, (_ms(last_played), _ms(completed))), default=None)
|
activity = max(filter(None, (_ms(last_played), _ms(completed))), default=None)
|
||||||
in_queue = item_id in queued
|
in_queue = item_id in queued
|
||||||
favorite = item_id in favorites
|
favorite = item_id in favorites
|
||||||
stateful = read in (NEW, PLAYED) or position > 0 or activity is not None or in_queue or favorite
|
stateful = read in (NEW, PLAYED) or position > 0 or activity is not None or in_queue or favorite
|
||||||
|
# FeedMedia.downloaded: when the file was downloaded (ms; 1 from
|
||||||
|
# older versions), 0 = not on the phone.
|
||||||
|
downloaded = (downloaded or 0) > 0
|
||||||
|
if downloaded:
|
||||||
|
size = int(size or 0)
|
||||||
|
downloads.append((int(pub_ms or 0), item_id, size if 0 < size <= MAX_FILESIZE else 0))
|
||||||
if feed_id is None:
|
if feed_id is None:
|
||||||
st.in_skipped_feeds += stateful
|
st.in_skipped_feeds += stateful
|
||||||
continue
|
continue
|
||||||
|
|
@ -463,9 +489,14 @@ class AntennaPodImport:
|
||||||
if ep_id is not None:
|
if ep_id is not None:
|
||||||
not_new.add(ep_id)
|
not_new.add(ep_id)
|
||||||
item_episode[item_id] = ep_id
|
item_episode[item_id] = ep_id
|
||||||
continue
|
continue
|
||||||
|
if not downloaded:
|
||||||
st.stateful += 1
|
continue
|
||||||
|
# On the phone in AntennaPod but not in diora yet (a feed
|
||||||
|
# not fetched so far): created as not new, so the app can
|
||||||
|
# offer to download it again.
|
||||||
|
else:
|
||||||
|
st.stateful += 1
|
||||||
played = read == PLAYED
|
played = read == PLAYED
|
||||||
pub_date = _ms(pub_ms)
|
pub_date = _ms(pub_ms)
|
||||||
state = ItemState(
|
state = ItemState(
|
||||||
|
|
@ -484,7 +515,7 @@ class AntennaPodImport:
|
||||||
item_episode[item_id] = ep_id
|
item_episode[item_id] = ep_id
|
||||||
continue
|
continue
|
||||||
if normalize_url(audio) is None or len(audio) > 1000:
|
if normalize_url(audio) is None or len(audio) > 1000:
|
||||||
st.no_audio += 1
|
st.no_audio += stateful
|
||||||
continue
|
continue
|
||||||
key = (feed_id, guid)
|
key = (feed_id, guid)
|
||||||
if key in to_create:
|
if key in to_create:
|
||||||
|
|
@ -493,12 +524,13 @@ class AntennaPodImport:
|
||||||
pending[1].append(item_id)
|
pending[1].append(item_id)
|
||||||
else:
|
else:
|
||||||
to_create[key] = [state, [item_id], audio, max(int(duration or 0), 0) // 1000, pub_date]
|
to_create[key] = [state, [item_id], audio, max(int(duration or 0), 0) // 1000, pub_date]
|
||||||
|
st.downloads_created += not stateful
|
||||||
|
|
||||||
if to_create:
|
if to_create:
|
||||||
self._create_episodes(db, to_create, states, item_episode)
|
self._create_episodes(db, to_create, states, item_episode)
|
||||||
not_new -= states.keys()
|
not_new -= states.keys()
|
||||||
st.stateless_matched = len(not_new)
|
st.stateless_matched = len(not_new)
|
||||||
return states, not_new, item_episode
|
return states, not_new, item_episode, downloads
|
||||||
|
|
||||||
def _create_episodes(self, db, to_create, states, item_episode):
|
def _create_episodes(self, db, to_create, states, item_episode):
|
||||||
st = self.stats
|
st = self.stats
|
||||||
|
|
@ -522,7 +554,7 @@ class AntennaPodImport:
|
||||||
duration_seconds=duration, pub_date=pub_date, artwork_url=_web_url(image),
|
duration_seconds=duration, pub_date=pub_date, artwork_url=_web_url(image),
|
||||||
))
|
))
|
||||||
PodcastEpisode.objects.bulk_create(batch)
|
PodcastEpisode.objects.bulk_create(batch)
|
||||||
st.episodes_created = len(to_create)
|
st.episodes_created = len(to_create) - st.downloads_created
|
||||||
|
|
||||||
by_feed = {}
|
by_feed = {}
|
||||||
for feed_id, guid in to_create:
|
for feed_id, guid in to_create:
|
||||||
|
|
@ -627,6 +659,30 @@ class AntennaPodImport:
|
||||||
if _keep_sorted(user):
|
if _keep_sorted(user):
|
||||||
st.keep_sorted = PodcastSettings.objects.get(user=user).keep_sorted
|
st.keep_sorted = PodcastSettings.objects.get(user=user).keep_sorted
|
||||||
|
|
||||||
|
# -- downloads ----------------------------------------------------------
|
||||||
|
|
||||||
|
def _import_downloads(self, settings, downloads, item_episode):
|
||||||
|
"""The episodes AntennaPod had on the phone, newest first: the app
|
||||||
|
can't take its files over, so it offers to download them again.
|
||||||
|
Replaces what an earlier import left."""
|
||||||
|
st = self.stats
|
||||||
|
st.downloads_backup = len(downloads)
|
||||||
|
entries, seen = [], set()
|
||||||
|
for _, item_id, size in sorted(downloads, key=lambda d: (-d[0], -d[1])):
|
||||||
|
ep_id = item_episode.get(item_id)
|
||||||
|
if ep_id is None:
|
||||||
|
st.downloads_missing += 1
|
||||||
|
continue
|
||||||
|
if ep_id in seen:
|
||||||
|
continue
|
||||||
|
seen.add(ep_id)
|
||||||
|
entries.append({'episode_id': ep_id, 'size': size})
|
||||||
|
st.downloads = len(entries)
|
||||||
|
st.downloads_bytes = sum(e['size'] for e in entries)
|
||||||
|
st.downloads_offered = self.offer_downloads
|
||||||
|
settings.import_downloads = entries if self.offer_downloads else []
|
||||||
|
settings.save(update_fields=['import_downloads'])
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Summary
|
# Summary
|
||||||
|
|
@ -670,6 +726,10 @@ def summary_lines(st, username, path, dry_run, global_auto_download):
|
||||||
f'{st.queue_after} after ({st.queue_appended} diora entries after AntennaPod\'s)'
|
f'{st.queue_after} after ({st.queue_appended} diora entries after AntennaPod\'s)'
|
||||||
+ (f'; kept sorted by "{st.keep_sorted}"' if st.keep_sorted else ''),
|
+ (f'; kept sorted by "{st.keep_sorted}"' if st.keep_sorted else ''),
|
||||||
f'Inbox: {st.inbox_before} before, {st.inbox_after} after',
|
f'Inbox: {st.inbox_before} before, {st.inbox_after} after',
|
||||||
|
f'Downloads: {st.downloads} on the phone in AntennaPod ({st.downloads_bytes / 1_000_000:.0f} MB) '
|
||||||
|
+ ('-> offered to the app for download' if st.downloads_offered else '-> not offered (--no-downloads)')
|
||||||
|
+ (f' ({st.downloads_created} episodes created for them)' if st.downloads_created else '')
|
||||||
|
+ (f'; {st.downloads_missing} not found (feeds not imported, no audio URL)' if st.downloads_missing else ''),
|
||||||
f'Created feeds ({len(st.created_titles)}):',
|
f'Created feeds ({len(st.created_titles)}):',
|
||||||
*(f' - {t}' for t in st.created_titles),
|
*(f' - {t}' for t in st.created_titles),
|
||||||
f'Skipped feeds ({len(st.skipped_feeds)}):',
|
f'Skipped feeds ({len(st.skipped_feeds)}):',
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,8 @@ from podcasts.antennapod import AntennaPodImport, AntennaPodImportError, summary
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
help = (
|
help = (
|
||||||
"Import an AntennaPod database backup (Settings > Database > Export) into one user's podcasts: "
|
"Import an AntennaPod database backup (Settings > Database > Export) into one user's podcasts: "
|
||||||
'subscriptions with their settings, played/new/started episodes, favourites and the queue. '
|
'subscriptions with their settings, played/new/started episodes, favourites and the queue; '
|
||||||
|
'the episodes downloaded in AntennaPod are offered to the app for download. '
|
||||||
'Feeds it creates are fetched by the next refresh; running the import once more after that '
|
'Feeds it creates are fetched by the next refresh; running the import once more after that '
|
||||||
'also takes their older episodes that were not new in AntennaPod out of the inbox. '
|
'also takes their older episodes that were not new in AntennaPod out of the inbox. '
|
||||||
'Running it twice changes nothing more.'
|
'Running it twice changes nothing more.'
|
||||||
|
|
@ -24,6 +25,8 @@ class Command(BaseCommand):
|
||||||
help='Do everything inside a transaction that is rolled back, print the summary')
|
help='Do everything inside a transaction that is rolled back, print the summary')
|
||||||
parser.add_argument('--global-auto-download', choices=('on', 'off'), default='off',
|
parser.add_argument('--global-auto-download', choices=('on', 'off'), default='off',
|
||||||
help="Auto-download for feeds that use AntennaPod's global setting (default off)")
|
help="Auto-download for feeds that use AntennaPod's global setting (default off)")
|
||||||
|
parser.add_argument('--no-downloads', action='store_true',
|
||||||
|
help="Don't offer AntennaPod's downloaded episodes to the app (they stay in AntennaPod)")
|
||||||
parser.add_argument('--backup-date',
|
parser.add_argument('--backup-date',
|
||||||
help='When the backup was made (ISO date/time, UTC if no zone); default: the file date. '
|
help='When the backup was made (ISO date/time, UTC if no zone); default: the file date. '
|
||||||
'diora changes after it win over the backup.')
|
'diora changes after it win over the backup.')
|
||||||
|
|
@ -49,7 +52,8 @@ class Command(BaseCommand):
|
||||||
|
|
||||||
global_auto_download = options['global_auto_download'] == 'on'
|
global_auto_download = options['global_auto_download'] == 'on'
|
||||||
try:
|
try:
|
||||||
stats = AntennaPodImport(user, path, backup_date, global_auto_download).run(dry_run=options['dry_run'])
|
stats = AntennaPodImport(user, path, backup_date, global_auto_download,
|
||||||
|
offer_downloads=not options['no_downloads']).run(dry_run=options['dry_run'])
|
||||||
except AntennaPodImportError as e:
|
except AntennaPodImportError as e:
|
||||||
raise CommandError(str(e))
|
raise CommandError(str(e))
|
||||||
for line in summary_lines(stats, user.username, path, options['dry_run'], global_auto_download):
|
for line in summary_lines(stats, user.username, path, options['dry_run'], global_auto_download):
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,11 @@ class Command(BaseCommand):
|
||||||
def add_arguments(self, parser):
|
def add_arguments(self, parser):
|
||||||
parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID (even when backing off)')
|
parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID (even when backing off)')
|
||||||
parser.add_argument('--limit', type=int, default=0, help='Max number of feeds to refresh')
|
parser.add_argument('--limit', type=int, default=0, help='Max number of feeds to refresh')
|
||||||
|
parser.add_argument(
|
||||||
|
'--full', action='store_true',
|
||||||
|
help='Download every feed in full (ignore ETag/Last-Modified) and take in episodes skipped before '
|
||||||
|
'(e.g. video ones); those out before the last refresh go into the list, not into "Neu"',
|
||||||
|
)
|
||||||
|
|
||||||
def handle(self, *args, **options):
|
def handle(self, *args, **options):
|
||||||
qs = PodcastFeed.objects.all().order_by('last_refreshed_at')
|
qs = PodcastFeed.objects.all().order_by('last_refreshed_at')
|
||||||
|
|
@ -30,5 +35,5 @@ class Command(BaseCommand):
|
||||||
return
|
return
|
||||||
|
|
||||||
# Parallel downloads, one per distinct feed URL (see _refresh_feeds).
|
# Parallel downloads, one per distinct feed URL (see _refresh_feeds).
|
||||||
refreshed, failed, new = _refresh_feeds(qs)
|
refreshed, failed, new = _refresh_feeds(qs, full=options['full'])
|
||||||
self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes')
|
self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes')
|
||||||
|
|
|
||||||
18
podcasts/migrations/0011_settings_import_downloads.py
Normal file
18
podcasts/migrations/0011_settings_import_downloads.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 6.1 on 2026-10-02 09:51
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('podcasts', '0010_feed_auto_download'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='podcastsettings',
|
||||||
|
name='import_downloads',
|
||||||
|
field=models.JSONField(blank=True, default=list),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -236,6 +236,10 @@ class PodcastSettings(models.Model):
|
||||||
keep_sorted = models.CharField(max_length=10, choices=SORT_ORDERS, default='', blank=True)
|
keep_sorted = models.CharField(max_length=10, choices=SORT_ORDERS, default='', blank=True)
|
||||||
# What PodcastFeed.auto_download starts as for a new subscription.
|
# What PodcastFeed.auto_download starts as for a new subscription.
|
||||||
auto_download_default = models.BooleanField(default=False)
|
auto_download_default = models.BooleanField(default=False)
|
||||||
|
# Episodes that were downloaded in AntennaPod when its backup was
|
||||||
|
# imported: [{episode_id, size (bytes, 0 = unknown)}]. The app offers once
|
||||||
|
# to download them again, then clears the list.
|
||||||
|
import_downloads = models.JSONField(default=list, blank=True)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def for_user(cls, user):
|
def for_user(cls, user):
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,7 @@ class Backup:
|
||||||
item_identifier TEXT, image_url TEXT);
|
item_identifier TEXT, image_url TEXT);
|
||||||
CREATE TABLE FeedMedia (id INTEGER PRIMARY KEY AUTOINCREMENT, duration INTEGER, download_url TEXT,
|
CREATE TABLE FeedMedia (id INTEGER PRIMARY KEY AUTOINCREMENT, duration INTEGER, download_url TEXT,
|
||||||
position INTEGER, playback_completion_date INTEGER, feeditem INTEGER, played_duration INTEGER,
|
position INTEGER, playback_completion_date INTEGER, feeditem INTEGER, played_duration INTEGER,
|
||||||
last_played_time INTEGER);
|
last_played_time INTEGER, downloaded INTEGER DEFAULT 0, filesize INTEGER DEFAULT 0);
|
||||||
CREATE TABLE Queue (id INTEGER PRIMARY KEY AUTOINCREMENT, feeditem INTEGER, feed INTEGER);
|
CREATE TABLE Queue (id INTEGER PRIMARY KEY AUTOINCREMENT, feeditem INTEGER, feed INTEGER);
|
||||||
CREATE TABLE Favorites (id INTEGER PRIMARY KEY AUTOINCREMENT, feeditem INTEGER, feed INTEGER);
|
CREATE TABLE Favorites (id INTEGER PRIMARY KEY AUTOINCREMENT, feeditem INTEGER, feed INTEGER);
|
||||||
"""
|
"""
|
||||||
|
|
@ -55,7 +55,7 @@ class Backup:
|
||||||
return cur.lastrowid
|
return cur.lastrowid
|
||||||
|
|
||||||
def item(self, feed, guid, read=0, audio=None, pub=T0, duration=1800, position=0, completed=None,
|
def item(self, feed, guid, read=0, audio=None, pub=T0, duration=1800, position=0, completed=None,
|
||||||
last_played=None, title=None, description='', image_url=None, media=True):
|
last_played=None, title=None, description='', image_url=None, media=True, downloaded=0, size=0):
|
||||||
cur = self.db.execute(
|
cur = self.db.execute(
|
||||||
'INSERT INTO FeedItems (title, pubDate, read, description, feed, item_identifier, image_url) '
|
'INSERT INTO FeedItems (title, pubDate, read, description, feed, item_identifier, image_url) '
|
||||||
'VALUES (?,?,?,?,?,?,?)',
|
'VALUES (?,?,?,?,?,?,?)',
|
||||||
|
|
@ -64,9 +64,10 @@ class Backup:
|
||||||
if media:
|
if media:
|
||||||
self.db.execute(
|
self.db.execute(
|
||||||
'INSERT INTO FeedMedia (duration, download_url, position, playback_completion_date, feeditem, '
|
'INSERT INTO FeedMedia (duration, download_url, position, playback_completion_date, feeditem, '
|
||||||
'played_duration, last_played_time) VALUES (?,?,?,?,?,?,?)',
|
'played_duration, last_played_time, downloaded, filesize) VALUES (?,?,?,?,?,?,?,?,?)',
|
||||||
(duration * 1000, audio or f'https://cdn.example.org/{guid}.mp3', position * 1000,
|
(duration * 1000, audio or f'https://cdn.example.org/{guid}.mp3', position * 1000,
|
||||||
ms(completed) if completed else 0, item_id, 0, ms(last_played) if last_played else 0))
|
ms(completed) if completed else 0, item_id, 0, ms(last_played) if last_played else 0,
|
||||||
|
downloaded, size))
|
||||||
self.db.commit()
|
self.db.commit()
|
||||||
return item_id
|
return item_id
|
||||||
|
|
||||||
|
|
@ -444,6 +445,137 @@ class EpisodeImportTests(ImportTestCase):
|
||||||
self.assertIn('Progress: 0 new, 0 updated, 4 unchanged, 0 kept (diora newer); 0 taken out', out)
|
self.assertIn('Progress: 0 new, 0 updated, 4 unchanged, 0 kept (diora newer); 0 taken out', out)
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadImportTests(ImportTestCase):
|
||||||
|
"""Episodes on the phone in AntennaPod: offered to the app for download."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super().setUp()
|
||||||
|
self.feed = self.diora_feed()
|
||||||
|
self.ap_feed = self.ap.feed('https://example.org/feed')
|
||||||
|
|
||||||
|
def offered(self):
|
||||||
|
return PodcastSettings.for_user(self.user).import_downloads
|
||||||
|
|
||||||
|
def test_downloads_are_offered(self):
|
||||||
|
played = self.episode(self.feed, 'played')
|
||||||
|
known = self.episode(self.feed, 'known')
|
||||||
|
self.ap.item(self.ap_feed, 'played', read=1, pub=T0 + timedelta(days=1), downloaded=1, size=30_000_000)
|
||||||
|
self.ap.item(self.ap_feed, 'known', read=0, pub=T0 + timedelta(days=3), downloaded=ms(T0), size=20_400_000)
|
||||||
|
self.ap.item(self.ap_feed, 'on-phone', read=0, pub=T0 + timedelta(days=2), downloaded=1,
|
||||||
|
size=27_634_898_567_168) # garbage: unknown
|
||||||
|
self.ap.item(self.ap_feed, 'not-downloaded', read=0)
|
||||||
|
self.ap.item(self.ap_feed, 'local', read=0, audio='antennapod_local:x', downloaded=1, size=5)
|
||||||
|
self.ap.item(self.ap.feed('antennapod_local:folder'), 'file', read=0, downloaded=1, size=7)
|
||||||
|
|
||||||
|
before = self.snapshot()
|
||||||
|
out = self.run_import('--dry-run')
|
||||||
|
self.assertEqual(self.snapshot(), before)
|
||||||
|
self.assertFalse(PodcastSettings.objects.filter(user=self.user).exists())
|
||||||
|
self.assertIn('Downloads: 3 on the phone in AntennaPod (50 MB) -> offered to the app for download '
|
||||||
|
'(1 episodes created for them); 2 not found (feeds not imported, no audio URL)\n', out)
|
||||||
|
|
||||||
|
out = self.run_import()
|
||||||
|
created = PodcastEpisode.objects.get(feed=self.feed, guid='on-phone')
|
||||||
|
self.assertEqual(self.offered(), [ # newest first
|
||||||
|
{'episode_id': known.id, 'size': 20_400_000}, {'episode_id': created.id, 'size': 0},
|
||||||
|
{'episode_id': played.id, 'size': 30_000_000}])
|
||||||
|
# Created only to be downloaded; not new in AntennaPod, so not in the inbox.
|
||||||
|
p = self.progress(created)
|
||||||
|
self.assertEqual((p.played, p.dismissed), (False, True))
|
||||||
|
self.assertFalse(PodcastEpisode.objects.filter(guid__in=['not-downloaded', 'local', 'file']).exists())
|
||||||
|
self.assertIn('Episodes: 1 with a state: 1 matched by guid, 0 by URL, 0 created, '
|
||||||
|
'0 skipped (no audio URL), 0 in skipped feeds', out)
|
||||||
|
|
||||||
|
# Another backup later: its downloads replace the list.
|
||||||
|
self.ap.db.execute('UPDATE FeedMedia SET downloaded = 0 WHERE filesize != 30000000')
|
||||||
|
self.ap.db.commit()
|
||||||
|
out = self.run_import()
|
||||||
|
self.assertEqual(self.offered(), [{'episode_id': played.id, 'size': 30_000_000}])
|
||||||
|
self.assertIn('Downloads: 1 on the phone in AntennaPod (30 MB) -> offered to the app for download\n', out)
|
||||||
|
self.ap.db.execute('UPDATE FeedMedia SET downloaded = 0')
|
||||||
|
self.ap.db.commit()
|
||||||
|
out = self.run_import()
|
||||||
|
self.assertEqual(self.offered(), [])
|
||||||
|
self.assertIn('Downloads: 0 on the phone in AntennaPod (0 MB)', out)
|
||||||
|
|
||||||
|
def test_no_downloads_offers_nothing(self):
|
||||||
|
self.episode(self.feed, 'known')
|
||||||
|
self.ap.item(self.ap_feed, 'known', read=0, downloaded=1, size=20_400_000)
|
||||||
|
self.run_import()
|
||||||
|
self.assertEqual(len(self.offered()), 1)
|
||||||
|
out = self.run_import('--no-downloads')
|
||||||
|
self.assertEqual(self.offered(), [])
|
||||||
|
self.assertIn('Downloads: 1 on the phone in AntennaPod (20 MB) -> not offered (--no-downloads)\n', out)
|
||||||
|
|
||||||
|
def test_one_entry_per_episode(self):
|
||||||
|
ep = self.episode(self.feed, 'g')
|
||||||
|
self.ap.item(self.ap_feed, 'g', read=1, downloaded=1, size=10)
|
||||||
|
self.ap.item(self.ap.feed('http://www.example.org/feed/'), 'g', read=1, downloaded=1, size=10)
|
||||||
|
self.run_import()
|
||||||
|
self.assertEqual(self.offered(), [{'episode_id': ep.id, 'size': 10}])
|
||||||
|
|
||||||
|
def test_backup_without_download_columns(self):
|
||||||
|
self.ap.db.executescript('ALTER TABLE FeedMedia DROP COLUMN downloaded; '
|
||||||
|
'ALTER TABLE FeedMedia DROP COLUMN filesize;')
|
||||||
|
self.ap.db.execute('INSERT INTO FeedItems (title, read, feed, item_identifier) VALUES (?,?,?,?)',
|
||||||
|
('G', 1, self.ap_feed, 'g'))
|
||||||
|
self.ap.db.commit()
|
||||||
|
PodcastSettings.objects.create(user=self.user, import_downloads=[{'episode_id': 1, 'size': 1}])
|
||||||
|
out = self.run_import()
|
||||||
|
self.assertEqual(self.offered(), [])
|
||||||
|
self.assertIn('Downloads: 0 on the phone', out)
|
||||||
|
|
||||||
|
|
||||||
|
class ImportDownloadsViewTests(TestCase):
|
||||||
|
"""GET import-downloads/, its clear/, and the count in state/."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(username='alice', password='pw12345678')
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
feed = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/a', title='A')
|
||||||
|
self.ep = PodcastEpisode.objects.create(feed=feed, guid='e', title='E', audio_url='https://example.org/e.mp3',
|
||||||
|
duration_seconds=1800, pub_date=T0)
|
||||||
|
EpisodeProgress.objects.create(user=self.user, episode=self.ep, position_seconds=120)
|
||||||
|
no_audio = PodcastEpisode.objects.create(feed=feed, guid='n', title='N', audio_url='')
|
||||||
|
bob = User.objects.create_user(username='bob', password='pw12345678')
|
||||||
|
bobs_feed = PodcastFeed.objects.create(user=bob, rss_url='https://example.org/a', title='A')
|
||||||
|
bobs = PodcastEpisode.objects.create(feed=bobs_feed, guid='b', title='B', audio_url='https://example.org/b.mp3')
|
||||||
|
gone_feed = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/gone', title='Gone')
|
||||||
|
gone = PodcastEpisode.objects.create(feed=gone_feed, guid='g', title='G', audio_url='https://example.org/g.mp3')
|
||||||
|
gone_feed.delete() # unsubscribed since the import
|
||||||
|
PodcastSettings.objects.create(user=self.user, import_downloads=[
|
||||||
|
{'episode_id': i, 'size': 1000 + n} for n, i in enumerate((bobs.id, self.ep.id, no_audio.id, gone.id))])
|
||||||
|
|
||||||
|
def test_offered_episodes(self):
|
||||||
|
data = self.client.get('/podcasts/import-downloads/').json()
|
||||||
|
self.assertEqual(data['bytes'], 1001)
|
||||||
|
[e] = data['episodes']
|
||||||
|
self.assertEqual((e['id'], e['title'], e['feed__title'], e['duration_seconds'], e['pub_date'],
|
||||||
|
e['position_seconds'], e['in_queue'], e['size']),
|
||||||
|
(self.ep.id, 'E', 'A', 1800, T0.isoformat(), 120, False, 1001))
|
||||||
|
# The same keys as an inbox entry of state/, plus the size.
|
||||||
|
state = self.client.get('/podcasts/state/').json()
|
||||||
|
self.assertEqual(set(e), set(state['inbox'][0]) | {'size'})
|
||||||
|
self.assertEqual(state['import_downloads'], {'count': 1, 'bytes': 1001})
|
||||||
|
|
||||||
|
def test_clear(self):
|
||||||
|
self.assertEqual(self.client.get('/podcasts/import-downloads/clear/').status_code, 405)
|
||||||
|
r = self.client.post('/podcasts/import-downloads/clear/', '{}', content_type='application/json')
|
||||||
|
self.assertEqual(r.json(), {'ok': True})
|
||||||
|
self.assertEqual(PodcastSettings.objects.get(user=self.user).import_downloads, [])
|
||||||
|
self.assertEqual(self.client.get('/podcasts/import-downloads/').json(), {'episodes': [], 'bytes': 0})
|
||||||
|
self.assertEqual(self.client.get('/podcasts/state/').json()['import_downloads'], {'count': 0, 'bytes': 0})
|
||||||
|
|
||||||
|
def test_other_user_and_login(self):
|
||||||
|
self.client.force_login(User.objects.get(username='bob'))
|
||||||
|
self.assertEqual(self.client.get('/podcasts/import-downloads/').json(), {'episodes': [], 'bytes': 0})
|
||||||
|
self.assertEqual(self.client.get('/podcasts/state/').json()['import_downloads'], {'count': 0, 'bytes': 0})
|
||||||
|
self.client.logout()
|
||||||
|
self.assertEqual(self.client.get('/podcasts/import-downloads/').status_code, 401)
|
||||||
|
self.assertEqual(self.client.post('/podcasts/import-downloads/clear/').status_code, 401)
|
||||||
|
self.assertEqual(len(PodcastSettings.objects.get(user=self.user).import_downloads), 4)
|
||||||
|
|
||||||
|
|
||||||
class AfterFirstRefreshTests(ImportTestCase):
|
class AfterFirstRefreshTests(ImportTestCase):
|
||||||
"""The way to move: import, let the new feeds be fetched, import again."""
|
"""The way to move: import, let the new feeds be fetched, import again."""
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -223,6 +223,62 @@ class FirstFetchTests(TestCase):
|
||||||
self.assertEqual([e['title'] for e in inbox], ['e7', 'e6', 'e5', 'e4'])
|
self.assertEqual([e['title'] for e in inbox], ['e7', 'e6', 'e5', 'e4'])
|
||||||
|
|
||||||
|
|
||||||
|
class VideoEnclosureTests(TestCase):
|
||||||
|
"""Video podcasts are taken in (the players play their sound only)."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(username='alice', password='pw12345678')
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
self.feed = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/f', title='Feed')
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def rss(*items):
|
||||||
|
entries = ''.join(
|
||||||
|
f'<item><guid>{guid}</guid><title>{guid}</title><pubDate>{date}</pubDate>{enclosures}</item>'
|
||||||
|
for guid, date, enclosures in items
|
||||||
|
)
|
||||||
|
return f'<?xml version="1.0"?><rss version="2.0"><channel><title>Feed</title>{entries}</channel></rss>'
|
||||||
|
|
||||||
|
def refresh(self, rss, **kw):
|
||||||
|
with patch('podcasts.views._fetch_feed', lambda url, **_: _ok(rss)):
|
||||||
|
return _refresh_feed(self.feed, **kw)
|
||||||
|
|
||||||
|
def test_video_only_and_audio_preferred(self):
|
||||||
|
self.refresh(self.rss(
|
||||||
|
('v', 'Mon, 05 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/v.mp4" type="video/mp4" length="1"/>'),
|
||||||
|
('both', 'Tue, 06 Jan 2026 10:00:00 GMT',
|
||||||
|
'<enclosure url="https://example.org/b.mp4" type="video/mp4" length="1"/>'
|
||||||
|
'<enclosure url="https://example.org/b.m4a" type="audio/mp4" length="1"/>'),
|
||||||
|
('untyped', 'Wed, 07 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/u.m4v?x=1" length="1"/>'),
|
||||||
|
('none', 'Thu, 08 Jan 2026 10:00:00 GMT', '<enclosure url="https://example.org/n.pdf" type="application/pdf"/>'),
|
||||||
|
))
|
||||||
|
urls = dict(PodcastEpisode.objects.filter(feed=self.feed).values_list('guid', 'audio_url'))
|
||||||
|
self.assertEqual(urls, {
|
||||||
|
'v': 'https://example.org/v.mp4',
|
||||||
|
'both': 'https://example.org/b.m4a',
|
||||||
|
'untyped': 'https://example.org/u.m4v?x=1',
|
||||||
|
})
|
||||||
|
|
||||||
|
def test_full_reread_keeps_old_found_episodes_out_of_the_inbox(self):
|
||||||
|
audio = '<enclosure url="https://example.org/a.mp3" type="audio/mpeg" length="1"/>'
|
||||||
|
video = '<enclosure url="https://example.org/{}.mp4" type="video/mp4" length="1"/>'
|
||||||
|
self.refresh(self.rss(('a', 'Mon, 05 Jan 2026 10:00:00 GMT', audio)))
|
||||||
|
self.feed.refresh_from_db()
|
||||||
|
later = (self.feed.last_refreshed_at + timedelta(days=1)).strftime('%a, %d %b %Y %H:%M:%S GMT')
|
||||||
|
self.refresh(self.rss(
|
||||||
|
('a', 'Mon, 05 Jan 2026 10:00:00 GMT', audio),
|
||||||
|
('old-video', 'Tue, 06 Jan 2026 10:00:00 GMT', video.format('o')),
|
||||||
|
('new-video', later, video.format('n')),
|
||||||
|
), full=True)
|
||||||
|
inbox = [e['title'] for e in self.client.get('/podcasts/inbox/').json()['episodes']]
|
||||||
|
self.assertIn('new-video', inbox)
|
||||||
|
self.assertNotIn('old-video', inbox)
|
||||||
|
old = PodcastEpisode.objects.get(feed=self.feed, guid='old-video')
|
||||||
|
progress = EpisodeProgress.objects.get(user=self.user, episode=old)
|
||||||
|
self.assertTrue(progress.dismissed)
|
||||||
|
self.assertFalse(progress.played)
|
||||||
|
|
||||||
|
|
||||||
class RefreshAllTests(TestCase):
|
class RefreshAllTests(TestCase):
|
||||||
"""One request refreshes all feeds on the server; one request returns the state."""
|
"""One request refreshes all feeds on the server; one request returns the state."""
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -28,4 +28,6 @@ urlpatterns = [
|
||||||
path('progress/mark-played/', views.mark_played, name='podcast_mark_played'),
|
path('progress/mark-played/', views.mark_played, name='podcast_mark_played'),
|
||||||
path('progress/dismiss/', views.dismiss_episodes, name='podcast_dismiss_episodes'),
|
path('progress/dismiss/', views.dismiss_episodes, name='podcast_dismiss_episodes'),
|
||||||
path('inbox/', views.inbox, name='podcast_inbox'),
|
path('inbox/', views.inbox, name='podcast_inbox'),
|
||||||
|
path('import-downloads/', views.import_downloads, name='podcast_import_downloads'),
|
||||||
|
path('import-downloads/clear/', views.clear_import_downloads, name='podcast_clear_import_downloads'),
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -408,10 +408,17 @@ def _in_backoff_q(now=None):
|
||||||
return q
|
return q
|
||||||
|
|
||||||
|
|
||||||
def _refresh_feeds(feeds, chunk=32):
|
# File endings of an enclosure without a usable type.
|
||||||
|
AUDIO_EXTENSIONS = ('.mp3', '.m4a', '.m4b', '.ogg', '.oga', '.opus', '.aac', '.flac', '.wav')
|
||||||
|
VIDEO_EXTENSIONS = ('.mp4', '.m4v', '.mov', '.webm')
|
||||||
|
|
||||||
|
|
||||||
|
def _refresh_feeds(feeds, chunk=32, full=False):
|
||||||
"""Refresh many feeds at once: the downloads run in parallel and each
|
"""Refresh many feeds at once: the downloads run in parallel and each
|
||||||
feed URL is fetched once, even when several users subscribed to it; the
|
feed URL is fetched once, even when several users subscribed to it; the
|
||||||
database writes then happen one by one. Returns (refreshed, failed, new)."""
|
database writes then happen one by one. Returns (refreshed, failed, new).
|
||||||
|
`full`: download every feed again (no 304) and take in episodes the
|
||||||
|
reader skipped before (see _apply_feed)."""
|
||||||
feeds = list(feeds)
|
feeds = list(feeds)
|
||||||
refreshed = failed = new = 0
|
refreshed = failed = new = 0
|
||||||
for start in range(0, len(feeds), chunk):
|
for start in range(0, len(feeds), chunk):
|
||||||
|
|
@ -423,7 +430,7 @@ def _refresh_feeds(feeds, chunk=32):
|
||||||
|
|
||||||
def fetch(url):
|
def fetch(url):
|
||||||
try:
|
try:
|
||||||
return url, _fetch_feed(url, **_validators(by_url[url])), None
|
return url, _fetch_feed(url, **({} if full else _validators(by_url[url]))), None
|
||||||
except Exception as e: # network, HTTP or parser trouble: report and go on
|
except Exception as e: # network, HTTP or parser trouble: report and go on
|
||||||
return url, None, e
|
return url, None, e
|
||||||
|
|
||||||
|
|
@ -440,7 +447,7 @@ def _refresh_feeds(feeds, chunk=32):
|
||||||
_record_error(feed, err)
|
_record_error(feed, err)
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
new += _refresh_feed(feed, fetched)
|
new += _refresh_feed(feed, fetched, full=full)
|
||||||
refreshed += 1
|
refreshed += 1
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
failed += 1
|
failed += 1
|
||||||
|
|
@ -448,7 +455,7 @@ def _refresh_feeds(feeds, chunk=32):
|
||||||
return refreshed, failed, new
|
return refreshed, failed, new
|
||||||
|
|
||||||
|
|
||||||
def _refresh_feed(feed_obj, fetched=None):
|
def _refresh_feed(feed_obj, fetched=None, full=False):
|
||||||
"""Fetch rss_url (unless `fetched`, a FeedFetch, is given) and upsert
|
"""Fetch rss_url (unless `fetched`, a FeedFetch, is given) and upsert
|
||||||
episodes. Returns count of new episodes. A failure is stored on the feed
|
episodes. Returns count of new episodes. A failure is stored on the feed
|
||||||
(last_error, error_count) and raised again."""
|
(last_error, error_count) and raised again."""
|
||||||
|
|
@ -459,7 +466,7 @@ def _refresh_feed(feed_obj, fetched=None):
|
||||||
_record_error(feed_obj, e)
|
_record_error(feed_obj, e)
|
||||||
raise
|
raise
|
||||||
try:
|
try:
|
||||||
return _apply_feed(feed_obj, fetched)
|
return _apply_feed(feed_obj, fetched, full=full)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_record_error(feed_obj, e)
|
_record_error(feed_obj, e)
|
||||||
raise
|
raise
|
||||||
|
|
@ -479,8 +486,27 @@ def _move_to_permanent_url(feed_obj, new_url):
|
||||||
feed_obj.rss_url = new_url
|
feed_obj.rss_url = new_url
|
||||||
|
|
||||||
|
|
||||||
def _apply_feed(feed_obj, fetched):
|
def _entry_media_url(entry):
|
||||||
|
"""The episode's file: its audio enclosure, else a video one (the players
|
||||||
|
play only its sound). '' when there is neither."""
|
||||||
|
video = ''
|
||||||
|
for enc in entry.get('enclosures', []):
|
||||||
|
url = enc.get('url', '')
|
||||||
|
mime = (enc.get('type') or '').lower()
|
||||||
|
path = urllib.parse.urlsplit(url).path.lower()
|
||||||
|
if mime.startswith('audio/') or path.endswith(AUDIO_EXTENSIONS):
|
||||||
|
return url
|
||||||
|
if not video and (mime.startswith('video/') or path.endswith(VIDEO_EXTENSIONS)):
|
||||||
|
video = url
|
||||||
|
return video
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_feed(feed_obj, fetched, full=False):
|
||||||
|
"""Store a fetched feed. `full` (a re-read after the reader learned to
|
||||||
|
take more episodes, e.g. video ones): episodes it finds that came out
|
||||||
|
before the last refresh go into the podcast's list, not into "Neu"."""
|
||||||
now = timezone.now()
|
now = timezone.now()
|
||||||
|
previous_refresh = feed_obj.last_refreshed_at
|
||||||
if fetched.status == 304:
|
if fetched.status == 304:
|
||||||
# Unchanged since the last full download: nothing to parse or write.
|
# Unchanged since the last full download: nothing to parse or write.
|
||||||
feed_obj.last_refreshed_at = now
|
feed_obj.last_refreshed_at = now
|
||||||
|
|
@ -524,13 +550,7 @@ def _apply_feed(feed_obj, fetched):
|
||||||
changed_episodes = {}
|
changed_episodes = {}
|
||||||
|
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
# Find audio enclosure
|
audio_url = _entry_media_url(entry)
|
||||||
audio_url = ''
|
|
||||||
for enc in entry.get('enclosures', []):
|
|
||||||
mime = enc.get('type', '')
|
|
||||||
if mime.startswith('audio/') or enc.get('url', '').endswith(('.mp3', '.m4a', '.ogg', '.opus', '.aac')):
|
|
||||||
audio_url = enc.get('url', '')
|
|
||||||
break
|
|
||||||
if not audio_url:
|
if not audio_url:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|
@ -598,6 +618,17 @@ def _apply_feed(feed_obj, fetched):
|
||||||
if changed_episodes:
|
if changed_episodes:
|
||||||
PodcastEpisode.objects.bulk_update(list(changed_episodes.values()), EXTRA_FIELDS, batch_size=100)
|
PodcastEpisode.objects.bulk_update(list(changed_episodes.values()), EXTRA_FIELDS, batch_size=100)
|
||||||
|
|
||||||
|
if full and not first_fetch and new_episodes_list:
|
||||||
|
# Found only now, but out before the last refresh: not news.
|
||||||
|
late = [e for e in new_episodes_list if e.pub_date is None or e.pub_date < previous_refresh]
|
||||||
|
if late:
|
||||||
|
EpisodeProgress.objects.bulk_create(
|
||||||
|
[EpisodeProgress(user=feed_obj.user, episode=e, dismissed=True) for e in late],
|
||||||
|
ignore_conflicts=True,
|
||||||
|
)
|
||||||
|
gone = {e.id for e in late}
|
||||||
|
new_episodes_list = [e for e in new_episodes_list if e.id not in gone]
|
||||||
|
|
||||||
if first_fetch and len(new_episodes_list) > NEW_ON_SUBSCRIBE:
|
if first_fetch and len(new_episodes_list) > NEW_ON_SUBSCRIBE:
|
||||||
# Only the newest episodes count as new; the rest stay in the
|
# Only the newest episodes count as new; the rest stay in the
|
||||||
# podcast's episode list, out of the inbox (dismissed, not played).
|
# podcast's episode list, out of the inbox (dismissed, not played).
|
||||||
|
|
@ -1405,7 +1436,8 @@ def refresh_all_feeds(request):
|
||||||
@csrf_exempt
|
@csrf_exempt
|
||||||
def podcast_state(request):
|
def podcast_state(request):
|
||||||
"""Everything a client shows first, in one response: feeds, queue, the
|
"""Everything a client shows first, in one response: feeds, queue, the
|
||||||
first page of the inbox and the settings."""
|
first page of the inbox, the settings and how many AntennaPod downloads
|
||||||
|
wait to be offered (import-downloads/)."""
|
||||||
if not request.user.is_authenticated:
|
if not request.user.is_authenticated:
|
||||||
return JsonResponse({'error': 'authentication required'}, status=401)
|
return JsonResponse({'error': 'authentication required'}, status=401)
|
||||||
try:
|
try:
|
||||||
|
|
@ -1413,12 +1445,14 @@ def podcast_state(request):
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return JsonResponse({'error': 'inbox_limit must be a number'}, status=400)
|
return JsonResponse({'error': 'inbox_limit must be a number'}, status=400)
|
||||||
s = PodcastSettings.for_user(request.user)
|
s = PodcastSettings.for_user(request.user)
|
||||||
|
downloads = _import_downloads(request.user, s.import_downloads)
|
||||||
return JsonResponse({
|
return JsonResponse({
|
||||||
'feeds': _feeds_data(request.user),
|
'feeds': _feeds_data(request.user),
|
||||||
'queue': _queue_data(request.user),
|
'queue': _queue_data(request.user),
|
||||||
'inbox': _inbox_data(request.user, 0, limit, with_description=False),
|
'inbox': _inbox_data(request.user, 0, limit, with_description=False),
|
||||||
'settings': {'enqueue_position': s.enqueue_position, 'auto_queue_position': s.auto_queue_position,
|
'settings': {'enqueue_position': s.enqueue_position, 'auto_queue_position': s.auto_queue_position,
|
||||||
'keep_sorted': s.keep_sorted, 'auto_download_default': s.auto_download_default},
|
'keep_sorted': s.keep_sorted, 'auto_download_default': s.auto_download_default},
|
||||||
|
'import_downloads': {'count': len(downloads), 'bytes': sum(e['size'] for e in downloads)},
|
||||||
'server_time': timezone.now().isoformat(),
|
'server_time': timezone.now().isoformat(),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -1699,29 +1733,17 @@ def inbox(request):
|
||||||
return JsonResponse({'episodes': _inbox_data(request.user, offset, limit), 'offset': offset, 'limit': limit})
|
return JsonResponse({'episodes': _inbox_data(request.user, offset, limit), 'offset': offset, 'limit': limit})
|
||||||
|
|
||||||
|
|
||||||
def _inbox_data(user, offset, limit, with_description=True):
|
def _inbox_fields(with_description=True):
|
||||||
"""Inbox episodes. Shownotes can be large (up to 50 KB each); the apps'
|
|
||||||
state/ leaves them out and loads one episode's at a time."""
|
|
||||||
fields = ['id', 'title', 'audio_url', 'duration_seconds', 'pub_date', 'artwork_url',
|
fields = ['id', 'title', 'audio_url', 'duration_seconds', 'pub_date', 'artwork_url',
|
||||||
'feed__id', 'feed__title', 'feed__artwork_url']
|
'feed__id', 'feed__title', 'feed__artwork_url']
|
||||||
if with_description:
|
if with_description:
|
||||||
fields.insert(2, 'description')
|
fields.insert(2, 'description')
|
||||||
hidden_ids = set(
|
return fields
|
||||||
EpisodeProgress.objects.filter(
|
|
||||||
user=user,
|
|
||||||
).filter(
|
|
||||||
Q(played=True) | Q(dismissed=True)
|
|
||||||
).values_list('episode_id', flat=True)
|
|
||||||
)
|
|
||||||
|
|
||||||
episodes = list(
|
|
||||||
PodcastEpisode.objects.filter(feed__user=user)
|
|
||||||
.exclude(id__in=hidden_ids)
|
|
||||||
.select_related('feed')
|
|
||||||
.order_by('-pub_date')
|
|
||||||
.values(*fields)[offset:offset + limit]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
def _add_user_state(user, episodes):
|
||||||
|
"""Complete inbox-style episode dicts (_inbox_fields) with the user's
|
||||||
|
position and whether they are queued."""
|
||||||
progress_map = {
|
progress_map = {
|
||||||
ep['episode_id']: ep
|
ep['episode_id']: ep
|
||||||
for ep in EpisodeProgress.objects.filter(
|
for ep in EpisodeProgress.objects.filter(
|
||||||
|
|
@ -1742,3 +1764,68 @@ def _inbox_data(user, offset, limit, with_description=True):
|
||||||
ep['position_seconds'] = prog.get('position_seconds', 0)
|
ep['position_seconds'] = prog.get('position_seconds', 0)
|
||||||
ep['in_queue'] = ep['id'] in queued_ids
|
ep['in_queue'] = ep['id'] in queued_ids
|
||||||
return episodes
|
return episodes
|
||||||
|
|
||||||
|
|
||||||
|
def _inbox_data(user, offset, limit, with_description=True):
|
||||||
|
"""Inbox episodes. Shownotes can be large (up to 50 KB each); the apps'
|
||||||
|
state/ leaves them out and loads one episode's at a time."""
|
||||||
|
hidden_ids = set(
|
||||||
|
EpisodeProgress.objects.filter(
|
||||||
|
user=user,
|
||||||
|
).filter(
|
||||||
|
Q(played=True) | Q(dismissed=True)
|
||||||
|
).values_list('episode_id', flat=True)
|
||||||
|
)
|
||||||
|
|
||||||
|
episodes = list(
|
||||||
|
PodcastEpisode.objects.filter(feed__user=user)
|
||||||
|
.exclude(id__in=hidden_ids)
|
||||||
|
.select_related('feed')
|
||||||
|
.order_by('-pub_date')
|
||||||
|
.values(*_inbox_fields(with_description))[offset:offset + limit]
|
||||||
|
)
|
||||||
|
return _add_user_state(user, episodes)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Episodes downloaded in AntennaPod (antennapod.py): offered once for download
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _import_downloads(user, entries, fields=('id',)):
|
||||||
|
"""The entries of PodcastSettings.import_downloads still worth offering
|
||||||
|
(the episode is still there, the user's, with an audio URL), in their
|
||||||
|
order: dicts of `fields` plus 'size' (bytes, 0 = unknown)."""
|
||||||
|
sizes = {}
|
||||||
|
for e in entries or []:
|
||||||
|
if isinstance(e, dict) and isinstance(e.get('episode_id'), int):
|
||||||
|
size = e.get('size')
|
||||||
|
sizes.setdefault(e['episode_id'], size if isinstance(size, int) and size > 0 else 0)
|
||||||
|
ids = list(sizes)
|
||||||
|
found = {}
|
||||||
|
for i in range(0, len(ids), 500):
|
||||||
|
for ep in (PodcastEpisode.objects.filter(id__in=ids[i:i + 500], feed__user=user)
|
||||||
|
.exclude(audio_url='').values(*fields)):
|
||||||
|
found[ep['id']] = ep
|
||||||
|
return [{**found[ep_id], 'size': sizes[ep_id]} for ep_id in ids if ep_id in found]
|
||||||
|
|
||||||
|
|
||||||
|
@csrf_exempt
|
||||||
|
@require_http_methods(['GET'])
|
||||||
|
def import_downloads(request):
|
||||||
|
"""The episodes that were on the phone in AntennaPod, inbox-style plus
|
||||||
|
their size; the app offers once to download them again."""
|
||||||
|
if not request.user.is_authenticated:
|
||||||
|
return JsonResponse({'error': 'authentication required'}, status=401)
|
||||||
|
entries = PodcastSettings.objects.filter(user=request.user).values_list('import_downloads', flat=True).first()
|
||||||
|
episodes = _add_user_state(request.user, _import_downloads(request.user, entries, _inbox_fields(False)))
|
||||||
|
return JsonResponse({'episodes': episodes, 'bytes': sum(e['size'] for e in episodes)})
|
||||||
|
|
||||||
|
|
||||||
|
@csrf_exempt
|
||||||
|
@require_http_methods(['POST'])
|
||||||
|
def clear_import_downloads(request):
|
||||||
|
"""Offered (downloaded or declined): the list is done with."""
|
||||||
|
if not request.user.is_authenticated:
|
||||||
|
return JsonResponse({'error': 'authentication required'}, status=401)
|
||||||
|
PodcastSettings.objects.filter(user=request.user).update(import_downloads=[])
|
||||||
|
return JsonResponse({'ok': True})
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue