diff --git a/gpodder/views.py b/gpodder/views.py index a16eacb..2e05015 100644 --- a/gpodder/views.py +++ b/gpodder/views.py @@ -8,7 +8,7 @@ from django.http import JsonResponse from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods -from podcasts.models import PodcastFeed, PodcastEpisode, EpisodeProgress, counts_as_played +from podcasts.models import PodcastFeed, PodcastEpisode, EpisodeProgress, PodcastSettings, counts_as_played from podcasts.views import _refresh_feed from .models import GpodderDevice, GpodderSubscriptionChange, GpodderEpisodeAction @@ -151,12 +151,13 @@ def subscriptions_by_device(request, username, deviceid): except (json.JSONDecodeError, ValueError): return JsonResponse({'error': 'invalid JSON'}, status=400) + new_feed = PodcastSettings.new_feed_defaults(request.user) for url in body.get('add', []): if not url: continue PodcastFeed.objects.get_or_create( user=request.user, rss_url=url, - defaults={'title': url} + defaults={'title': url, **new_feed} ) GpodderSubscriptionChange.objects.create(user=request.user, url=url, action='add') diff --git a/podcasts/antennapod.py b/podcasts/antennapod.py new file mode 100644 index 0000000..de2839d --- /dev/null +++ b/podcasts/antennapod.py @@ -0,0 +1,679 @@ +"""Import an AntennaPod database backup into one user's podcasts. + +AntennaPod's "Datenbank exportieren" writes its whole SQLite database. From +it come the subscriptions with their settings, the state of every episode +that has one (played, new, started, queued, favourite) and the queue. See +the import_antennapod command for how to run it. + +The backup is only read. Episodes without any state are not imported: the +feed's current episodes come with its next refresh anyway. Those that diora +already has, though, are taken out of the inbox (AntennaPod: not new). +""" + +import re +import sqlite3 +import time +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import datetime, timezone as dt_timezone +from itertools import groupby +from pathlib import Path +from urllib.parse import urlsplit + +from django.db import transaction +from django.db.models import Q + +from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings +from .views import MAX_DESCRIPTION, _keep_sorted, _leave_inbox, _queue_order, _write_queue_order + +CHUNK = 500 # ids per IN (...) query and rows per bulk write + +# Feeds.feed_volume_adaption -> diora volume factor (0 = off = 1.0) +VOLUME_CODES = {1: 0.5, 2: 0.2, 3: 1.5, 4: 2.0, 5: 2.5} +# Feeds.new_episodes_action; 0 = the global setting (not in the backup) +NEW_EPISODE_ACTIONS = {1: PodcastFeed.NEW_INBOX, 2: PodcastFeed.NEW_IGNORE, 3: PodcastFeed.NEW_QUEUE} +# Feeds.auto_download; 1 = the global setting (not in the backup) +AUTO_DOWNLOAD_OFF, AUTO_DOWNLOAD_GLOBAL, AUTO_DOWNLOAD_ON = 0, 1, 2 +# FeedItems.read +NEW, UNPLAYED, PLAYED = -1, 0, 1 +SUBSCRIBED = 0 # Feeds.state + +SETTING_FIELDS = ('playback_speed', 'skip_intro', 'skip_outro', 'volume', 'include_words', + 'exclude_words', 'min_duration', 'new_episodes', 'auto_download') + +# Columns read from the backup, with the value to assume if an older +# AntennaPod didn't have the column yet. +FEED_COLUMNS = { + 'id': None, 'title': '', 'custom_title': None, 'download_url': '', 'link': '', 'author': '', + 'image_url': '', 'keep_updated': 1, 'include_filter': '', 'exclude_filter': '', + 'minimal_duration_filter': -1, 'new_episodes_action': 0, 'feed_playback_speed': -1, + 'feed_skip_intro': 0, 'feed_skip_ending': 0, 'feed_volume_adaption': 0, 'auto_download': 1, + 'state': SUBSCRIBED, +} + + +class AntennaPodImportError(Exception): + """The file isn't an AntennaPod backup (or the user doesn't exist).""" + + +# --------------------------------------------------------------------------- +# Small helpers +# --------------------------------------------------------------------------- + +def normalize_url(url): + """A URL reduced to what identifies it: no scheme (http = https), host in + lower case without a leading www., no default port, no trailing slash, no + fragment. None for anything but http(s).""" + url = (url or '').strip() + try: + parts = urlsplit(url) + port = parts.port + except ValueError: + return None + if parts.scheme.lower() not in ('http', 'https') or not parts.hostname: + return None + host = parts.hostname.lower() + if host.startswith('www.'): + host = host[4:] + if port and port not in (80, 443): + host = f'{host}:{port}' + path = parts.path.rstrip('/') + return host + path + (f'?{parts.query}' if parts.query else '') + + +def _web_url(url, max_length=1000): + """url if it is a usable http(s) URL for a URLField, else ''.""" + url = (url or '').strip() + if normalize_url(url) is None or len(url) > max_length: + return '' + return url + + +# AntennaPod's filter syntax: words separated by spaces, "quoted phrases". +_TERM = re.compile(r'([^"]\S*|".+?")\s*') + + +def parse_filter_terms(raw): + """AntennaPod filter text -> its terms, e.g. 'a "b c"' -> ['a', 'b c'].""" + terms = [] + for m in _TERM.finditer(raw or ''): + # diora separates its words by commas, so a comma can't be in one. + term = ' '.join(m.group(1).replace('"', '').replace(',', ' ').split()) + if term and term.casefold() not in {t.casefold() for t in terms}: + terms.append(term) + return terms + + +def filter_words(raw, max_length=500): + """AntennaPod filter text -> diora's comma-separated words ('' if none); + as many whole terms as fit into the field.""" + out = '' + for term in parse_filter_terms(raw): + candidate = f'{out}, {term}' if out else term + if len(candidate) > max_length: + break + out = candidate + return out + + +def _ms(value): + """A millisecond timestamp -> aware datetime; None for 0/empty/absurd.""" + try: + value = int(value or 0) + except (TypeError, ValueError): + return None + if value <= 0: + return None + try: + return datetime.fromtimestamp(value / 1000, tz=dt_timezone.utc) + except (OverflowError, OSError, ValueError): + return None + + +def _chunks(seq, size=CHUNK): + seq = list(seq) + for i in range(0, len(seq), size): + yield seq[i:i + size] + + +@contextmanager +def _given_updated_at(): + """Let bulk_create keep the EpisodeProgress.updated_at we set (auto_now + would put now everywhere, and the history and the statistics go by it).""" + f = EpisodeProgress._meta.get_field('updated_at') + f.auto_now = False + try: + yield + finally: + f.auto_now = True + + +# --------------------------------------------------------------------------- +# Feed settings +# --------------------------------------------------------------------------- + +def feed_settings(ap, global_auto_download=False): + """diora settings for one AntennaPod feed row: only those AntennaPod sets + to something other than its default. Returns (settings, not_kept_updated).""" + s = {} + speed = ap['feed_playback_speed'] + if speed is not None and speed > 0: # -1 = the global speed + s['playback_speed'] = min(max(round(float(speed), 2), 0.5), 3.0) + for src, dst in (('feed_skip_intro', 'skip_intro'), ('feed_skip_ending', 'skip_outro')): + seconds = int(ap[src] or 0) + if seconds > 0: + s[dst] = min(seconds, 600) + volume = VOLUME_CODES.get(ap['feed_volume_adaption'] or 0) + if volume: + s['volume'] = volume + for src, dst in (('include_filter', 'include_words'), ('exclude_filter', 'exclude_words')): + words = filter_words(ap[src]) + if words: + s[dst] = words + minutes = int(ap['minimal_duration_filter'] or 0) // 60 # seconds, -1 = none + if minutes > 0: + s['min_duration'] = min(minutes, 600) + action = NEW_EPISODE_ACTIONS.get(ap['new_episodes_action'] or 0) + # "Don't update this podcast": AntennaPod never shows anything new of it. + not_kept_updated = not ap['keep_updated'] and action != PodcastFeed.NEW_QUEUE + if not_kept_updated: + action = PodcastFeed.NEW_IGNORE + if action: + s['new_episodes'] = action + code = ap['auto_download'] + if code == AUTO_DOWNLOAD_ON: + s['auto_download'] = True + elif code == AUTO_DOWNLOAD_GLOBAL: + s['auto_download'] = bool(global_auto_download) + elif code == AUTO_DOWNLOAD_OFF: + s['auto_download'] = False + return s, not_kept_updated + + +# --------------------------------------------------------------------------- +# The import +# --------------------------------------------------------------------------- + +@dataclass +class ItemState: + """What one AntennaPod episode says about its diora episode.""" + played: bool + dismissed: bool + position: int # seconds + favorite: bool + activity: datetime | None # last played or finished, if known + when: datetime # what updated_at becomes: activity, else pub date, else the backup date + + def merge(self, other): + """Two AntennaPod episodes (duplicate feeds) for one diora episode: + the one listened to last, a star from either.""" + a = self if self.when >= other.when else other + return ItemState(a.played, a.dismissed, a.position, self.favorite or other.favorite, a.activity, a.when) + + +@dataclass +class Stats: + backup_feeds: int = 0 + not_subscribed: int = 0 + matched: int = 0 + created: int = 0 + duplicates: int = 0 + created_titles: list = field(default_factory=list) + skipped_feeds: list = field(default_factory=list) # (title, url) + settings: dict = field(default_factory=lambda: {name: 0 for name in SETTING_FIELDS}) + not_kept_updated: int = 0 + auto_download_on: int = 0 + auto_download_explicit: int = 0 + auto_download_global: int = 0 + auto_download_default: bool = False + auto_download_default_changed: bool = False + stateful: int = 0 + by_guid: int = 0 + by_url: int = 0 + episodes_created: int = 0 + no_audio: int = 0 + in_skipped_feeds: int = 0 + stateless_matched: int = 0 + progress_new: int = 0 + progress_updated: int = 0 + progress_unchanged: int = 0 + progress_kept: int = 0 + left_inbox: int = 0 + favorites_backup: int = 0 + favorites_before: int = 0 + favorites_after: int = 0 + queue_backup: int = 0 + queue_missing: int = 0 + queue_before: int = 0 + queue_after: int = 0 + queue_appended: int = 0 + keep_sorted: str = '' + inbox_before: int = 0 + inbox_after: int = 0 + backup_date: datetime | None = None + seconds: float = 0.0 + + +def _inbox_count(user): + hidden = EpisodeProgress.objects.filter(user=user).filter(Q(played=True) | Q(dismissed=True)) + return PodcastEpisode.objects.filter(feed__user=user).exclude(id__in=hidden.values('episode_id')).count() + + +def _open_backup(path): + uri = Path(path).resolve().as_uri() + '?mode=ro' + db = None + try: + db = sqlite3.connect(uri, uri=True) + tables = {r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")} + except sqlite3.DatabaseError as e: + if db is not None: + db.close() + raise AntennaPodImportError(f'{path} is not an SQLite database: {e}') + missing = {'Feeds', 'FeedItems', 'FeedMedia', 'Queue', 'Favorites'} - tables + if missing: + db.close() + raise AntennaPodImportError( + f'{path} is not an AntennaPod backup (missing tables: {", ".join(sorted(missing))})') + return db + + +def _columns(db, table): + return {r[1] for r in db.execute(f'PRAGMA table_info({table})')} + + +class AntennaPodImport: + def __init__(self, user, path, backup_date, global_auto_download=False): + self.user = user + self.path = path + self.backup_date = backup_date + self.global_auto_download = global_auto_download + self.stats = Stats(backup_date=backup_date) + + def run(self, dry_run=False): + started = time.monotonic() + db = _open_backup(self.path) + try: + with transaction.atomic(): + self._run(db) + if dry_run: + transaction.set_rollback(True) + finally: + db.close() + self.stats.seconds = time.monotonic() - started + return self.stats + + def _run(self, db): + st = self.stats + user = self.user + st.queue_before = len(_queue_order(user)) + st.inbox_before = _inbox_count(user) + st.favorites_before = EpisodeProgress.objects.filter(user=user, favorite=True).count() + + # AntennaPod downloads new episodes of every podcast by default: so + # do diora's future subscriptions. + settings = PodcastSettings.for_user(user) + if self.global_auto_download and not settings.auto_download_default: + settings.auto_download_default = True + settings.save(update_fields=['auto_download_default']) + st.auto_download_default_changed = True + st.auto_download_default = settings.auto_download_default + self.new_feed = PodcastSettings.new_feed_defaults(user) + + feed_map = self._import_feeds(db) # AntennaPod feed id -> diora feed id + queue_items = [r[0] for r in db.execute('SELECT feeditem FROM Queue ORDER BY id')] + favorite_items = {r[0] for r in db.execute('SELECT feeditem FROM Favorites')} + st.queue_backup = len(queue_items) + st.favorites_backup = len(favorite_items) + + states, not_new, item_episode = self._import_episodes(db, feed_map, set(queue_items), favorite_items) + self._import_progress(states, not_new) + self._import_queue(queue_items, item_episode) + + st.queue_after = len(_queue_order(user)) + st.inbox_after = _inbox_count(user) + st.favorites_after = EpisodeProgress.objects.filter(user=user, favorite=True).count() + + # -- feeds -------------------------------------------------------------- + + def _import_feeds(self, db): + st = self.stats + have = _columns(db, 'Feeds') + select = ', '.join(c if c in have else f'{default!r} AS {c}' if default is not None else f'NULL AS {c}' + for c, default in FEED_COLUMNS.items()) + db.row_factory = sqlite3.Row + try: + rows = db.execute(f'SELECT {select} FROM Feeds ORDER BY id').fetchall() + finally: + db.row_factory = None + st.backup_feeds = len(rows) + + mine = {} + for f in PodcastFeed.objects.filter(user=self.user).order_by('id'): + mine.setdefault(normalize_url(f.rss_url) or f.rss_url, f) + + feed_map = {} # AntennaPod id -> PodcastFeed (saved or still to create) + to_create = {} # normalised URL -> new PodcastFeed + changed = {} # diora id -> PodcastFeed with changed settings + for ap in rows: + if ap['state'] != SUBSCRIBED: + st.not_subscribed += 1 + continue + url = (ap['download_url'] or '').strip() + title = (ap['custom_title'] or ap['title'] or url).strip() + key = normalize_url(url) + if key is None or len(url) > 1000: + st.skipped_feeds.append((title, url)) + continue + feed = mine.get(key) or to_create.get(key) + if feed is None: + feed = PodcastFeed( + user=self.user, rss_url=url, title=title[:300], author=(ap['author'] or '')[:300], + link=_web_url(ap['link']), artwork_url=_web_url(ap['image_url']), **self.new_feed, + ) + to_create[key] = feed + st.created += 1 + st.created_titles.append(feed.title) + elif any(f is feed for f in feed_map.values()): + st.duplicates += 1 + elif feed.pk: + st.matched += 1 + feed_map[ap['id']] = feed + + settings, not_kept_updated = feed_settings(ap, self.global_auto_download) + st.not_kept_updated += not_kept_updated + st.auto_download_explicit += ap['auto_download'] == AUTO_DOWNLOAD_ON + st.auto_download_global += ap['auto_download'] == AUTO_DOWNLOAD_GLOBAL + for name, value in settings.items(): + if getattr(feed, name) != value: + setattr(feed, name, value) + st.settings[name] += 1 + if feed.pk: + changed[feed.pk] = feed + feed.auto_queue = feed.new_episodes == PodcastFeed.NEW_QUEUE + + if changed: + PodcastFeed.objects.bulk_update(list(changed.values()), [*SETTING_FIELDS, 'auto_queue'], + batch_size=CHUNK) + if to_create: + # last_refreshed_at stays empty: the next refresh fetches them, + # with the first-fetch rule (only the newest episode is new). + PodcastFeed.objects.bulk_create(list(to_create.values()), batch_size=CHUNK) + ids = {} + for urls in _chunks(f.rss_url for f in to_create.values()): + ids.update(PodcastFeed.objects.filter(user=self.user, rss_url__in=urls).values_list('rss_url', 'id')) + for f in to_create.values(): + f.pk = ids[f.rss_url] + st.auto_download_on = sum(1 for f in {id(f): f for f in feed_map.values()}.values() if f.auto_download) + return {ap_id: f.pk for ap_id, f in feed_map.items()} + + # -- episodes ----------------------------------------------------------- + + def _import_episodes(self, db, feed_map, queued, favorites): + """Match AntennaPod's episodes to diora's, create the missing ones + that have a state. Returns ({episode id: ItemState} for those with a + state, {episode ids} of matched ones without (not new), {AntennaPod + item id: episode id}).""" + st = self.stats + backup = self.backup_date + states = {} + not_new = set() + item_episode = {} + to_create = {} # (diora feed id, guid) -> [ItemState, AntennaPod item id, audio url, duration, pub_date] + + cur = db.execute( + 'SELECT i.feed, i.id, i.item_identifier, i.read, i.pubDate, m.download_url, m.duration, ' + 'm.position, m.last_played_time, m.playback_completion_date ' + 'FROM FeedItems i LEFT JOIN FeedMedia m ON m.feeditem = i.id ORDER BY i.feed, i.id' + ) + + def rows(): + while True: + batch = cur.fetchmany(5000) + if not batch: + return + yield from batch + + for ap_feed, items in groupby(rows(), key=lambda r: r[0]): + feed_id = feed_map.get(ap_feed) + by_guid = by_url = None + for (_, item_id, ident, read, pub_ms, audio, duration, position, last_played, completed) in items: + position = max(int(position or 0), 0) // 1000 + activity = max(filter(None, (_ms(last_played), _ms(completed))), default=None) + in_queue = item_id in queued + favorite = item_id in favorites + stateful = read in (NEW, PLAYED) or position > 0 or activity is not None or in_queue or favorite + if feed_id is None: + st.in_skipped_feeds += stateful + continue + if by_guid is None: + by_guid, by_url = {}, {} + for ep_id, guid, url in (PodcastEpisode.objects.filter(feed_id=feed_id) + .order_by('id').values_list('id', 'guid', 'audio_url')): + by_guid.setdefault(guid, ep_id) + by_url.setdefault(normalize_url(url), ep_id) + ident = (ident or '').strip() + audio = (audio or '').strip() + guid = (ident or audio)[:1000] + ep_id = by_guid.get(guid) if guid else None + how = 'guid' + if ep_id is None and audio: + ep_id = by_url.get(normalize_url(audio)) + how = 'url' + if not stateful: + if ep_id is not None: + not_new.add(ep_id) + item_episode[item_id] = ep_id + continue + + st.stateful += 1 + played = read == PLAYED + pub_date = _ms(pub_ms) + state = ItemState( + played=played, + # Only AntennaPod's "new" stays in the inbox; queued is never new. + dismissed=read != NEW or in_queue, + position=0 if played else position, + favorite=favorite, + activity=activity, + when=activity or (min(pub_date, backup) if pub_date else backup), + ) + if ep_id is not None: + st.by_guid += how == 'guid' + st.by_url += how == 'url' + states[ep_id] = state.merge(states[ep_id]) if ep_id in states else state + item_episode[item_id] = ep_id + continue + if normalize_url(audio) is None or len(audio) > 1000: + st.no_audio += 1 + continue + key = (feed_id, guid) + if key in to_create: + pending = to_create[key] + pending[0] = state.merge(pending[0]) + pending[1].append(item_id) + else: + to_create[key] = [state, [item_id], audio, max(int(duration or 0), 0) // 1000, pub_date] + + if to_create: + self._create_episodes(db, to_create, states, item_episode) + not_new -= states.keys() + st.stateless_matched = len(not_new) + return states, not_new, item_episode + + def _create_episodes(self, db, to_create, states, item_episode): + st = self.stats + # A few at a time: shownotes can be large. + for keys in _chunks(to_create, 200): + first_item = {to_create[key][1][0]: key for key in keys} + marks = ','.join('?' * len(first_item)) + details = { + first_item[item_id]: (title, description, image) + for item_id, title, description, image in db.execute( + f'SELECT id, title, description, image_url FROM FeedItems WHERE id IN ({marks})', + list(first_item)) + } + batch = [] + for key in keys: + _, _, audio, duration, pub_date = to_create[key] + title, description, image = details.get(key, ('', '', '')) + batch.append(PodcastEpisode( + feed_id=key[0], guid=key[1], title=(title or 'Untitled')[:500], + description=(description or '')[:MAX_DESCRIPTION], audio_url=audio, + duration_seconds=duration, pub_date=pub_date, artwork_url=_web_url(image), + )) + PodcastEpisode.objects.bulk_create(batch) + st.episodes_created = len(to_create) + + by_feed = {} + for feed_id, guid in to_create: + by_feed.setdefault(feed_id, []).append(guid) + for feed_id, guids in by_feed.items(): + for chunk in _chunks(guids): + created = PodcastEpisode.objects.filter(feed_id=feed_id, guid__in=chunk).values_list('guid', 'id') + for guid, ep_id in created: + state, item_ids = to_create[(feed_id, guid)][:2] + states[ep_id] = state + for item_id in item_ids: + item_episode[item_id] = ep_id + + # -- progress ----------------------------------------------------------- + + def _import_progress(self, states, not_new): + st = self.stats + user = self.user + backup = self.backup_date + existing = {} + for ids in _chunks([*states, *not_new]): + for p in EpisodeProgress.objects.filter(user=user, episode_id__in=ids): + existing[p.episode_id] = p + + def diora_newer(p, activity): + # Changed in diora after the backup was made (and after the + # episode was last played in AntennaPod): diora's state stays. + return p.updated_at > backup and (activity is None or p.updated_at > activity) + + new_rows, changed, favorite_only, dismiss_only = [], [], [], [] + for ep_id, s in states.items(): + p = existing.get(ep_id) + if p is None: + new_rows.append(EpisodeProgress( + user=user, episode_id=ep_id, position_seconds=s.position, played=s.played, + dismissed=s.dismissed, favorite=s.favorite, updated_at=s.when, + )) + st.progress_new += 1 + continue + favorite = p.favorite or s.favorite # stars from both + if diora_newer(p, s.activity): + st.progress_kept += 1 + if favorite != p.favorite: + favorite_only.append(p.pk) + continue + wanted = (s.position, s.played, s.dismissed, favorite, s.when) + if (p.position_seconds, p.played, p.dismissed, p.favorite, p.updated_at) == wanted: + st.progress_unchanged += 1 + continue + p.position_seconds, p.played, p.dismissed, p.favorite, p.updated_at = wanted + changed.append(p) + st.progress_updated += 1 + + # Known to AntennaPod, but not new there: out of the inbox. + for ep_id in not_new: + p = existing.get(ep_id) + if p is None: + new_rows.append(EpisodeProgress(user=user, episode_id=ep_id, dismissed=True, updated_at=backup)) + st.left_inbox += 1 + elif not p.played and not p.dismissed and not diora_newer(p, None): + dismiss_only.append(p.pk) + st.left_inbox += 1 + + with _given_updated_at(): + EpisodeProgress.objects.bulk_create(new_rows, batch_size=CHUNK) + # bulk_update writes updated_at as given (no auto_now there). + EpisodeProgress.objects.bulk_update( + changed, ['position_seconds', 'played', 'dismissed', 'favorite', 'updated_at'], batch_size=CHUNK) + # A star or leaving the inbox isn't listening: updated_at stays. + for ids in _chunks(favorite_only): + EpisodeProgress.objects.filter(pk__in=ids).update(favorite=True) + for ids in _chunks(dismiss_only): + EpisodeProgress.objects.filter(pk__in=ids).update(dismissed=True) + + # -- queue -------------------------------------------------------------- + + def _import_queue(self, queue_items, item_episode): + """AntennaPod's queue first, then what else was in diora's.""" + st = self.stats + user = self.user + before = _queue_order(user) + front = [] + for item_id in queue_items: + ep_id = item_episode.get(item_id) + if ep_id is None: + st.queue_missing += 1 + elif ep_id not in front: + front.append(ep_id) + if not front: + return + in_front = set(front) + rest = [ep_id for ep_id in before if ep_id not in in_front] + st.queue_appended = len(rest) + present = set(before) + PodcastQueue.objects.bulk_create( + [PodcastQueue(user=user, episode_id=ep_id, position=len(before) + i) + for i, ep_id in enumerate(e for e in front if e not in present)], + batch_size=CHUNK, + ) + _write_queue_order(user, front + rest) + _leave_inbox(user, front) # queued = not new + if _keep_sorted(user): + st.keep_sorted = PodcastSettings.objects.get(user=user).keep_sorted + + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- + +SETTING_LABELS = { + 'playback_speed': 'speed', 'skip_intro': 'skip intro', 'skip_outro': 'skip outro', 'volume': 'volume', + 'include_words': 'include words', 'exclude_words': 'exclude words', 'min_duration': 'min duration', + 'new_episodes': 'new episodes', 'auto_download': 'auto-download', +} + + +def _on(flag): + return 'on' if flag else 'off' + + +def summary_lines(st, username, path, dry_run, global_auto_download): + when = st.backup_date.strftime('%Y-%m-%d %H:%M UTC') if st.backup_date else '?' + subscribed = st.backup_feeds - st.not_subscribed + settings = ', '.join(f'{SETTING_LABELS[k]} {v}' for k, v in st.settings.items()) + lines = [ + f"AntennaPod import for user '{username}' from {Path(path).name} (backup date {when})" + + (' -- DRY RUN, nothing was written' if dry_run else ''), + f'Feeds: {st.backup_feeds} in backup, {subscribed} subscribed ({st.not_subscribed} not subscribed ' + f'ignored); {st.matched} matched, {st.created} created, {len(st.skipped_feeds)} skipped (not http/https), ' + f'{st.duplicates} duplicates', + f'Settings: changed: {settings}', + f' {st.not_kept_updated} feeds not kept updated in AntennaPod -> new episodes "ignore"', + f'Auto-download: on for {st.auto_download_on} feeds ({st.auto_download_explicit} on in AntennaPod, ' + f"{st.auto_download_global} use AntennaPod's global setting -> {_on(global_auto_download)} " + f'by --global-auto-download); default for new subscriptions: {_on(st.auto_download_default)}' + + (' (set by this import)' if st.auto_download_default_changed else ''), + f'Episodes: {st.stateful} with a state: {st.by_guid} matched by guid, {st.by_url} by URL, ' + f'{st.episodes_created} created, {st.no_audio} skipped (no audio URL), {st.in_skipped_feeds} in skipped feeds', + f' {st.stateless_matched} without a state matched to diora episodes (not new in AntennaPod)', + f'Progress: {st.progress_new} new, {st.progress_updated} updated, ' + f'{st.progress_unchanged} unchanged, {st.progress_kept} kept (diora newer); ' + f'{st.left_inbox} taken out of the inbox (not new in AntennaPod)', + f'Favourites: {st.favorites_backup} in backup; {st.favorites_before} before, {st.favorites_after} after', + f'Queue: {st.queue_backup} in backup ({st.queue_missing} not found); {st.queue_before} before, ' + f'{st.queue_after} after ({st.queue_appended} diora entries after AntennaPod\'s)' + + (f'; kept sorted by "{st.keep_sorted}"' if st.keep_sorted else ''), + f'Inbox: {st.inbox_before} before, {st.inbox_after} after', + f'Created feeds ({len(st.created_titles)}):', + *(f' - {t}' for t in st.created_titles), + f'Skipped feeds ({len(st.skipped_feeds)}):', + *(f' - {t} ({u})' for t, u in st.skipped_feeds), + f'Done in {st.seconds:.1f} s.', + ] + return lines diff --git a/podcasts/management/commands/import_antennapod.py b/podcasts/management/commands/import_antennapod.py new file mode 100644 index 0000000..1c14216 --- /dev/null +++ b/podcasts/management/commands/import_antennapod.py @@ -0,0 +1,56 @@ +import os +from datetime import datetime, timezone as dt_timezone + +from django.contrib.auth.models import User +from django.core.management.base import BaseCommand, CommandError +from django.utils.dateparse import parse_datetime + +from podcasts.antennapod import AntennaPodImport, AntennaPodImportError, summary_lines + + +class Command(BaseCommand): + help = ( + "Import an AntennaPod database backup (Settings > Database > Export) into one user's podcasts: " + 'subscriptions with their settings, played/new/started episodes, favourites and the queue. ' + 'Feeds it creates are fetched by the next refresh; running the import once more after that ' + 'also takes their older episodes that were not new in AntennaPod out of the inbox. ' + 'Running it twice changes nothing more.' + ) + + def add_arguments(self, parser): + parser.add_argument('path', help='AntennaPod backup (.db)') + parser.add_argument('--user', required=True, help='diora username') + parser.add_argument('--dry-run', action='store_true', + help='Do everything inside a transaction that is rolled back, print the summary') + parser.add_argument('--global-auto-download', choices=('on', 'off'), default='off', + help="Auto-download for feeds that use AntennaPod's global setting (default off)") + parser.add_argument('--backup-date', + help='When the backup was made (ISO date/time, UTC if no zone); default: the file date. ' + 'diora changes after it win over the backup.') + + def handle(self, *args, **options): + path = options['path'] + if not os.path.isfile(path): + raise CommandError(f'No such file: {path}') + try: + user = User.objects.get(username=options['user']) + except User.DoesNotExist: + raise CommandError(f"No such user: {options['user']}") + + if options['backup_date']: + raw = options['backup_date'] + backup_date = parse_datetime(raw) or parse_datetime(f'{raw}T00:00') + if backup_date is None: + raise CommandError(f'Invalid --backup-date: {raw}') + if backup_date.tzinfo is None: + backup_date = backup_date.replace(tzinfo=dt_timezone.utc) + else: + backup_date = datetime.fromtimestamp(os.path.getmtime(path), tz=dt_timezone.utc) + + global_auto_download = options['global_auto_download'] == 'on' + try: + stats = AntennaPodImport(user, path, backup_date, global_auto_download).run(dry_run=options['dry_run']) + except AntennaPodImportError as e: + raise CommandError(str(e)) + for line in summary_lines(stats, user.username, path, options['dry_run'], global_auto_download): + self.stdout.write(line) diff --git a/podcasts/migrations/0010_feed_auto_download.py b/podcasts/migrations/0010_feed_auto_download.py new file mode 100644 index 0000000..f71d566 --- /dev/null +++ b/podcasts/migrations/0010_feed_auto_download.py @@ -0,0 +1,21 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('podcasts', '0009_episode_transcript_cache'), + ] + + operations = [ + migrations.AddField( + model_name='podcastfeed', + name='auto_download', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='podcastsettings', + name='auto_download_default', + field=models.BooleanField(default=False), + ), + ] diff --git a/podcasts/models.py b/podcasts/models.py index 90eb0c5..b248e20 100644 --- a/podcasts/models.py +++ b/podcasts/models.py @@ -48,6 +48,9 @@ class PodcastFeed(models.Model): include_words = models.CharField(max_length=500, blank=True) exclude_words = models.CharField(max_length=500, blank=True) min_duration = models.PositiveIntegerField(default=0) # minutes, 0 = any + # Devices download this feed's new episodes (those in the inbox) on + # their own, for listening offline. + auto_download = models.BooleanField(default=False) class Meta: unique_together = ('user', 'rss_url') @@ -231,11 +234,19 @@ class PodcastSettings(models.Model): # front = how auto-queue always worked before this setting existed auto_queue_position = models.CharField(max_length=20, choices=POSITIONS, default=FRONT) keep_sorted = models.CharField(max_length=10, choices=SORT_ORDERS, default='', blank=True) + # What PodcastFeed.auto_download starts as for a new subscription. + auto_download_default = models.BooleanField(default=False) @classmethod def for_user(cls, user): obj, _ = cls.objects.get_or_create(user=user) return obj + @classmethod + def new_feed_defaults(cls, user): + """Field values a new PodcastFeed of the user starts with.""" + auto_download = cls.objects.filter(user=user).values_list('auto_download_default', flat=True).first() + return {'auto_download': bool(auto_download)} + def __str__(self): return f'Podcast settings of {self.user}' diff --git a/podcasts/test_antennapod.py b/podcasts/test_antennapod.py new file mode 100644 index 0000000..953644a --- /dev/null +++ b/podcasts/test_antennapod.py @@ -0,0 +1,501 @@ +import os +import sqlite3 +import tempfile +from datetime import datetime, timedelta, timezone +from io import StringIO +from unittest.mock import patch + +from django.contrib.auth.models import User +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import TestCase + +from .antennapod import filter_words, normalize_url, parse_filter_terms +from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings + +BACKUP_DATE = datetime(2026, 9, 1, 12, 0, tzinfo=timezone.utc) +T0 = datetime(2026, 8, 1, tzinfo=timezone.utc) + + +def ms(dt): + return int(dt.timestamp() * 1000) + + +class Backup: + """A small AntennaPod database: the tables and columns the import reads.""" + + SCHEMA = """ + CREATE TABLE Feeds (id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, custom_title TEXT, + download_url TEXT, link TEXT, author TEXT, image_url TEXT, keep_updated INTEGER DEFAULT 1, + include_filter TEXT DEFAULT '', exclude_filter TEXT DEFAULT '', + minimal_duration_filter INTEGER DEFAULT -1, new_episodes_action INTEGER DEFAULT 0, + feed_playback_speed REAL DEFAULT -1, feed_skip_intro INTEGER DEFAULT 0, + feed_skip_ending INTEGER DEFAULT 0, feed_volume_adaption INTEGER DEFAULT 0, + auto_download INTEGER DEFAULT 0, state INTEGER DEFAULT 0, tags TEXT); + CREATE TABLE FeedItems (id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, pubDate INTEGER, + read INTEGER, link TEXT, description TEXT, media INTEGER, feed INTEGER, + item_identifier TEXT, image_url TEXT); + CREATE TABLE FeedMedia (id INTEGER PRIMARY KEY AUTOINCREMENT, duration INTEGER, download_url TEXT, + position INTEGER, playback_completion_date INTEGER, feeditem INTEGER, played_duration INTEGER, + last_played_time INTEGER); + CREATE TABLE Queue (id INTEGER PRIMARY KEY AUTOINCREMENT, feeditem INTEGER, feed INTEGER); + CREATE TABLE Favorites (id INTEGER PRIMARY KEY AUTOINCREMENT, feeditem INTEGER, feed INTEGER); + """ + + def __init__(self, path): + self.path = path + self.db = sqlite3.connect(path) + self.db.executescript(self.SCHEMA) + + def feed(self, url, title='Feed', **cols): + cols = {'download_url': url, 'title': title, **cols} + cur = self.db.execute(f'INSERT INTO Feeds ({",".join(cols)}) VALUES ({",".join("?" * len(cols))})', + list(cols.values())) + self.db.commit() + return cur.lastrowid + + def item(self, feed, guid, read=0, audio=None, pub=T0, duration=1800, position=0, completed=None, + last_played=None, title=None, description='', image_url=None, media=True): + cur = self.db.execute( + 'INSERT INTO FeedItems (title, pubDate, read, description, feed, item_identifier, image_url) ' + 'VALUES (?,?,?,?,?,?,?)', + (title or f'Episode {guid}', ms(pub) if pub else 0, read, description, feed, guid, image_url)) + item_id = cur.lastrowid + if media: + self.db.execute( + 'INSERT INTO FeedMedia (duration, download_url, position, playback_completion_date, feeditem, ' + 'played_duration, last_played_time) VALUES (?,?,?,?,?,?,?)', + (duration * 1000, audio or f'https://cdn.example.org/{guid}.mp3', position * 1000, + ms(completed) if completed else 0, item_id, 0, ms(last_played) if last_played else 0)) + self.db.commit() + return item_id + + def queue(self, *items): + for i in items: + self.db.execute('INSERT INTO Queue (feeditem, feed) SELECT id, feed FROM FeedItems WHERE id=?', (i,)) + self.db.commit() + + def favorite(self, *items): + for i in items: + self.db.execute('INSERT INTO Favorites (feeditem, feed) SELECT id, feed FROM FeedItems WHERE id=?', (i,)) + self.db.commit() + + def close(self): + self.db.close() + + +class ImportTestCase(TestCase): + def setUp(self): + self.user = User.objects.create_user(username='alice', password='pw12345678') + self.tmp = tempfile.TemporaryDirectory() + self.ap = Backup(os.path.join(self.tmp.name, 'AntennaPodBackup.db')) + + def tearDown(self): + self.ap.close() + self.tmp.cleanup() + + def run_import(self, *args, user='alice'): + out = StringIO() + call_command('import_antennapod', self.ap.path, '--user', user, + '--backup-date', BACKUP_DATE.isoformat(), *args, stdout=out) + return out.getvalue() + + def diora_feed(self, url='https://example.org/feed', **kw): + return PodcastFeed.objects.create(user=self.user, rss_url=url, title=kw.pop('title', 'Diora'), **kw) + + def episode(self, feed, guid, audio=None, **kw): + return PodcastEpisode.objects.create(feed=feed, guid=guid, title=guid, + audio_url=audio or f'https://cdn.example.org/{guid}.mp3', **kw) + + def progress(self, episode): + return EpisodeProgress.objects.filter(user=self.user, episode=episode).first() + + def snapshot(self): + return ( + list(PodcastFeed.objects.order_by('id').values()), + list(PodcastEpisode.objects.order_by('id').values()), + list(EpisodeProgress.objects.order_by('id').values()), + list(PodcastQueue.objects.order_by('id').values('id', 'user_id', 'episode_id', 'position')), + ) + + def queue(self): + return list(PodcastQueue.objects.filter(user=self.user).order_by('position') + .values_list('episode_id', flat=True)) + + +class HelperTests(TestCase): + def test_normalize_url(self): + same = ['https://www.Example.org/feed/', 'http://example.org/feed', 'HTTPS://EXAMPLE.ORG/feed', + 'https://example.org:443/feed#x', 'http://example.org:80/feed//'] + self.assertEqual({normalize_url(u) for u in same}, {'example.org/feed'}) + self.assertNotEqual(normalize_url('https://example.org/Feed'), normalize_url('https://example.org/feed')) + self.assertEqual(normalize_url('https://example.org:8080/f?a=1'), 'example.org:8080/f?a=1') + for bad in ('antennapod_local:content://x', '', None, 'ftp://example.org/f', 'https://'): + self.assertIsNone(normalize_url(bad)) + + def test_filter_terms(self): + self.assertEqual(parse_filter_terms('news "long talk" Teaser'), ['news', 'long talk', 'Teaser']) + self.assertEqual(parse_filter_terms('"news headlines" '), ['news headlines']) + self.assertEqual(parse_filter_terms(' a b '), ['a', 'b']) + self.assertEqual(parse_filter_terms('"open quote'), ['open', 'quote']) + self.assertEqual(parse_filter_terms('a,b A'), ['a b', 'A']) + self.assertEqual(parse_filter_terms('x X'), ['x']) + self.assertEqual(parse_filter_terms(''), []) + self.assertEqual(filter_words('news "long talk"'), 'news, long talk') + self.assertEqual(len(filter_words(' '.join(['word%03d' % i for i in range(100)]))), 493) + + +class FeedImportTests(ImportTestCase): + def test_feeds_match_by_normalised_url_and_missing_ones_are_created(self): + mine = self.diora_feed('https://www.Example.org/feed/') + only_diora = self.diora_feed('https://example.org/only-in-diora') + self.ap.feed('http://example.org/feed', title='Same') + self.ap.feed('https://new.example.org/rss', title='Original', custom_title='Renamed', author='Ann', + link='https://new.example.org/', image_url='https://new.example.org/art.jpg') + self.ap.feed('antennapod_local:content://folder', title='Local') + self.ap.feed('https://gone.example.org/rss', title='Not subscribed', state=1) + + out = self.run_import() + + self.assertEqual(PodcastFeed.objects.filter(user=self.user).count(), 3) + mine.refresh_from_db() + self.assertEqual(mine.title, 'Diora') # matched, not renamed + self.assertTrue(PodcastFeed.objects.filter(pk=only_diora.pk).exists()) + new = PodcastFeed.objects.get(user=self.user, rss_url='https://new.example.org/rss') + self.assertEqual((new.title, new.author, new.link, new.artwork_url, new.last_refreshed_at), + ('Renamed', 'Ann', 'https://new.example.org/', 'https://new.example.org/art.jpg', None)) + self.assertIn('Feeds: 4 in backup, 3 subscribed (1 not subscribed ignored); 1 matched, 1 created, ' + '1 skipped (not http/https), 0 duplicates', out) + self.assertIn('Created feeds (1):\n - Renamed\n', out) + self.assertIn('Skipped feeds (1):\n - Local (antennapod_local:content://folder)\n', out) + + def test_two_backup_feeds_for_one_url(self): + self.ap.feed('https://example.org/feed') + self.ap.feed('http://www.example.org/feed/') + out = self.run_import() + self.assertEqual(PodcastFeed.objects.filter(user=self.user).count(), 1) + self.assertIn('0 matched, 1 created, 0 skipped (not http/https), 1 duplicates', out) + + def test_settings(self): + keep = self.diora_feed('https://example.org/keep', playback_speed=1.5, skip_intro=10, volume=2.0, + new_episodes='queue', include_words='x', min_duration=5, auto_download=True) + changed = self.diora_feed('https://example.org/changed', playback_speed=1.5) + self.ap.feed('https://example.org/keep', auto_download=1) # all AntennaPod defaults + self.ap.feed('https://example.org/changed', feed_playback_speed=1.2000000476837158, feed_skip_intro=30, + feed_skip_ending=700, feed_volume_adaption=2, include_filter='news "long talk" ', + exclude_filter='"Trailer"', minimal_duration_filter=1500, new_episodes_action=2, + auto_download=2) + self.ap.feed('https://example.org/fast', feed_playback_speed=3.5, feed_volume_adaption=5, + minimal_duration_filter=59, new_episodes_action=3) + self.ap.feed('https://example.org/slow', feed_playback_speed=0.25, feed_volume_adaption=3, + new_episodes_action=1) + self.ap.feed('https://example.org/loud', feed_volume_adaption=4) + self.ap.feed('https://example.org/quiet', feed_volume_adaption=1) + + out = self.run_import() + + keep.refresh_from_db() + self.assertEqual( + (keep.playback_speed, keep.skip_intro, keep.volume, keep.new_episodes, keep.auto_queue, + keep.include_words, keep.min_duration, keep.auto_download), + (1.5, 10, 2.0, 'queue', True, 'x', 5, False), # auto_download "global", --global-auto-download off + ) + changed.refresh_from_db() + self.assertEqual( + (changed.playback_speed, changed.skip_intro, changed.skip_outro, changed.volume, changed.include_words, + changed.exclude_words, changed.min_duration, changed.new_episodes, changed.auto_download), + (1.2, 30, 600, 0.2, 'news, long talk', 'Trailer', 25, 'ignore', True), + ) + feeds = {f.rss_url.rsplit('/', 1)[1]: f for f in PodcastFeed.objects.filter(user=self.user)} + fast, slow = feeds['fast'], feeds['slow'] + self.assertEqual((fast.playback_speed, fast.volume, fast.min_duration, fast.new_episodes, fast.auto_queue), + (3.0, 2.5, 0, 'queue', True)) + self.assertEqual((slow.playback_speed, slow.volume, slow.new_episodes, slow.auto_queue), + (0.5, 1.5, 'inbox', False)) + self.assertEqual((feeds['loud'].volume, feeds['quiet'].volume), (2.0, 0.5)) + self.assertIn('Settings: changed: speed 3, skip intro 1, skip outro 1, volume 5, include words 1, ' + 'exclude words 1, min duration 1, new episodes 2, auto-download 2', out) + + def test_not_kept_updated_feeds_bring_nothing_new(self): + self.ap.feed('https://example.org/a', keep_updated=0) + self.ap.feed('https://example.org/b', keep_updated=0, new_episodes_action=1) + self.ap.feed('https://example.org/c', keep_updated=0, new_episodes_action=3) + self.ap.feed('https://example.org/d', keep_updated=1) + out = self.run_import() + new = dict(PodcastFeed.objects.filter(user=self.user).values_list('rss_url', 'new_episodes')) + self.assertEqual(new, {'https://example.org/a': 'ignore', 'https://example.org/b': 'ignore', + 'https://example.org/c': 'queue', 'https://example.org/d': 'inbox'}) + self.assertIn('2 feeds not kept updated in AntennaPod -> new episodes "ignore"', out) + + def test_auto_download(self): + self.diora_feed('https://example.org/off', auto_download=True) + for name, code in (('off', 0), ('global', 1), ('on', 2), ('global2', 1)): + self.ap.feed(f'https://example.org/{name}', auto_download=code) + + def values(): + return dict(PodcastFeed.objects.filter(user=self.user).values_list('rss_url', 'auto_download')) + + out = self.run_import() + self.assertEqual(values(), {'https://example.org/off': False, 'https://example.org/global': False, + 'https://example.org/on': True, 'https://example.org/global2': False}) + self.assertIn("Auto-download: on for 1 feeds (1 on in AntennaPod, 2 use AntennaPod's global setting -> off " + 'by --global-auto-download); default for new subscriptions: off\n', out) + self.assertFalse(PodcastSettings.for_user(self.user).auto_download_default) + + out = self.run_import('--global-auto-download', 'on') + self.assertEqual(values(), {'https://example.org/off': False, 'https://example.org/global': True, + 'https://example.org/on': True, 'https://example.org/global2': True}) + self.assertIn("Auto-download: on for 3 feeds (1 on in AntennaPod, 2 use AntennaPod's global setting -> on " + 'by --global-auto-download); default for new subscriptions: on (set by this import)\n', out) + self.assertTrue(PodcastSettings.for_user(self.user).auto_download_default) + # Once more: nothing to set any more; "off" leaves the default alone. + self.assertIn('default for new subscriptions: on\n', self.run_import('--global-auto-download', 'on')) + self.assertIn('default for new subscriptions: on\n', self.run_import()) + + def test_created_feeds_start_with_the_users_default(self): + PodcastSettings.objects.create(user=self.user, auto_download_default=True) + self.ap.feed('https://example.org/explicit-off', auto_download=0) + self.ap.feed('https://example.org/on', auto_download=2) + self.run_import() + values = dict(PodcastFeed.objects.filter(user=self.user).values_list('rss_url', 'auto_download')) + self.assertEqual(values, {'https://example.org/explicit-off': False, 'https://example.org/on': True}) + + +class EpisodeImportTests(ImportTestCase): + def setUp(self): + super().setUp() + self.feed = self.diora_feed() + self.ap_feed = self.ap.feed('https://example.org/feed') + + def test_matching_and_creating_episodes(self): + by_guid = self.episode(self.feed, 'g1') + by_url = self.episode(self.feed, 'other-guid', audio='http://www.cdn.example.org/g2.mp3') + stateless = self.episode(self.feed, 'g4') + untouched = self.episode(self.feed, 'g6') + self.ap.item(self.ap_feed, 'g1', read=1) + self.ap.item(self.ap_feed, 'g2', read=1, audio='https://cdn.example.org/g2.mp3') + self.ap.item(self.ap_feed, 'g3', read=1, title='Created', description='
Notes
', duration=3600, + pub=T0 + timedelta(days=3), image_url='https://example.org/g3.jpg') + self.ap.item(self.ap_feed, 'g4', read=0) + self.ap.item(self.ap_feed, 'g5', read=0) # no state, not in diora: not imported + self.ap.item(self.ap_feed, None, read=-1, audio='https://cdn.example.org/noguid.mp3') + self.ap.item(self.ap_feed, 'g7', read=1, media=False) + + out = self.run_import() + + self.assertTrue(self.progress(by_guid).played) + self.assertTrue(self.progress(by_url).played) + created = PodcastEpisode.objects.get(feed=self.feed, guid='g3') + self.assertEqual( + (created.title, created.description, created.audio_url, created.duration_seconds, created.pub_date, + created.artwork_url), + ('Created', 'Notes
', 'https://cdn.example.org/g3.mp3', 3600, T0 + timedelta(days=3), + 'https://example.org/g3.jpg'), + ) + self.assertTrue(self.progress(created).played) + # Without a guid, the audio URL is the guid (as when diora reads the feed). + noguid = PodcastEpisode.objects.get(feed=self.feed, guid='https://cdn.example.org/noguid.mp3') + self.assertFalse(self.progress(noguid).dismissed) + self.assertFalse(PodcastEpisode.objects.filter(guid__in=['g5', 'g7']).exists()) + # Known to AntennaPod and not new there: out of the inbox, not played. + p = self.progress(stateless) + self.assertEqual((p.dismissed, p.played), (True, False)) + self.assertIsNone(self.progress(untouched)) + self.assertIn('Episodes: 5 with a state: 1 matched by guid, 1 by URL, 2 created, ' + '1 skipped (no audio URL), 0 in skipped feeds', out) + self.assertIn('1 without a state matched to diora episodes', out) + self.assertIn('Inbox: 4 before, 2 after', out) # g6 and noguid + + def test_states(self): + played_at = T0 + timedelta(days=10) + self.ap.item(self.ap_feed, 'played', read=1, position=0, completed=played_at, last_played=played_at) + self.ap.item(self.ap_feed, 'marked', read=1, pub=T0 + timedelta(days=2)) + self.ap.item(self.ap_feed, 'started', read=0, position=600, last_played=played_at - timedelta(hours=1)) + self.ap.item(self.ap_feed, 'new', read=-1, pub=T0 + timedelta(days=5)) + self.ap.item(self.ap_feed, 'nodate', read=1, pub=None) + self.ap.item(self.ap_feed, 'future', read=1, pub=BACKUP_DATE + timedelta(days=30)) + self.run_import() + + def state(guid): + p = self.progress(PodcastEpisode.objects.get(guid=guid)) + return p.played, p.dismissed, p.position_seconds, p.updated_at + + self.assertEqual(state('played'), (True, True, 0, played_at)) + self.assertEqual(state('marked'), (True, True, 0, T0 + timedelta(days=2))) # no activity: pub date + self.assertEqual(state('started'), (False, True, 600, played_at - timedelta(hours=1))) + self.assertEqual(state('new'), (False, False, 0, T0 + timedelta(days=5))) + self.assertEqual(state('nodate'), (True, True, 0, BACKUP_DATE)) + self.assertEqual(state('future'), (True, True, 0, BACKUP_DATE)) + + def test_favourites_are_a_union(self): + diora_star = self.episode(self.feed, 'a') + p = EpisodeProgress.objects.create(user=self.user, episode=diora_star, favorite=True) + EpisodeProgress.objects.filter(pk=p.pk).update(updated_at=BACKUP_DATE - timedelta(days=5)) + both = self.episode(self.feed, 'b') + self.ap.item(self.ap_feed, 'a', read=1) # AntennaPod wins, but diora's star stays + b = self.ap.item(self.ap_feed, 'b', read=0) + c = self.ap.item(self.ap_feed, 'c', read=0) + self.ap.favorite(b, c) + out = self.run_import() + p = self.progress(diora_star) + self.assertEqual((p.favorite, p.played), (True, True)) + self.assertTrue(self.progress(both).favorite) + self.assertTrue(self.progress(PodcastEpisode.objects.get(guid='c')).favorite) + self.assertIn('Favourites: 2 in backup; 1 before, 3 after', out) + + def test_merge_rule(self): + newer = self.episode(self.feed, 'newer') + older = self.episode(self.feed, 'older') + dismissed = self.episode(self.feed, 'dismissed') + after_backup = BACKUP_DATE + timedelta(days=1) + for ep, values in ((newer, dict(position_seconds=100)), (older, dict(position_seconds=100)), + (dismissed, dict(dismissed=True))): + p = EpisodeProgress.objects.create(user=self.user, episode=ep, **values) + EpisodeProgress.objects.filter(pk=p.pk).update( + updated_at=after_backup if ep is not older else BACKUP_DATE - timedelta(days=1)) + self.ap.item(self.ap_feed, 'newer', read=1, completed=BACKUP_DATE - timedelta(days=2)) + self.ap.item(self.ap_feed, 'older', read=1, completed=BACKUP_DATE - timedelta(days=2)) + n = self.ap.item(self.ap_feed, 'dismissed', read=-1) + self.ap.favorite(n) + out = self.run_import() + p = self.progress(newer) + self.assertEqual((p.played, p.position_seconds, p.updated_at), (False, 100, after_backup)) + p = self.progress(older) + self.assertEqual((p.played, p.position_seconds, p.updated_at), (True, 0, BACKUP_DATE - timedelta(days=2))) + # diora's newer "not new" stays; the star comes over anyway, without + # touching updated_at. + p = self.progress(dismissed) + self.assertEqual((p.dismissed, p.favorite, p.updated_at), (True, True, after_backup)) + self.assertIn('Progress: 0 new, 1 updated, 0 unchanged, 2 kept (diora newer)', out) + + def test_activity_after_diora_change_wins(self): + ep = self.episode(self.feed, 'e') + p = EpisodeProgress.objects.create(user=self.user, episode=ep, position_seconds=50) + EpisodeProgress.objects.filter(pk=p.pk).update(updated_at=BACKUP_DATE + timedelta(hours=1)) + self.ap.item(self.ap_feed, 'e', read=0, position=300, last_played=BACKUP_DATE + timedelta(hours=2)) + self.run_import() + self.assertEqual(self.progress(ep).position_seconds, 300) + + def test_queue(self): + x = self.episode(self.feed, 'x') + y = self.episode(self.feed, 'y') + PodcastQueue.objects.create(user=self.user, episode=x, position=0) + PodcastQueue.objects.create(user=self.user, episode=y, position=1) + a = self.ap.item(self.ap_feed, 'a', read=-1) + b = self.ap.item(self.ap_feed, 'b', read=0) + y_ap = self.ap.item(self.ap_feed, 'y', read=0) + local = self.ap.feed('antennapod_local:x') + lost = self.ap.item(local, 'l', read=0) + self.ap.queue(b, y_ap, lost, a) + out = self.run_import() + ids = {e.guid: e.id for e in PodcastEpisode.objects.filter(feed=self.feed)} + self.assertEqual(self.queue(), [ids['b'], y.id, ids['a'], x.id]) + self.assertEqual(list(PodcastQueue.objects.filter(user=self.user).order_by('position') + .values_list('position', flat=True)), [0, 1, 2, 3]) + # Queued is not new, even if AntennaPod still had it as new. + self.assertTrue(self.progress(PodcastEpisode.objects.get(guid='a')).dismissed) + self.assertIn('Queue: 4 in backup (1 not found); 2 before, 4 after (1 diora entries after', out) + + def test_queue_kept_sorted(self): + PodcastSettings.objects.create(user=self.user, keep_sorted='old') + a = self.ap.item(self.ap_feed, 'a', read=0, pub=T0 + timedelta(days=2)) + b = self.ap.item(self.ap_feed, 'b', read=0, pub=T0 + timedelta(days=1)) + self.ap.queue(a, b) + out = self.run_import() + self.assertEqual([PodcastEpisode.objects.get(pk=i).guid for i in self.queue()], ['b', 'a']) + self.assertIn('kept sorted by "old"', out) + + def test_dry_run_writes_nothing(self): + self.episode(self.feed, 'g1') + PodcastQueue.objects.create(user=self.user, episode=self.episode(self.feed, 'q'), position=0) + g = self.ap.item(self.ap_feed, 'g1', read=1) + self.ap.item(self.ap_feed, 'g2', read=-1) + self.ap.feed('https://new.example.org/rss', feed_playback_speed=2.0) + self.ap.queue(g) + before = self.snapshot() + out = self.run_import('--dry-run') + self.assertEqual(self.snapshot(), before) + self.assertIn('DRY RUN', out) + self.assertIn('1 created', out) + self.assertIn('Queue: 1 in backup (0 not found); 1 before, 2 after', out) + self.assertNotIn('DRY RUN', self.run_import()) + self.assertNotEqual(self.snapshot(), before) + + def test_running_twice_changes_nothing_more(self): + self.episode(self.feed, 'g1') + self.episode(self.feed, 'stateless') + PodcastQueue.objects.create(user=self.user, episode=self.episode(self.feed, 'q'), position=0) + g1 = self.ap.item(self.ap_feed, 'g1', read=0, position=60, last_played=T0) + self.ap.item(self.ap_feed, 'stateless', read=0) + g2 = self.ap.item(self.ap_feed, 'g2', read=-1) + self.ap.item(self.ap_feed, 'g3', read=1) + self.ap.item(self.ap_feed, 'g4', read=1, pub=None) + self.ap.feed('https://new.example.org/rss', feed_playback_speed=2.0, keep_updated=0) + self.ap.queue(g2, g1) + self.ap.favorite(g1) + self.run_import() + first = self.snapshot() + out = self.run_import() + self.assertEqual(self.snapshot(), first) + self.assertIn('2 matched, 0 created', out) + self.assertIn('Settings: changed: speed 0, skip intro 0, skip outro 0, volume 0, include words 0, ' + 'exclude words 0, min duration 0, new episodes 0, auto-download 0', out) + self.assertIn('0 created, 0 skipped', out) + self.assertIn('Progress: 0 new, 0 updated, 4 unchanged, 0 kept (diora newer); 0 taken out', out) + + +class AfterFirstRefreshTests(ImportTestCase): + """The way to move: import, let the new feeds be fetched, import again.""" + + def test_second_import_after_the_first_fetch(self): + import feedparser + + from .views import FeedFetch, _refresh_feed + + ap_feed = self.ap.feed('https://new.example.org/rss') + days = {'old1': 1, 'old2': 2, 'new': 3, 'newest': 4} + for guid, read in (('old1', 0), ('old2', 1), ('new', -1), ('newest', 0)): + self.ap.item(ap_feed, guid, read=read, pub=T0 + timedelta(days=days[guid])) + self.run_import() + feed = PodcastFeed.objects.get(user=self.user) + self.assertEqual(sorted(feed.episodes.values_list('guid', flat=True)), ['new', 'old2']) + + def date(d): + return (T0 + timedelta(days=d)).strftime('%a, %d %b %Y %H:%M:%S GMT') + + items = ''.join( + f'