mrwnslz
54fbd58293
- PodcastSettings (Migration 0014): skip_back / skip_forward (5–60 s),
smart_played, speed_presets (0,5–3,5×), keep_skipped, delete_dequeue,
mobile_downloads, mobile_streaming (allow/ask/never), times_by_speed;
die ersten vier sind anfangs leer (null) und werden mit POST
settings/ {"initial": {...}} nur dort gesetzt, wo noch nichts steht
- GET/POST /podcasts/settings/ und state.settings mit allen Feldern,
geprüft, nichts ändert sich bei einem ungültigen Wert
- PodcastFeed: episode_sort, episode_filter, skip_silence in
feeds/<id>/settings/ und den Feeds des States
- Tempo pro Podcast bis 3,5× (Server und AntennaPod-Import)
- AntennaPod-Import: Sortierung, Filter und Stille pro Podcast, soweit
diora sie zeigen kann; nicht darstellbare werden gezählt
- Browser-Host: Sprungweiten, "gehört" und keep_skipped aus dem Konto,
Tempo bis 3,5×, neue Einstellungen werden gespeichert
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
878 lines
40 KiB
Python
878 lines
40 KiB
Python
"""Import an AntennaPod database backup into one user's podcasts.
|
|
|
|
AntennaPod's "Datenbank exportieren" writes its whole SQLite database. From
|
|
it come the subscriptions with their settings, the state of every episode
|
|
that has one (played, new, started, queued, favourite), the queue and which
|
|
episodes were on the phone (the app offers to download those again). See
|
|
the import_antennapod command for how to run it.
|
|
|
|
The backup is only read. Episodes without any state (and not on the phone)
|
|
are not imported: the feed's current episodes come with its next refresh
|
|
anyway. Those that diora already has, though, are taken out of the inbox
|
|
(AntennaPod: not new).
|
|
"""
|
|
|
|
import re
|
|
import sqlite3
|
|
import time
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone as dt_timezone
|
|
from itertools import groupby
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
from django.db import transaction
|
|
from django.db.models import Q
|
|
|
|
from .models import MAX_SPEED, MIN_SPEED, EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings
|
|
from .views import MAX_DESCRIPTION, _keep_sorted, _leave_inbox, _queue_order, _write_queue_order
|
|
|
|
CHUNK = 500 # ids per IN (...) query and rows per bulk write
|
|
|
|
# Feeds.feed_volume_adaption -> diora volume factor (0 = off = 1.0)
|
|
VOLUME_CODES = {1: 0.5, 2: 0.2, 3: 1.5, 4: 2.0, 5: 2.5}
|
|
# Feeds.new_episodes_action; 0 = the global setting (not in the backup)
|
|
NEW_EPISODE_ACTIONS = {1: PodcastFeed.NEW_INBOX, 2: PodcastFeed.NEW_IGNORE, 3: PodcastFeed.NEW_QUEUE}
|
|
# Feeds.auto_download; 1 = the global setting (not in the backup)
|
|
AUTO_DOWNLOAD_OFF, AUTO_DOWNLOAD_GLOBAL, AUTO_DOWNLOAD_ON = 0, 1, 2
|
|
# FeedItems.read
|
|
NEW, UNPLAYED, PLAYED = -1, 0, 1
|
|
SUBSCRIBED = 0 # Feeds.state
|
|
# FeedMedia.filesize above this is garbage (seen: 10^13); taken as unknown.
|
|
MAX_FILESIZE = 4_000_000_000
|
|
|
|
SETTING_FIELDS = ('playback_speed', 'skip_intro', 'skip_outro', 'volume', 'include_words',
|
|
'exclude_words', 'min_duration', 'new_episodes', 'auto_download', 'notify',
|
|
'episode_sort', 'episode_filter', 'skip_silence')
|
|
|
|
# Feeds.sort_order (SortOrder codes) -> episode_sort; 11 = the global order
|
|
# (not in the backup). The others (title Z-A, file name, size) diora can't show.
|
|
SORT_ORDERS = {1: 'old', 2: 'new', 3: 'title', 5: 'short', 6: 'long'}
|
|
GLOBAL_SORT = 11
|
|
# Feeds.hide (FeedItemFilter, comma-separated) -> episode_filter
|
|
FILTERS = {'unplayed': 'unplayed', 'played': 'played', 'paused': 'in_progress', 'is_favorite': 'favorites',
|
|
'queued': 'queued', 'downloaded': 'downloaded'}
|
|
# Feeds.feed_skip_silence (SkipSilence codes); 1 = the global setting
|
|
SKIP_SILENCE = {0: False, 2: True}
|
|
|
|
# Columns read from the backup, with the value to assume if an older
|
|
# AntennaPod didn't have the column yet.
|
|
FEED_COLUMNS = {
|
|
'id': None, 'title': '', 'custom_title': None, 'download_url': '', 'link': '', 'author': '',
|
|
'image_url': '', 'keep_updated': 1, 'include_filter': '', 'exclude_filter': '',
|
|
'minimal_duration_filter': -1, 'new_episodes_action': 0, 'feed_playback_speed': -1,
|
|
'feed_skip_intro': 0, 'feed_skip_ending': 0, 'feed_volume_adaption': 0, 'auto_download': 1,
|
|
'episode_notification': 0, 'state': SUBSCRIBED, 'sort_order': None, 'hide': '', 'feed_skip_silence': 1,
|
|
}
|
|
|
|
|
|
class AntennaPodImportError(Exception):
|
|
"""The file isn't an AntennaPod backup (or the user doesn't exist)."""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Small helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def normalize_url(url):
|
|
"""A URL reduced to what identifies it: no scheme (http = https), host in
|
|
lower case without a leading www., no default port, no trailing slash, no
|
|
fragment. None for anything but http(s)."""
|
|
url = (url or '').strip()
|
|
try:
|
|
parts = urlsplit(url)
|
|
port = parts.port
|
|
except ValueError:
|
|
return None
|
|
if parts.scheme.lower() not in ('http', 'https') or not parts.hostname:
|
|
return None
|
|
host = parts.hostname.lower()
|
|
if host.startswith('www.'):
|
|
host = host[4:]
|
|
if port and port not in (80, 443):
|
|
host = f'{host}:{port}'
|
|
path = parts.path.rstrip('/')
|
|
return host + path + (f'?{parts.query}' if parts.query else '')
|
|
|
|
|
|
def _web_url(url, max_length=1000):
|
|
"""url if it is a usable http(s) URL for a URLField, else ''."""
|
|
url = (url or '').strip()
|
|
if normalize_url(url) is None or len(url) > max_length:
|
|
return ''
|
|
return url
|
|
|
|
|
|
# AntennaPod's filter syntax: words separated by spaces, "quoted phrases".
|
|
_TERM = re.compile(r'([^"]\S*|".+?")\s*')
|
|
|
|
|
|
def parse_filter_terms(raw):
|
|
"""AntennaPod filter text -> its terms, e.g. 'a "b c"' -> ['a', 'b c']."""
|
|
terms = []
|
|
for m in _TERM.finditer(raw or ''):
|
|
# diora separates its words by commas, so a comma can't be in one.
|
|
term = ' '.join(m.group(1).replace('"', '').replace(',', ' ').split())
|
|
if term and term.casefold() not in {t.casefold() for t in terms}:
|
|
terms.append(term)
|
|
return terms
|
|
|
|
|
|
def filter_words(raw, max_length=500):
|
|
"""AntennaPod filter text -> diora's comma-separated words ('' if none);
|
|
as many whole terms as fit into the field."""
|
|
out = ''
|
|
for term in parse_filter_terms(raw):
|
|
candidate = f'{out}, {term}' if out else term
|
|
if len(candidate) > max_length:
|
|
break
|
|
out = candidate
|
|
return out
|
|
|
|
|
|
def _ms(value):
|
|
"""A millisecond timestamp -> aware datetime; None for 0/empty/absurd."""
|
|
try:
|
|
value = int(value or 0)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if value <= 0:
|
|
return None
|
|
try:
|
|
return datetime.fromtimestamp(value / 1000, tz=dt_timezone.utc)
|
|
except (OverflowError, OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def _chunks(seq, size=CHUNK):
|
|
seq = list(seq)
|
|
for i in range(0, len(seq), size):
|
|
yield seq[i:i + size]
|
|
|
|
|
|
@contextmanager
|
|
def _given_updated_at():
|
|
"""Let bulk_create keep the EpisodeProgress.updated_at we set (auto_now
|
|
would put now everywhere, and the history and the statistics go by it)."""
|
|
f = EpisodeProgress._meta.get_field('updated_at')
|
|
f.auto_now = False
|
|
try:
|
|
yield
|
|
finally:
|
|
f.auto_now = True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Feed settings
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def episode_sort(code):
|
|
"""Feeds.sort_order -> (episode_sort or None, representable). None and
|
|
the global order are no setting of the feed (representable, nothing to set)."""
|
|
try:
|
|
code = int(code) if code not in (None, '') else None
|
|
except (TypeError, ValueError):
|
|
return None, False
|
|
if code is None or code == GLOBAL_SORT:
|
|
return None, True
|
|
return SORT_ORDERS.get(code), code in SORT_ORDERS
|
|
|
|
|
|
def episode_filter(raw):
|
|
"""Feeds.hide -> (episode_filter or None, representable): the filter only
|
|
if diora can show all of its terms (a part of it would be another list)."""
|
|
terms = [t.strip() for t in (raw or '').split(',') if t.strip()]
|
|
if not terms:
|
|
return None, True
|
|
if any(t not in FILTERS for t in terms):
|
|
return None, False
|
|
return list(dict.fromkeys(FILTERS[t] for t in terms)), True
|
|
|
|
|
|
def feed_settings(ap, global_auto_download=False):
|
|
"""diora settings for one AntennaPod feed row: only those AntennaPod sets
|
|
to something other than its default. Returns (settings, not_kept_updated)."""
|
|
s = {}
|
|
speed = ap['feed_playback_speed']
|
|
if speed is not None and speed > 0: # -1 = the global speed
|
|
s['playback_speed'] = min(max(round(float(speed), 2), MIN_SPEED), MAX_SPEED)
|
|
for src, dst in (('feed_skip_intro', 'skip_intro'), ('feed_skip_ending', 'skip_outro')):
|
|
seconds = int(ap[src] or 0)
|
|
if seconds > 0:
|
|
s[dst] = min(seconds, 600)
|
|
volume = VOLUME_CODES.get(ap['feed_volume_adaption'] or 0)
|
|
if volume:
|
|
s['volume'] = volume
|
|
for src, dst in (('include_filter', 'include_words'), ('exclude_filter', 'exclude_words')):
|
|
words = filter_words(ap[src])
|
|
if words:
|
|
s[dst] = words
|
|
minutes = int(ap['minimal_duration_filter'] or 0) // 60 # seconds, -1 = none
|
|
if minutes > 0:
|
|
s['min_duration'] = min(minutes, 600)
|
|
action = NEW_EPISODE_ACTIONS.get(ap['new_episodes_action'] or 0)
|
|
# "Don't update this podcast": AntennaPod never shows anything new of it.
|
|
not_kept_updated = not ap['keep_updated'] and action != PodcastFeed.NEW_QUEUE
|
|
if not_kept_updated:
|
|
action = PodcastFeed.NEW_IGNORE
|
|
if action:
|
|
s['new_episodes'] = action
|
|
code = ap['auto_download']
|
|
if code == AUTO_DOWNLOAD_ON:
|
|
s['auto_download'] = True
|
|
elif code == AUTO_DOWNLOAD_GLOBAL:
|
|
s['auto_download'] = bool(global_auto_download)
|
|
elif code == AUTO_DOWNLOAD_OFF:
|
|
s['auto_download'] = False
|
|
# Feeds.episode_notification: 1 = a notification for each new episode
|
|
# (0, AntennaPod's default, leaves diora's setting alone).
|
|
if ap['episode_notification']:
|
|
s['notify'] = True
|
|
# The podcast's episode list: its order and filter, where diora has them.
|
|
sort, _ = episode_sort(ap['sort_order'])
|
|
if sort:
|
|
s['episode_sort'] = sort
|
|
filters, _ = episode_filter(ap['hide'])
|
|
if filters:
|
|
s['episode_filter'] = filters
|
|
# Skip silence: off or on for this podcast (the global setting: diora's stays).
|
|
if ap['feed_skip_silence'] in SKIP_SILENCE:
|
|
s['skip_silence'] = SKIP_SILENCE[ap['feed_skip_silence']]
|
|
return s, not_kept_updated
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The import
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@dataclass
|
|
class ItemState:
|
|
"""What one AntennaPod episode says about its diora episode."""
|
|
played: bool
|
|
dismissed: bool
|
|
position: int # seconds
|
|
favorite: bool
|
|
activity: datetime | None # last played or finished, if known
|
|
when: datetime # what updated_at becomes: activity, else pub date, else the backup date
|
|
|
|
def merge(self, other):
|
|
"""Two AntennaPod episodes (duplicate feeds) for one diora episode:
|
|
the one listened to last, a star from either."""
|
|
a = self if self.when >= other.when else other
|
|
return ItemState(a.played, a.dismissed, a.position, self.favorite or other.favorite, a.activity, a.when)
|
|
|
|
|
|
@dataclass
|
|
class Stats:
|
|
backup_feeds: int = 0
|
|
not_subscribed: int = 0
|
|
matched: int = 0
|
|
created: int = 0
|
|
duplicates: int = 0
|
|
created_titles: list = field(default_factory=list)
|
|
skipped_feeds: list = field(default_factory=list) # (title, url)
|
|
settings: dict = field(default_factory=lambda: {name: 0 for name in SETTING_FIELDS})
|
|
not_kept_updated: int = 0
|
|
sort_unshown: int = 0 # per-feed orders / filters diora has no equivalent of
|
|
filter_unshown: int = 0
|
|
auto_download_on: int = 0
|
|
auto_download_explicit: int = 0
|
|
auto_download_global: int = 0
|
|
auto_download_default: bool = False
|
|
auto_download_default_changed: bool = False
|
|
stateful: int = 0
|
|
by_guid: int = 0
|
|
by_url: int = 0
|
|
episodes_created: int = 0
|
|
no_audio: int = 0
|
|
in_skipped_feeds: int = 0
|
|
stateless_matched: int = 0
|
|
progress_new: int = 0
|
|
progress_updated: int = 0
|
|
progress_unchanged: int = 0
|
|
progress_kept: int = 0
|
|
left_inbox: int = 0
|
|
favorites_backup: int = 0
|
|
favorites_before: int = 0
|
|
favorites_after: int = 0
|
|
queue_backup: int = 0
|
|
queue_missing: int = 0
|
|
queue_before: int = 0
|
|
queue_after: int = 0
|
|
queue_appended: int = 0
|
|
keep_sorted: str = ''
|
|
inbox_before: int = 0
|
|
inbox_after: int = 0
|
|
downloads_backup: int = 0
|
|
downloads: int = 0
|
|
downloads_bytes: int = 0
|
|
downloads_created: int = 0
|
|
downloads_missing: int = 0
|
|
downloads_offered: bool = True
|
|
ignored_left_inbox: int = 0
|
|
queue_removed: int = 0
|
|
removed_titles: list = field(default_factory=list)
|
|
diora_only_left_inbox: int = 0
|
|
backup_date: datetime | None = None
|
|
seconds: float = 0.0
|
|
|
|
|
|
def _inbox_count(user):
|
|
hidden = EpisodeProgress.objects.filter(user=user).filter(Q(played=True) | Q(dismissed=True))
|
|
return PodcastEpisode.objects.filter(feed__user=user).exclude(id__in=hidden.values('episode_id')).count()
|
|
|
|
|
|
def _open_backup(path):
|
|
uri = Path(path).resolve().as_uri() + '?mode=ro'
|
|
db = None
|
|
try:
|
|
db = sqlite3.connect(uri, uri=True)
|
|
tables = {r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
except sqlite3.DatabaseError as e:
|
|
if db is not None:
|
|
db.close()
|
|
raise AntennaPodImportError(f'{path} is not an SQLite database: {e}')
|
|
missing = {'Feeds', 'FeedItems', 'FeedMedia', 'Queue', 'Favorites'} - tables
|
|
if missing:
|
|
db.close()
|
|
raise AntennaPodImportError(
|
|
f'{path} is not an AntennaPod backup (missing tables: {", ".join(sorted(missing))})')
|
|
return db
|
|
|
|
|
|
def _columns(db, table):
|
|
return {r[1] for r in db.execute(f'PRAGMA table_info({table})')}
|
|
|
|
|
|
class AntennaPodImport:
|
|
def __init__(self, user, path, backup_date, global_auto_download=False, offer_downloads=True,
|
|
quiet_diora_only=False, replace=False):
|
|
self.user = user
|
|
self.path = path
|
|
self.backup_date = backup_date
|
|
self.global_auto_download = global_auto_download
|
|
# False: the app doesn't offer AntennaPod's downloads (they stay in AntennaPod).
|
|
self.offer_downloads = offer_downloads
|
|
# True: subscriptions only diora has keep their episodes, but the
|
|
# ones out before the backup leave the inbox.
|
|
self.quiet_diora_only = quiet_diora_only
|
|
# A clean slate: AntennaPod's state is the new start. Podcasts only
|
|
# diora has are unsubscribed, the queue is AntennaPod's alone, and
|
|
# AntennaPod wins also over later changes in diora.
|
|
self.replace = replace
|
|
self.stats = Stats(backup_date=backup_date)
|
|
|
|
def run(self, dry_run=False):
|
|
started = time.monotonic()
|
|
db = _open_backup(self.path)
|
|
try:
|
|
with transaction.atomic():
|
|
self._run(db)
|
|
if dry_run:
|
|
transaction.set_rollback(True)
|
|
finally:
|
|
db.close()
|
|
self.stats.seconds = time.monotonic() - started
|
|
return self.stats
|
|
|
|
def _run(self, db):
|
|
st = self.stats
|
|
user = self.user
|
|
st.queue_before = len(_queue_order(user))
|
|
st.inbox_before = _inbox_count(user)
|
|
st.favorites_before = EpisodeProgress.objects.filter(user=user, favorite=True).count()
|
|
|
|
# AntennaPod downloads new episodes of every podcast by default: so
|
|
# do diora's future subscriptions.
|
|
settings = PodcastSettings.for_user(user)
|
|
if self.global_auto_download and not settings.auto_download_default:
|
|
settings.auto_download_default = True
|
|
settings.save(update_fields=['auto_download_default'])
|
|
st.auto_download_default_changed = True
|
|
st.auto_download_default = settings.auto_download_default
|
|
self.new_feed = PodcastSettings.new_feed_defaults(user)
|
|
|
|
feed_map = self._import_feeds(db) # AntennaPod feed id -> diora feed id
|
|
if self.replace:
|
|
self._remove_diora_only(feed_map)
|
|
queue_items = [r[0] for r in db.execute('SELECT feeditem FROM Queue ORDER BY id')]
|
|
favorite_items = {r[0] for r in db.execute('SELECT feeditem FROM Favorites')}
|
|
st.queue_backup = len(queue_items)
|
|
st.favorites_backup = len(favorite_items)
|
|
|
|
states, not_new, item_episode, downloads = self._import_episodes(
|
|
db, feed_map, set(queue_items), favorite_items)
|
|
self._import_progress(states, not_new)
|
|
self._quiet_inbox(feed_map, states)
|
|
self._import_queue(queue_items, item_episode)
|
|
self._import_downloads(settings, downloads, item_episode)
|
|
|
|
st.queue_after = len(_queue_order(user))
|
|
st.inbox_after = _inbox_count(user)
|
|
st.favorites_after = EpisodeProgress.objects.filter(user=user, favorite=True).count()
|
|
|
|
# -- feeds --------------------------------------------------------------
|
|
|
|
def _import_feeds(self, db):
|
|
st = self.stats
|
|
have = _columns(db, 'Feeds')
|
|
select = ', '.join(c if c in have else f'{default!r} AS {c}' if default is not None else f'NULL AS {c}'
|
|
for c, default in FEED_COLUMNS.items())
|
|
db.row_factory = sqlite3.Row
|
|
try:
|
|
rows = db.execute(f'SELECT {select} FROM Feeds ORDER BY id').fetchall()
|
|
finally:
|
|
db.row_factory = None
|
|
st.backup_feeds = len(rows)
|
|
|
|
mine = {}
|
|
for f in PodcastFeed.objects.filter(user=self.user).order_by('id'):
|
|
mine.setdefault(normalize_url(f.rss_url) or f.rss_url, f)
|
|
|
|
feed_map = {} # AntennaPod id -> PodcastFeed (saved or still to create)
|
|
to_create = {} # normalised URL -> new PodcastFeed
|
|
changed = {} # diora id -> PodcastFeed with changed settings
|
|
for ap in rows:
|
|
if ap['state'] != SUBSCRIBED:
|
|
st.not_subscribed += 1
|
|
continue
|
|
url = (ap['download_url'] or '').strip()
|
|
title = (ap['custom_title'] or ap['title'] or url).strip()
|
|
key = normalize_url(url)
|
|
if key is None or len(url) > 1000:
|
|
st.skipped_feeds.append((title, url))
|
|
continue
|
|
feed = mine.get(key) or to_create.get(key)
|
|
if feed is None:
|
|
feed = PodcastFeed(
|
|
user=self.user, rss_url=url, title=title[:300], author=(ap['author'] or '')[:300],
|
|
link=_web_url(ap['link']), artwork_url=_web_url(ap['image_url']), **self.new_feed,
|
|
)
|
|
to_create[key] = feed
|
|
st.created += 1
|
|
st.created_titles.append(feed.title)
|
|
elif any(f is feed for f in feed_map.values()):
|
|
st.duplicates += 1
|
|
elif feed.pk:
|
|
st.matched += 1
|
|
feed_map[ap['id']] = feed
|
|
|
|
settings, not_kept_updated = feed_settings(ap, self.global_auto_download)
|
|
st.not_kept_updated += not_kept_updated
|
|
st.sort_unshown += not episode_sort(ap['sort_order'])[1]
|
|
st.filter_unshown += not episode_filter(ap['hide'])[1]
|
|
st.auto_download_explicit += ap['auto_download'] == AUTO_DOWNLOAD_ON
|
|
st.auto_download_global += ap['auto_download'] == AUTO_DOWNLOAD_GLOBAL
|
|
for name, value in settings.items():
|
|
if getattr(feed, name) != value:
|
|
setattr(feed, name, value)
|
|
st.settings[name] += 1
|
|
if feed.pk:
|
|
changed[feed.pk] = feed
|
|
feed.auto_queue = feed.new_episodes == PodcastFeed.NEW_QUEUE
|
|
|
|
if changed:
|
|
PodcastFeed.objects.bulk_update(list(changed.values()), [*SETTING_FIELDS, 'auto_queue'],
|
|
batch_size=CHUNK)
|
|
if to_create:
|
|
# last_refreshed_at stays empty: the next refresh fetches them,
|
|
# with the first-fetch rule (only the newest episode is new).
|
|
PodcastFeed.objects.bulk_create(list(to_create.values()), batch_size=CHUNK)
|
|
ids = {}
|
|
for urls in _chunks(f.rss_url for f in to_create.values()):
|
|
ids.update(PodcastFeed.objects.filter(user=self.user, rss_url__in=urls).values_list('rss_url', 'id'))
|
|
for f in to_create.values():
|
|
f.pk = ids[f.rss_url]
|
|
st.auto_download_on = sum(1 for f in {id(f): f for f in feed_map.values()}.values() if f.auto_download)
|
|
return {ap_id: f.pk for ap_id, f in feed_map.items()}
|
|
|
|
# -- episodes -----------------------------------------------------------
|
|
|
|
def _import_episodes(self, db, feed_map, queued, favorites):
|
|
"""Match AntennaPod's episodes to diora's, create the missing ones
|
|
that have a state (or were downloaded). Returns ({episode id: ItemState}
|
|
for those with a state, {episode ids} of matched ones without (not new),
|
|
{AntennaPod item id: episode id}, [(pub date ms, AntennaPod item id,
|
|
size)] of the downloaded ones)."""
|
|
st = self.stats
|
|
backup = self.backup_date
|
|
states = {}
|
|
not_new = set()
|
|
item_episode = {}
|
|
downloads = []
|
|
to_create = {} # (diora feed id, guid) -> [ItemState, AntennaPod item id, audio url, duration, pub_date]
|
|
|
|
media = _columns(db, 'FeedMedia')
|
|
cur = db.execute(
|
|
'SELECT i.feed, i.id, i.item_identifier, i.read, i.pubDate, m.download_url, m.duration, '
|
|
'm.position, m.last_played_time, m.playback_completion_date, '
|
|
+ ('m.downloaded, ' if 'downloaded' in media else '0, ')
|
|
+ ('m.filesize ' if 'filesize' in media else '0 ')
|
|
+ 'FROM FeedItems i LEFT JOIN FeedMedia m ON m.feeditem = i.id ORDER BY i.feed, i.id'
|
|
)
|
|
|
|
def rows():
|
|
while True:
|
|
batch = cur.fetchmany(5000)
|
|
if not batch:
|
|
return
|
|
yield from batch
|
|
|
|
for ap_feed, items in groupby(rows(), key=lambda r: r[0]):
|
|
feed_id = feed_map.get(ap_feed)
|
|
by_guid = by_url = None
|
|
for (_, item_id, ident, read, pub_ms, audio, duration, position, last_played, completed,
|
|
downloaded, size) in items:
|
|
position = max(int(position or 0), 0) // 1000
|
|
activity = max(filter(None, (_ms(last_played), _ms(completed))), default=None)
|
|
in_queue = item_id in queued
|
|
favorite = item_id in favorites
|
|
stateful = read in (NEW, PLAYED) or position > 0 or activity is not None or in_queue or favorite
|
|
# FeedMedia.downloaded: when the file was downloaded (ms; 1 from
|
|
# older versions), 0 = not on the phone.
|
|
downloaded = (downloaded or 0) > 0
|
|
if downloaded:
|
|
size = int(size or 0)
|
|
downloads.append((int(pub_ms or 0), item_id, size if 0 < size <= MAX_FILESIZE else 0))
|
|
if feed_id is None:
|
|
st.in_skipped_feeds += stateful
|
|
continue
|
|
if by_guid is None:
|
|
by_guid, by_url = {}, {}
|
|
for ep_id, guid, url in (PodcastEpisode.objects.filter(feed_id=feed_id)
|
|
.order_by('id').values_list('id', 'guid', 'audio_url')):
|
|
by_guid.setdefault(guid, ep_id)
|
|
by_url.setdefault(normalize_url(url), ep_id)
|
|
ident = (ident or '').strip()
|
|
audio = (audio or '').strip()
|
|
guid = (ident or audio)[:1000]
|
|
ep_id = by_guid.get(guid) if guid else None
|
|
how = 'guid'
|
|
if ep_id is None and audio:
|
|
ep_id = by_url.get(normalize_url(audio))
|
|
how = 'url'
|
|
if not stateful:
|
|
if ep_id is not None:
|
|
not_new.add(ep_id)
|
|
item_episode[item_id] = ep_id
|
|
continue
|
|
if not downloaded:
|
|
continue
|
|
# On the phone in AntennaPod but not in diora yet (a feed
|
|
# not fetched so far): created as not new, so the app can
|
|
# offer to download it again.
|
|
else:
|
|
st.stateful += 1
|
|
played = read == PLAYED
|
|
pub_date = _ms(pub_ms)
|
|
state = ItemState(
|
|
played=played,
|
|
# Only AntennaPod's "new" stays in the inbox; queued is never new.
|
|
dismissed=read != NEW or in_queue,
|
|
position=0 if played else position,
|
|
favorite=favorite,
|
|
activity=activity,
|
|
when=activity or (min(pub_date, backup) if pub_date else backup),
|
|
)
|
|
if ep_id is not None:
|
|
st.by_guid += how == 'guid'
|
|
st.by_url += how == 'url'
|
|
states[ep_id] = state.merge(states[ep_id]) if ep_id in states else state
|
|
item_episode[item_id] = ep_id
|
|
continue
|
|
if normalize_url(audio) is None or len(audio) > 1000:
|
|
st.no_audio += stateful
|
|
continue
|
|
key = (feed_id, guid)
|
|
if key in to_create:
|
|
pending = to_create[key]
|
|
pending[0] = state.merge(pending[0])
|
|
pending[1].append(item_id)
|
|
else:
|
|
to_create[key] = [state, [item_id], audio, max(int(duration or 0), 0) // 1000, pub_date]
|
|
st.downloads_created += not stateful
|
|
|
|
if to_create:
|
|
self._create_episodes(db, to_create, states, item_episode)
|
|
not_new -= states.keys()
|
|
st.stateless_matched = len(not_new)
|
|
return states, not_new, item_episode, downloads
|
|
|
|
def _create_episodes(self, db, to_create, states, item_episode):
|
|
st = self.stats
|
|
# A few at a time: shownotes can be large.
|
|
for keys in _chunks(to_create, 200):
|
|
first_item = {to_create[key][1][0]: key for key in keys}
|
|
marks = ','.join('?' * len(first_item))
|
|
details = {
|
|
first_item[item_id]: (title, description, image)
|
|
for item_id, title, description, image in db.execute(
|
|
f'SELECT id, title, description, image_url FROM FeedItems WHERE id IN ({marks})',
|
|
list(first_item))
|
|
}
|
|
batch = []
|
|
for key in keys:
|
|
_, _, audio, duration, pub_date = to_create[key]
|
|
title, description, image = details.get(key, ('', '', ''))
|
|
batch.append(PodcastEpisode(
|
|
feed_id=key[0], guid=key[1], title=(title or 'Untitled')[:500],
|
|
description=(description or '')[:MAX_DESCRIPTION], audio_url=audio,
|
|
duration_seconds=duration, pub_date=pub_date, artwork_url=_web_url(image),
|
|
))
|
|
PodcastEpisode.objects.bulk_create(batch)
|
|
st.episodes_created = len(to_create) - st.downloads_created
|
|
|
|
by_feed = {}
|
|
for feed_id, guid in to_create:
|
|
by_feed.setdefault(feed_id, []).append(guid)
|
|
for feed_id, guids in by_feed.items():
|
|
for chunk in _chunks(guids):
|
|
created = PodcastEpisode.objects.filter(feed_id=feed_id, guid__in=chunk).values_list('guid', 'id')
|
|
for guid, ep_id in created:
|
|
state, item_ids = to_create[(feed_id, guid)][:2]
|
|
states[ep_id] = state
|
|
for item_id in item_ids:
|
|
item_episode[item_id] = ep_id
|
|
|
|
# -- progress -----------------------------------------------------------
|
|
|
|
def _import_progress(self, states, not_new):
|
|
st = self.stats
|
|
user = self.user
|
|
backup = self.backup_date
|
|
existing = {}
|
|
for ids in _chunks([*states, *not_new]):
|
|
for p in EpisodeProgress.objects.filter(user=user, episode_id__in=ids):
|
|
existing[p.episode_id] = p
|
|
|
|
def diora_newer(p, activity):
|
|
# Changed in diora after the backup was made (and after the
|
|
# episode was last played in AntennaPod): diora's state stays.
|
|
return not self.replace and p.updated_at > backup and (activity is None or p.updated_at > activity)
|
|
|
|
new_rows, changed, favorite_only, dismiss_only = [], [], [], []
|
|
for ep_id, s in states.items():
|
|
p = existing.get(ep_id)
|
|
if p is None:
|
|
new_rows.append(EpisodeProgress(
|
|
user=user, episode_id=ep_id, position_seconds=s.position, played=s.played,
|
|
dismissed=s.dismissed, favorite=s.favorite, updated_at=s.when,
|
|
))
|
|
st.progress_new += 1
|
|
continue
|
|
favorite = p.favorite or s.favorite # stars from both
|
|
if diora_newer(p, s.activity):
|
|
st.progress_kept += 1
|
|
if favorite != p.favorite:
|
|
favorite_only.append(p.pk)
|
|
continue
|
|
wanted = (s.position, s.played, s.dismissed, favorite, s.when)
|
|
if (p.position_seconds, p.played, p.dismissed, p.favorite, p.updated_at) == wanted:
|
|
st.progress_unchanged += 1
|
|
continue
|
|
p.position_seconds, p.played, p.dismissed, p.favorite, p.updated_at = wanted
|
|
changed.append(p)
|
|
st.progress_updated += 1
|
|
|
|
# Known to AntennaPod, but not new there: out of the inbox.
|
|
for ep_id in not_new:
|
|
p = existing.get(ep_id)
|
|
if p is None:
|
|
new_rows.append(EpisodeProgress(user=user, episode_id=ep_id, dismissed=True, updated_at=backup))
|
|
st.left_inbox += 1
|
|
elif not p.played and not p.dismissed and not diora_newer(p, None):
|
|
dismiss_only.append(p.pk)
|
|
st.left_inbox += 1
|
|
|
|
with _given_updated_at():
|
|
EpisodeProgress.objects.bulk_create(new_rows, batch_size=CHUNK)
|
|
# bulk_update writes updated_at as given (no auto_now there).
|
|
EpisodeProgress.objects.bulk_update(
|
|
changed, ['position_seconds', 'played', 'dismissed', 'favorite', 'updated_at'], batch_size=CHUNK)
|
|
# A star or leaving the inbox isn't listening: updated_at stays.
|
|
for ids in _chunks(favorite_only):
|
|
EpisodeProgress.objects.filter(pk__in=ids).update(favorite=True)
|
|
for ids in _chunks(dismiss_only):
|
|
EpisodeProgress.objects.filter(pk__in=ids).update(dismissed=True)
|
|
|
|
def _quiet_inbox(self, feed_map, states):
|
|
"""Episodes AntennaPod never showed as new, though diora has them in
|
|
the inbox: those of podcasts whose new episodes are ignored (AntennaPod
|
|
stopped fetching archived ones, so it never saw them), and with
|
|
quiet_diora_only those of podcasts AntennaPod doesn't have. Only ones
|
|
out before the backup; AntennaPod's own new ones stay."""
|
|
st = self.stats
|
|
user = self.user
|
|
backup = self.backup_date
|
|
imported = set(feed_map.values())
|
|
feeds = set(PodcastFeed.objects.filter(pk__in=imported, new_episodes=PodcastFeed.NEW_IGNORE)
|
|
.values_list('pk', flat=True))
|
|
diora_only = set()
|
|
if self.quiet_diora_only:
|
|
diora_only = set(PodcastFeed.objects.filter(user=user).exclude(pk__in=imported).values_list('pk', flat=True))
|
|
if not feeds and not diora_only:
|
|
return
|
|
keep = {ep_id for ep_id, s in states.items() if not s.dismissed}
|
|
hidden = EpisodeProgress.objects.filter(user=user).filter(Q(played=True) | Q(dismissed=True))
|
|
rows = (PodcastEpisode.objects.filter(feed_id__in=feeds | diora_only)
|
|
.filter(Q(pub_date__lte=backup) | Q(pub_date__isnull=True))
|
|
.exclude(id__in=hidden.values('episode_id'))
|
|
.values_list('id', 'feed_id'))
|
|
ids = []
|
|
for ep_id, feed_id in rows:
|
|
if ep_id in keep:
|
|
continue
|
|
ids.append(ep_id)
|
|
if feed_id in feeds:
|
|
st.ignored_left_inbox += 1
|
|
else:
|
|
st.diora_only_left_inbox += 1
|
|
# Changed in diora after the backup: diora's state stays.
|
|
newer = set()
|
|
for chunk in ([] if self.replace else _chunks(ids)):
|
|
newer.update(EpisodeProgress.objects.filter(user=user, episode_id__in=chunk, updated_at__gt=backup)
|
|
.values_list('episode_id', flat=True))
|
|
ids = [i for i in ids if i not in newer]
|
|
for chunk in _chunks(ids):
|
|
EpisodeProgress.objects.filter(user=user, episode_id__in=chunk, dismissed=False).update(dismissed=True)
|
|
have = set(EpisodeProgress.objects.filter(user=user, episode_id__in=chunk)
|
|
.values_list('episode_id', flat=True))
|
|
with _given_updated_at():
|
|
EpisodeProgress.objects.bulk_create(
|
|
[EpisodeProgress(user=user, episode_id=i, dismissed=True, updated_at=backup)
|
|
for i in chunk if i not in have],
|
|
ignore_conflicts=True,
|
|
)
|
|
|
|
def _remove_diora_only(self, feed_map):
|
|
"""Unsubscribe the podcasts the backup doesn't have (their episodes,
|
|
states and queue entries go with them)."""
|
|
st = self.stats
|
|
gone = PodcastFeed.objects.filter(user=self.user).exclude(pk__in=set(feed_map.values()))
|
|
st.removed_titles = sorted(gone.values_list('title', flat=True))
|
|
gone.delete()
|
|
|
|
# -- queue --------------------------------------------------------------
|
|
|
|
def _import_queue(self, queue_items, item_episode):
|
|
"""AntennaPod's queue first, then what else was in diora's."""
|
|
st = self.stats
|
|
user = self.user
|
|
before = _queue_order(user)
|
|
front = []
|
|
for item_id in queue_items:
|
|
ep_id = item_episode.get(item_id)
|
|
if ep_id is None:
|
|
st.queue_missing += 1
|
|
elif ep_id not in front:
|
|
front.append(ep_id)
|
|
if not front:
|
|
return
|
|
in_front = set(front)
|
|
rest = [ep_id for ep_id in before if ep_id not in in_front]
|
|
if self.replace and rest:
|
|
for chunk in _chunks(rest):
|
|
PodcastQueue.objects.filter(user=user, episode_id__in=chunk).delete()
|
|
st.queue_removed = len(rest)
|
|
before = [ep_id for ep_id in before if ep_id in in_front]
|
|
rest = []
|
|
st.queue_appended = len(rest)
|
|
present = set(before)
|
|
PodcastQueue.objects.bulk_create(
|
|
[PodcastQueue(user=user, episode_id=ep_id, position=len(before) + i)
|
|
for i, ep_id in enumerate(e for e in front if e not in present)],
|
|
batch_size=CHUNK,
|
|
)
|
|
_write_queue_order(user, front + rest)
|
|
_leave_inbox(user, front) # queued = not new
|
|
if _keep_sorted(user):
|
|
st.keep_sorted = PodcastSettings.objects.get(user=user).keep_sorted
|
|
|
|
# -- downloads ----------------------------------------------------------
|
|
|
|
def _import_downloads(self, settings, downloads, item_episode):
|
|
"""The episodes AntennaPod had on the phone, newest first: the app
|
|
can't take its files over, so it offers to download them again.
|
|
Replaces what an earlier import left."""
|
|
st = self.stats
|
|
st.downloads_backup = len(downloads)
|
|
entries, seen = [], set()
|
|
for _, item_id, size in sorted(downloads, key=lambda d: (-d[0], -d[1])):
|
|
ep_id = item_episode.get(item_id)
|
|
if ep_id is None:
|
|
st.downloads_missing += 1
|
|
continue
|
|
if ep_id in seen:
|
|
continue
|
|
seen.add(ep_id)
|
|
entries.append({'episode_id': ep_id, 'size': size})
|
|
st.downloads = len(entries)
|
|
st.downloads_bytes = sum(e['size'] for e in entries)
|
|
st.downloads_offered = self.offer_downloads
|
|
settings.import_downloads = entries if self.offer_downloads else []
|
|
settings.save(update_fields=['import_downloads'])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Summary
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SETTING_LABELS = {
|
|
'playback_speed': 'speed', 'skip_intro': 'skip intro', 'skip_outro': 'skip outro', 'volume': 'volume',
|
|
'include_words': 'include words', 'exclude_words': 'exclude words', 'min_duration': 'min duration',
|
|
'new_episodes': 'new episodes', 'auto_download': 'auto-download', 'notify': 'notifications',
|
|
'episode_sort': 'episode order', 'episode_filter': 'episode filter', 'skip_silence': 'skip silence',
|
|
}
|
|
|
|
|
|
def _on(flag):
|
|
return 'on' if flag else 'off'
|
|
|
|
|
|
def summary_lines(st, username, path, dry_run, global_auto_download):
|
|
when = st.backup_date.strftime('%Y-%m-%d %H:%M UTC') if st.backup_date else '?'
|
|
subscribed = st.backup_feeds - st.not_subscribed
|
|
settings = ', '.join(f'{SETTING_LABELS[k]} {v}' for k, v in st.settings.items())
|
|
lines = [
|
|
f"AntennaPod import for user '{username}' from {Path(path).name} (backup date {when})"
|
|
+ (' -- DRY RUN, nothing was written' if dry_run else ''),
|
|
f'Feeds: {st.backup_feeds} in backup, {subscribed} subscribed ({st.not_subscribed} not subscribed '
|
|
f'ignored); {st.matched} matched, {st.created} created, {len(st.skipped_feeds)} skipped (not http/https), '
|
|
f'{st.duplicates} duplicates'
|
|
+ (f', {len(st.removed_titles)} only in diora unsubscribed (--replace)' if st.removed_titles else ''),
|
|
f'Settings: changed: {settings}',
|
|
f' {st.not_kept_updated} feeds not kept updated in AntennaPod -> new episodes "ignore"',
|
|
f' episode lists diora can\'t show that way (left as they are): {st.sort_unshown} orders, '
|
|
f'{st.filter_unshown} filters',
|
|
f'Auto-download: on for {st.auto_download_on} feeds ({st.auto_download_explicit} on in AntennaPod, '
|
|
f"{st.auto_download_global} use AntennaPod's global setting -> {_on(global_auto_download)} "
|
|
f'by --global-auto-download); default for new subscriptions: {_on(st.auto_download_default)}'
|
|
+ (' (set by this import)' if st.auto_download_default_changed else ''),
|
|
f'Episodes: {st.stateful} with a state: {st.by_guid} matched by guid, {st.by_url} by URL, '
|
|
f'{st.episodes_created} created, {st.no_audio} skipped (no audio URL), {st.in_skipped_feeds} in skipped feeds',
|
|
f' {st.stateless_matched} without a state matched to diora episodes (not new in AntennaPod)',
|
|
f'Progress: {st.progress_new} new, {st.progress_updated} updated, '
|
|
f'{st.progress_unchanged} unchanged, {st.progress_kept} kept (diora newer); '
|
|
f'{st.left_inbox} taken out of the inbox (not new in AntennaPod)',
|
|
f'Favourites: {st.favorites_backup} in backup; {st.favorites_before} before, {st.favorites_after} after',
|
|
f'Queue: {st.queue_backup} in backup ({st.queue_missing} not found); {st.queue_before} before, '
|
|
f'{st.queue_after} after ({st.queue_appended} diora entries after AntennaPod\'s'
|
|
+ (f', {st.queue_removed} removed (--replace)' if st.queue_removed else '') + ')'
|
|
+ (f'; kept sorted by "{st.keep_sorted}"' if st.keep_sorted else ''),
|
|
f'Inbox: {st.inbox_before} before, {st.inbox_after} after; out of it: {st.ignored_left_inbox} of podcasts '
|
|
f'whose new episodes are ignored (AntennaPod never showed them)'
|
|
+ (f', {st.diora_only_left_inbox} of podcasts only diora has' if st.diora_only_left_inbox else ''),
|
|
f'Downloads: {st.downloads} on the phone in AntennaPod ({st.downloads_bytes / 1_000_000:.0f} MB) '
|
|
+ ('-> offered to the app for download' if st.downloads_offered else '-> not offered (--no-downloads)')
|
|
+ (f' ({st.downloads_created} episodes created for them)' if st.downloads_created else '')
|
|
+ (f'; {st.downloads_missing} not found (feeds not imported, no audio URL)' if st.downloads_missing else ''),
|
|
f'Created feeds ({len(st.created_titles)}):',
|
|
*(f' - {t}' for t in st.created_titles),
|
|
*([f'Unsubscribed feeds ({len(st.removed_titles)}):'] if st.removed_titles else []),
|
|
*(f' - {t}' for t in st.removed_titles),
|
|
f'Skipped feeds ({len(st.skipped_feeds)}):',
|
|
*(f' - {t} ({u})' for t, u in st.skipped_feeds),
|
|
f'Done in {st.seconds:.1f} s.',
|
|
]
|
|
return lines
|