mrwnslz
5a2841e7a6
- Neuer Befehl `manage.py import_antennapod <backup.db> --user <name> [--dry-run] [--global-auto-download on|off] [--backup-date ...]`: übernimmt aus einer AntennaPod-Datenbanksicherung die Abos (Abgleich über normalisierte URL, fehlende werden angelegt, lokale übersprungen), die Podcast-Einstellungen (Tempo, Intro/Outro, Lautstärke, Filter mit Phrasen in Anführungszeichen, Mindestlänge, neue Folgen, „nicht aktualisieren“ -> ignorieren, automatischer Download), den Zustand jeder Folge mit Zustand (gehört, neu, Position, Favorit, Zeitpunkt), die Warteschlange (AntennaPods Reihenfolge zuerst) und nimmt bekannte, dort nicht neue Folgen aus „Neu“. Neuere Änderungen in diora bleiben, Favoriten werden vereinigt. Alles in einer Transaktion; ein zweiter Lauf ändert nichts mehr. - PodcastFeed.auto_download (Einstellung pro Podcast, in feeds/, state/ und feeds/<id>/settings/) und PodcastSettings.auto_download_default (in settings/ und state/), mit dem neue Abos (Hinzufügen, OPML, gpodder, Import) starten. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
679 lines
29 KiB
Python
679 lines
29 KiB
Python
"""Import an AntennaPod database backup into one user's podcasts.
|
|
|
|
AntennaPod's "Datenbank exportieren" writes its whole SQLite database. From
|
|
it come the subscriptions with their settings, the state of every episode
|
|
that has one (played, new, started, queued, favourite) and the queue. See
|
|
the import_antennapod command for how to run it.
|
|
|
|
The backup is only read. Episodes without any state are not imported: the
|
|
feed's current episodes come with its next refresh anyway. Those that diora
|
|
already has, though, are taken out of the inbox (AntennaPod: not new).
|
|
"""
|
|
|
|
import re
|
|
import sqlite3
|
|
import time
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone as dt_timezone
|
|
from itertools import groupby
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
from django.db import transaction
|
|
from django.db.models import Q
|
|
|
|
from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings
|
|
from .views import MAX_DESCRIPTION, _keep_sorted, _leave_inbox, _queue_order, _write_queue_order
|
|
|
|
CHUNK = 500 # ids per IN (...) query and rows per bulk write
|
|
|
|
# Feeds.feed_volume_adaption -> diora volume factor (0 = off = 1.0)
|
|
VOLUME_CODES = {1: 0.5, 2: 0.2, 3: 1.5, 4: 2.0, 5: 2.5}
|
|
# Feeds.new_episodes_action; 0 = the global setting (not in the backup)
|
|
NEW_EPISODE_ACTIONS = {1: PodcastFeed.NEW_INBOX, 2: PodcastFeed.NEW_IGNORE, 3: PodcastFeed.NEW_QUEUE}
|
|
# Feeds.auto_download; 1 = the global setting (not in the backup)
|
|
AUTO_DOWNLOAD_OFF, AUTO_DOWNLOAD_GLOBAL, AUTO_DOWNLOAD_ON = 0, 1, 2
|
|
# FeedItems.read
|
|
NEW, UNPLAYED, PLAYED = -1, 0, 1
|
|
SUBSCRIBED = 0 # Feeds.state
|
|
|
|
SETTING_FIELDS = ('playback_speed', 'skip_intro', 'skip_outro', 'volume', 'include_words',
|
|
'exclude_words', 'min_duration', 'new_episodes', 'auto_download')
|
|
|
|
# Columns read from the backup, with the value to assume if an older
|
|
# AntennaPod didn't have the column yet.
|
|
FEED_COLUMNS = {
|
|
'id': None, 'title': '', 'custom_title': None, 'download_url': '', 'link': '', 'author': '',
|
|
'image_url': '', 'keep_updated': 1, 'include_filter': '', 'exclude_filter': '',
|
|
'minimal_duration_filter': -1, 'new_episodes_action': 0, 'feed_playback_speed': -1,
|
|
'feed_skip_intro': 0, 'feed_skip_ending': 0, 'feed_volume_adaption': 0, 'auto_download': 1,
|
|
'state': SUBSCRIBED,
|
|
}
|
|
|
|
|
|
class AntennaPodImportError(Exception):
|
|
"""The file isn't an AntennaPod backup (or the user doesn't exist)."""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Small helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def normalize_url(url):
|
|
"""A URL reduced to what identifies it: no scheme (http = https), host in
|
|
lower case without a leading www., no default port, no trailing slash, no
|
|
fragment. None for anything but http(s)."""
|
|
url = (url or '').strip()
|
|
try:
|
|
parts = urlsplit(url)
|
|
port = parts.port
|
|
except ValueError:
|
|
return None
|
|
if parts.scheme.lower() not in ('http', 'https') or not parts.hostname:
|
|
return None
|
|
host = parts.hostname.lower()
|
|
if host.startswith('www.'):
|
|
host = host[4:]
|
|
if port and port not in (80, 443):
|
|
host = f'{host}:{port}'
|
|
path = parts.path.rstrip('/')
|
|
return host + path + (f'?{parts.query}' if parts.query else '')
|
|
|
|
|
|
def _web_url(url, max_length=1000):
|
|
"""url if it is a usable http(s) URL for a URLField, else ''."""
|
|
url = (url or '').strip()
|
|
if normalize_url(url) is None or len(url) > max_length:
|
|
return ''
|
|
return url
|
|
|
|
|
|
# AntennaPod's filter syntax: words separated by spaces, "quoted phrases".
|
|
_TERM = re.compile(r'([^"]\S*|".+?")\s*')
|
|
|
|
|
|
def parse_filter_terms(raw):
|
|
"""AntennaPod filter text -> its terms, e.g. 'a "b c"' -> ['a', 'b c']."""
|
|
terms = []
|
|
for m in _TERM.finditer(raw or ''):
|
|
# diora separates its words by commas, so a comma can't be in one.
|
|
term = ' '.join(m.group(1).replace('"', '').replace(',', ' ').split())
|
|
if term and term.casefold() not in {t.casefold() for t in terms}:
|
|
terms.append(term)
|
|
return terms
|
|
|
|
|
|
def filter_words(raw, max_length=500):
|
|
"""AntennaPod filter text -> diora's comma-separated words ('' if none);
|
|
as many whole terms as fit into the field."""
|
|
out = ''
|
|
for term in parse_filter_terms(raw):
|
|
candidate = f'{out}, {term}' if out else term
|
|
if len(candidate) > max_length:
|
|
break
|
|
out = candidate
|
|
return out
|
|
|
|
|
|
def _ms(value):
|
|
"""A millisecond timestamp -> aware datetime; None for 0/empty/absurd."""
|
|
try:
|
|
value = int(value or 0)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if value <= 0:
|
|
return None
|
|
try:
|
|
return datetime.fromtimestamp(value / 1000, tz=dt_timezone.utc)
|
|
except (OverflowError, OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def _chunks(seq, size=CHUNK):
|
|
seq = list(seq)
|
|
for i in range(0, len(seq), size):
|
|
yield seq[i:i + size]
|
|
|
|
|
|
@contextmanager
|
|
def _given_updated_at():
|
|
"""Let bulk_create keep the EpisodeProgress.updated_at we set (auto_now
|
|
would put now everywhere, and the history and the statistics go by it)."""
|
|
f = EpisodeProgress._meta.get_field('updated_at')
|
|
f.auto_now = False
|
|
try:
|
|
yield
|
|
finally:
|
|
f.auto_now = True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Feed settings
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def feed_settings(ap, global_auto_download=False):
|
|
"""diora settings for one AntennaPod feed row: only those AntennaPod sets
|
|
to something other than its default. Returns (settings, not_kept_updated)."""
|
|
s = {}
|
|
speed = ap['feed_playback_speed']
|
|
if speed is not None and speed > 0: # -1 = the global speed
|
|
s['playback_speed'] = min(max(round(float(speed), 2), 0.5), 3.0)
|
|
for src, dst in (('feed_skip_intro', 'skip_intro'), ('feed_skip_ending', 'skip_outro')):
|
|
seconds = int(ap[src] or 0)
|
|
if seconds > 0:
|
|
s[dst] = min(seconds, 600)
|
|
volume = VOLUME_CODES.get(ap['feed_volume_adaption'] or 0)
|
|
if volume:
|
|
s['volume'] = volume
|
|
for src, dst in (('include_filter', 'include_words'), ('exclude_filter', 'exclude_words')):
|
|
words = filter_words(ap[src])
|
|
if words:
|
|
s[dst] = words
|
|
minutes = int(ap['minimal_duration_filter'] or 0) // 60 # seconds, -1 = none
|
|
if minutes > 0:
|
|
s['min_duration'] = min(minutes, 600)
|
|
action = NEW_EPISODE_ACTIONS.get(ap['new_episodes_action'] or 0)
|
|
# "Don't update this podcast": AntennaPod never shows anything new of it.
|
|
not_kept_updated = not ap['keep_updated'] and action != PodcastFeed.NEW_QUEUE
|
|
if not_kept_updated:
|
|
action = PodcastFeed.NEW_IGNORE
|
|
if action:
|
|
s['new_episodes'] = action
|
|
code = ap['auto_download']
|
|
if code == AUTO_DOWNLOAD_ON:
|
|
s['auto_download'] = True
|
|
elif code == AUTO_DOWNLOAD_GLOBAL:
|
|
s['auto_download'] = bool(global_auto_download)
|
|
elif code == AUTO_DOWNLOAD_OFF:
|
|
s['auto_download'] = False
|
|
return s, not_kept_updated
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The import
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@dataclass
|
|
class ItemState:
|
|
"""What one AntennaPod episode says about its diora episode."""
|
|
played: bool
|
|
dismissed: bool
|
|
position: int # seconds
|
|
favorite: bool
|
|
activity: datetime | None # last played or finished, if known
|
|
when: datetime # what updated_at becomes: activity, else pub date, else the backup date
|
|
|
|
def merge(self, other):
|
|
"""Two AntennaPod episodes (duplicate feeds) for one diora episode:
|
|
the one listened to last, a star from either."""
|
|
a = self if self.when >= other.when else other
|
|
return ItemState(a.played, a.dismissed, a.position, self.favorite or other.favorite, a.activity, a.when)
|
|
|
|
|
|
@dataclass
|
|
class Stats:
|
|
backup_feeds: int = 0
|
|
not_subscribed: int = 0
|
|
matched: int = 0
|
|
created: int = 0
|
|
duplicates: int = 0
|
|
created_titles: list = field(default_factory=list)
|
|
skipped_feeds: list = field(default_factory=list) # (title, url)
|
|
settings: dict = field(default_factory=lambda: {name: 0 for name in SETTING_FIELDS})
|
|
not_kept_updated: int = 0
|
|
auto_download_on: int = 0
|
|
auto_download_explicit: int = 0
|
|
auto_download_global: int = 0
|
|
auto_download_default: bool = False
|
|
auto_download_default_changed: bool = False
|
|
stateful: int = 0
|
|
by_guid: int = 0
|
|
by_url: int = 0
|
|
episodes_created: int = 0
|
|
no_audio: int = 0
|
|
in_skipped_feeds: int = 0
|
|
stateless_matched: int = 0
|
|
progress_new: int = 0
|
|
progress_updated: int = 0
|
|
progress_unchanged: int = 0
|
|
progress_kept: int = 0
|
|
left_inbox: int = 0
|
|
favorites_backup: int = 0
|
|
favorites_before: int = 0
|
|
favorites_after: int = 0
|
|
queue_backup: int = 0
|
|
queue_missing: int = 0
|
|
queue_before: int = 0
|
|
queue_after: int = 0
|
|
queue_appended: int = 0
|
|
keep_sorted: str = ''
|
|
inbox_before: int = 0
|
|
inbox_after: int = 0
|
|
backup_date: datetime | None = None
|
|
seconds: float = 0.0
|
|
|
|
|
|
def _inbox_count(user):
|
|
hidden = EpisodeProgress.objects.filter(user=user).filter(Q(played=True) | Q(dismissed=True))
|
|
return PodcastEpisode.objects.filter(feed__user=user).exclude(id__in=hidden.values('episode_id')).count()
|
|
|
|
|
|
def _open_backup(path):
|
|
uri = Path(path).resolve().as_uri() + '?mode=ro'
|
|
db = None
|
|
try:
|
|
db = sqlite3.connect(uri, uri=True)
|
|
tables = {r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
except sqlite3.DatabaseError as e:
|
|
if db is not None:
|
|
db.close()
|
|
raise AntennaPodImportError(f'{path} is not an SQLite database: {e}')
|
|
missing = {'Feeds', 'FeedItems', 'FeedMedia', 'Queue', 'Favorites'} - tables
|
|
if missing:
|
|
db.close()
|
|
raise AntennaPodImportError(
|
|
f'{path} is not an AntennaPod backup (missing tables: {", ".join(sorted(missing))})')
|
|
return db
|
|
|
|
|
|
def _columns(db, table):
|
|
return {r[1] for r in db.execute(f'PRAGMA table_info({table})')}
|
|
|
|
|
|
class AntennaPodImport:
|
|
def __init__(self, user, path, backup_date, global_auto_download=False):
|
|
self.user = user
|
|
self.path = path
|
|
self.backup_date = backup_date
|
|
self.global_auto_download = global_auto_download
|
|
self.stats = Stats(backup_date=backup_date)
|
|
|
|
def run(self, dry_run=False):
|
|
started = time.monotonic()
|
|
db = _open_backup(self.path)
|
|
try:
|
|
with transaction.atomic():
|
|
self._run(db)
|
|
if dry_run:
|
|
transaction.set_rollback(True)
|
|
finally:
|
|
db.close()
|
|
self.stats.seconds = time.monotonic() - started
|
|
return self.stats
|
|
|
|
def _run(self, db):
|
|
st = self.stats
|
|
user = self.user
|
|
st.queue_before = len(_queue_order(user))
|
|
st.inbox_before = _inbox_count(user)
|
|
st.favorites_before = EpisodeProgress.objects.filter(user=user, favorite=True).count()
|
|
|
|
# AntennaPod downloads new episodes of every podcast by default: so
|
|
# do diora's future subscriptions.
|
|
settings = PodcastSettings.for_user(user)
|
|
if self.global_auto_download and not settings.auto_download_default:
|
|
settings.auto_download_default = True
|
|
settings.save(update_fields=['auto_download_default'])
|
|
st.auto_download_default_changed = True
|
|
st.auto_download_default = settings.auto_download_default
|
|
self.new_feed = PodcastSettings.new_feed_defaults(user)
|
|
|
|
feed_map = self._import_feeds(db) # AntennaPod feed id -> diora feed id
|
|
queue_items = [r[0] for r in db.execute('SELECT feeditem FROM Queue ORDER BY id')]
|
|
favorite_items = {r[0] for r in db.execute('SELECT feeditem FROM Favorites')}
|
|
st.queue_backup = len(queue_items)
|
|
st.favorites_backup = len(favorite_items)
|
|
|
|
states, not_new, item_episode = self._import_episodes(db, feed_map, set(queue_items), favorite_items)
|
|
self._import_progress(states, not_new)
|
|
self._import_queue(queue_items, item_episode)
|
|
|
|
st.queue_after = len(_queue_order(user))
|
|
st.inbox_after = _inbox_count(user)
|
|
st.favorites_after = EpisodeProgress.objects.filter(user=user, favorite=True).count()
|
|
|
|
# -- feeds --------------------------------------------------------------
|
|
|
|
def _import_feeds(self, db):
|
|
st = self.stats
|
|
have = _columns(db, 'Feeds')
|
|
select = ', '.join(c if c in have else f'{default!r} AS {c}' if default is not None else f'NULL AS {c}'
|
|
for c, default in FEED_COLUMNS.items())
|
|
db.row_factory = sqlite3.Row
|
|
try:
|
|
rows = db.execute(f'SELECT {select} FROM Feeds ORDER BY id').fetchall()
|
|
finally:
|
|
db.row_factory = None
|
|
st.backup_feeds = len(rows)
|
|
|
|
mine = {}
|
|
for f in PodcastFeed.objects.filter(user=self.user).order_by('id'):
|
|
mine.setdefault(normalize_url(f.rss_url) or f.rss_url, f)
|
|
|
|
feed_map = {} # AntennaPod id -> PodcastFeed (saved or still to create)
|
|
to_create = {} # normalised URL -> new PodcastFeed
|
|
changed = {} # diora id -> PodcastFeed with changed settings
|
|
for ap in rows:
|
|
if ap['state'] != SUBSCRIBED:
|
|
st.not_subscribed += 1
|
|
continue
|
|
url = (ap['download_url'] or '').strip()
|
|
title = (ap['custom_title'] or ap['title'] or url).strip()
|
|
key = normalize_url(url)
|
|
if key is None or len(url) > 1000:
|
|
st.skipped_feeds.append((title, url))
|
|
continue
|
|
feed = mine.get(key) or to_create.get(key)
|
|
if feed is None:
|
|
feed = PodcastFeed(
|
|
user=self.user, rss_url=url, title=title[:300], author=(ap['author'] or '')[:300],
|
|
link=_web_url(ap['link']), artwork_url=_web_url(ap['image_url']), **self.new_feed,
|
|
)
|
|
to_create[key] = feed
|
|
st.created += 1
|
|
st.created_titles.append(feed.title)
|
|
elif any(f is feed for f in feed_map.values()):
|
|
st.duplicates += 1
|
|
elif feed.pk:
|
|
st.matched += 1
|
|
feed_map[ap['id']] = feed
|
|
|
|
settings, not_kept_updated = feed_settings(ap, self.global_auto_download)
|
|
st.not_kept_updated += not_kept_updated
|
|
st.auto_download_explicit += ap['auto_download'] == AUTO_DOWNLOAD_ON
|
|
st.auto_download_global += ap['auto_download'] == AUTO_DOWNLOAD_GLOBAL
|
|
for name, value in settings.items():
|
|
if getattr(feed, name) != value:
|
|
setattr(feed, name, value)
|
|
st.settings[name] += 1
|
|
if feed.pk:
|
|
changed[feed.pk] = feed
|
|
feed.auto_queue = feed.new_episodes == PodcastFeed.NEW_QUEUE
|
|
|
|
if changed:
|
|
PodcastFeed.objects.bulk_update(list(changed.values()), [*SETTING_FIELDS, 'auto_queue'],
|
|
batch_size=CHUNK)
|
|
if to_create:
|
|
# last_refreshed_at stays empty: the next refresh fetches them,
|
|
# with the first-fetch rule (only the newest episode is new).
|
|
PodcastFeed.objects.bulk_create(list(to_create.values()), batch_size=CHUNK)
|
|
ids = {}
|
|
for urls in _chunks(f.rss_url for f in to_create.values()):
|
|
ids.update(PodcastFeed.objects.filter(user=self.user, rss_url__in=urls).values_list('rss_url', 'id'))
|
|
for f in to_create.values():
|
|
f.pk = ids[f.rss_url]
|
|
st.auto_download_on = sum(1 for f in {id(f): f for f in feed_map.values()}.values() if f.auto_download)
|
|
return {ap_id: f.pk for ap_id, f in feed_map.items()}
|
|
|
|
# -- episodes -----------------------------------------------------------
|
|
|
|
def _import_episodes(self, db, feed_map, queued, favorites):
|
|
"""Match AntennaPod's episodes to diora's, create the missing ones
|
|
that have a state. Returns ({episode id: ItemState} for those with a
|
|
state, {episode ids} of matched ones without (not new), {AntennaPod
|
|
item id: episode id})."""
|
|
st = self.stats
|
|
backup = self.backup_date
|
|
states = {}
|
|
not_new = set()
|
|
item_episode = {}
|
|
to_create = {} # (diora feed id, guid) -> [ItemState, AntennaPod item id, audio url, duration, pub_date]
|
|
|
|
cur = db.execute(
|
|
'SELECT i.feed, i.id, i.item_identifier, i.read, i.pubDate, m.download_url, m.duration, '
|
|
'm.position, m.last_played_time, m.playback_completion_date '
|
|
'FROM FeedItems i LEFT JOIN FeedMedia m ON m.feeditem = i.id ORDER BY i.feed, i.id'
|
|
)
|
|
|
|
def rows():
|
|
while True:
|
|
batch = cur.fetchmany(5000)
|
|
if not batch:
|
|
return
|
|
yield from batch
|
|
|
|
for ap_feed, items in groupby(rows(), key=lambda r: r[0]):
|
|
feed_id = feed_map.get(ap_feed)
|
|
by_guid = by_url = None
|
|
for (_, item_id, ident, read, pub_ms, audio, duration, position, last_played, completed) in items:
|
|
position = max(int(position or 0), 0) // 1000
|
|
activity = max(filter(None, (_ms(last_played), _ms(completed))), default=None)
|
|
in_queue = item_id in queued
|
|
favorite = item_id in favorites
|
|
stateful = read in (NEW, PLAYED) or position > 0 or activity is not None or in_queue or favorite
|
|
if feed_id is None:
|
|
st.in_skipped_feeds += stateful
|
|
continue
|
|
if by_guid is None:
|
|
by_guid, by_url = {}, {}
|
|
for ep_id, guid, url in (PodcastEpisode.objects.filter(feed_id=feed_id)
|
|
.order_by('id').values_list('id', 'guid', 'audio_url')):
|
|
by_guid.setdefault(guid, ep_id)
|
|
by_url.setdefault(normalize_url(url), ep_id)
|
|
ident = (ident or '').strip()
|
|
audio = (audio or '').strip()
|
|
guid = (ident or audio)[:1000]
|
|
ep_id = by_guid.get(guid) if guid else None
|
|
how = 'guid'
|
|
if ep_id is None and audio:
|
|
ep_id = by_url.get(normalize_url(audio))
|
|
how = 'url'
|
|
if not stateful:
|
|
if ep_id is not None:
|
|
not_new.add(ep_id)
|
|
item_episode[item_id] = ep_id
|
|
continue
|
|
|
|
st.stateful += 1
|
|
played = read == PLAYED
|
|
pub_date = _ms(pub_ms)
|
|
state = ItemState(
|
|
played=played,
|
|
# Only AntennaPod's "new" stays in the inbox; queued is never new.
|
|
dismissed=read != NEW or in_queue,
|
|
position=0 if played else position,
|
|
favorite=favorite,
|
|
activity=activity,
|
|
when=activity or (min(pub_date, backup) if pub_date else backup),
|
|
)
|
|
if ep_id is not None:
|
|
st.by_guid += how == 'guid'
|
|
st.by_url += how == 'url'
|
|
states[ep_id] = state.merge(states[ep_id]) if ep_id in states else state
|
|
item_episode[item_id] = ep_id
|
|
continue
|
|
if normalize_url(audio) is None or len(audio) > 1000:
|
|
st.no_audio += 1
|
|
continue
|
|
key = (feed_id, guid)
|
|
if key in to_create:
|
|
pending = to_create[key]
|
|
pending[0] = state.merge(pending[0])
|
|
pending[1].append(item_id)
|
|
else:
|
|
to_create[key] = [state, [item_id], audio, max(int(duration or 0), 0) // 1000, pub_date]
|
|
|
|
if to_create:
|
|
self._create_episodes(db, to_create, states, item_episode)
|
|
not_new -= states.keys()
|
|
st.stateless_matched = len(not_new)
|
|
return states, not_new, item_episode
|
|
|
|
def _create_episodes(self, db, to_create, states, item_episode):
|
|
st = self.stats
|
|
# A few at a time: shownotes can be large.
|
|
for keys in _chunks(to_create, 200):
|
|
first_item = {to_create[key][1][0]: key for key in keys}
|
|
marks = ','.join('?' * len(first_item))
|
|
details = {
|
|
first_item[item_id]: (title, description, image)
|
|
for item_id, title, description, image in db.execute(
|
|
f'SELECT id, title, description, image_url FROM FeedItems WHERE id IN ({marks})',
|
|
list(first_item))
|
|
}
|
|
batch = []
|
|
for key in keys:
|
|
_, _, audio, duration, pub_date = to_create[key]
|
|
title, description, image = details.get(key, ('', '', ''))
|
|
batch.append(PodcastEpisode(
|
|
feed_id=key[0], guid=key[1], title=(title or 'Untitled')[:500],
|
|
description=(description or '')[:MAX_DESCRIPTION], audio_url=audio,
|
|
duration_seconds=duration, pub_date=pub_date, artwork_url=_web_url(image),
|
|
))
|
|
PodcastEpisode.objects.bulk_create(batch)
|
|
st.episodes_created = len(to_create)
|
|
|
|
by_feed = {}
|
|
for feed_id, guid in to_create:
|
|
by_feed.setdefault(feed_id, []).append(guid)
|
|
for feed_id, guids in by_feed.items():
|
|
for chunk in _chunks(guids):
|
|
created = PodcastEpisode.objects.filter(feed_id=feed_id, guid__in=chunk).values_list('guid', 'id')
|
|
for guid, ep_id in created:
|
|
state, item_ids = to_create[(feed_id, guid)][:2]
|
|
states[ep_id] = state
|
|
for item_id in item_ids:
|
|
item_episode[item_id] = ep_id
|
|
|
|
# -- progress -----------------------------------------------------------
|
|
|
|
def _import_progress(self, states, not_new):
|
|
st = self.stats
|
|
user = self.user
|
|
backup = self.backup_date
|
|
existing = {}
|
|
for ids in _chunks([*states, *not_new]):
|
|
for p in EpisodeProgress.objects.filter(user=user, episode_id__in=ids):
|
|
existing[p.episode_id] = p
|
|
|
|
def diora_newer(p, activity):
|
|
# Changed in diora after the backup was made (and after the
|
|
# episode was last played in AntennaPod): diora's state stays.
|
|
return p.updated_at > backup and (activity is None or p.updated_at > activity)
|
|
|
|
new_rows, changed, favorite_only, dismiss_only = [], [], [], []
|
|
for ep_id, s in states.items():
|
|
p = existing.get(ep_id)
|
|
if p is None:
|
|
new_rows.append(EpisodeProgress(
|
|
user=user, episode_id=ep_id, position_seconds=s.position, played=s.played,
|
|
dismissed=s.dismissed, favorite=s.favorite, updated_at=s.when,
|
|
))
|
|
st.progress_new += 1
|
|
continue
|
|
favorite = p.favorite or s.favorite # stars from both
|
|
if diora_newer(p, s.activity):
|
|
st.progress_kept += 1
|
|
if favorite != p.favorite:
|
|
favorite_only.append(p.pk)
|
|
continue
|
|
wanted = (s.position, s.played, s.dismissed, favorite, s.when)
|
|
if (p.position_seconds, p.played, p.dismissed, p.favorite, p.updated_at) == wanted:
|
|
st.progress_unchanged += 1
|
|
continue
|
|
p.position_seconds, p.played, p.dismissed, p.favorite, p.updated_at = wanted
|
|
changed.append(p)
|
|
st.progress_updated += 1
|
|
|
|
# Known to AntennaPod, but not new there: out of the inbox.
|
|
for ep_id in not_new:
|
|
p = existing.get(ep_id)
|
|
if p is None:
|
|
new_rows.append(EpisodeProgress(user=user, episode_id=ep_id, dismissed=True, updated_at=backup))
|
|
st.left_inbox += 1
|
|
elif not p.played and not p.dismissed and not diora_newer(p, None):
|
|
dismiss_only.append(p.pk)
|
|
st.left_inbox += 1
|
|
|
|
with _given_updated_at():
|
|
EpisodeProgress.objects.bulk_create(new_rows, batch_size=CHUNK)
|
|
# bulk_update writes updated_at as given (no auto_now there).
|
|
EpisodeProgress.objects.bulk_update(
|
|
changed, ['position_seconds', 'played', 'dismissed', 'favorite', 'updated_at'], batch_size=CHUNK)
|
|
# A star or leaving the inbox isn't listening: updated_at stays.
|
|
for ids in _chunks(favorite_only):
|
|
EpisodeProgress.objects.filter(pk__in=ids).update(favorite=True)
|
|
for ids in _chunks(dismiss_only):
|
|
EpisodeProgress.objects.filter(pk__in=ids).update(dismissed=True)
|
|
|
|
# -- queue --------------------------------------------------------------
|
|
|
|
def _import_queue(self, queue_items, item_episode):
|
|
"""AntennaPod's queue first, then what else was in diora's."""
|
|
st = self.stats
|
|
user = self.user
|
|
before = _queue_order(user)
|
|
front = []
|
|
for item_id in queue_items:
|
|
ep_id = item_episode.get(item_id)
|
|
if ep_id is None:
|
|
st.queue_missing += 1
|
|
elif ep_id not in front:
|
|
front.append(ep_id)
|
|
if not front:
|
|
return
|
|
in_front = set(front)
|
|
rest = [ep_id for ep_id in before if ep_id not in in_front]
|
|
st.queue_appended = len(rest)
|
|
present = set(before)
|
|
PodcastQueue.objects.bulk_create(
|
|
[PodcastQueue(user=user, episode_id=ep_id, position=len(before) + i)
|
|
for i, ep_id in enumerate(e for e in front if e not in present)],
|
|
batch_size=CHUNK,
|
|
)
|
|
_write_queue_order(user, front + rest)
|
|
_leave_inbox(user, front) # queued = not new
|
|
if _keep_sorted(user):
|
|
st.keep_sorted = PodcastSettings.objects.get(user=user).keep_sorted
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Summary
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SETTING_LABELS = {
|
|
'playback_speed': 'speed', 'skip_intro': 'skip intro', 'skip_outro': 'skip outro', 'volume': 'volume',
|
|
'include_words': 'include words', 'exclude_words': 'exclude words', 'min_duration': 'min duration',
|
|
'new_episodes': 'new episodes', 'auto_download': 'auto-download',
|
|
}
|
|
|
|
|
|
def _on(flag):
|
|
return 'on' if flag else 'off'
|
|
|
|
|
|
def summary_lines(st, username, path, dry_run, global_auto_download):
|
|
when = st.backup_date.strftime('%Y-%m-%d %H:%M UTC') if st.backup_date else '?'
|
|
subscribed = st.backup_feeds - st.not_subscribed
|
|
settings = ', '.join(f'{SETTING_LABELS[k]} {v}' for k, v in st.settings.items())
|
|
lines = [
|
|
f"AntennaPod import for user '{username}' from {Path(path).name} (backup date {when})"
|
|
+ (' -- DRY RUN, nothing was written' if dry_run else ''),
|
|
f'Feeds: {st.backup_feeds} in backup, {subscribed} subscribed ({st.not_subscribed} not subscribed '
|
|
f'ignored); {st.matched} matched, {st.created} created, {len(st.skipped_feeds)} skipped (not http/https), '
|
|
f'{st.duplicates} duplicates',
|
|
f'Settings: changed: {settings}',
|
|
f' {st.not_kept_updated} feeds not kept updated in AntennaPod -> new episodes "ignore"',
|
|
f'Auto-download: on for {st.auto_download_on} feeds ({st.auto_download_explicit} on in AntennaPod, '
|
|
f"{st.auto_download_global} use AntennaPod's global setting -> {_on(global_auto_download)} "
|
|
f'by --global-auto-download); default for new subscriptions: {_on(st.auto_download_default)}'
|
|
+ (' (set by this import)' if st.auto_download_default_changed else ''),
|
|
f'Episodes: {st.stateful} with a state: {st.by_guid} matched by guid, {st.by_url} by URL, '
|
|
f'{st.episodes_created} created, {st.no_audio} skipped (no audio URL), {st.in_skipped_feeds} in skipped feeds',
|
|
f' {st.stateless_matched} without a state matched to diora episodes (not new in AntennaPod)',
|
|
f'Progress: {st.progress_new} new, {st.progress_updated} updated, '
|
|
f'{st.progress_unchanged} unchanged, {st.progress_kept} kept (diora newer); '
|
|
f'{st.left_inbox} taken out of the inbox (not new in AntennaPod)',
|
|
f'Favourites: {st.favorites_backup} in backup; {st.favorites_before} before, {st.favorites_after} after',
|
|
f'Queue: {st.queue_backup} in backup ({st.queue_missing} not found); {st.queue_before} before, '
|
|
f'{st.queue_after} after ({st.queue_appended} diora entries after AntennaPod\'s)'
|
|
+ (f'; kept sorted by "{st.keep_sorted}"' if st.keep_sorted else ''),
|
|
f'Inbox: {st.inbox_before} before, {st.inbox_after} after',
|
|
f'Created feeds ({len(st.created_titles)}):',
|
|
*(f' - {t}' for t in st.created_titles),
|
|
f'Skipped feeds ({len(st.skipped_feeds)}):',
|
|
*(f' - {t} ({u})' for t, u in st.skipped_feeds),
|
|
f'Done in {st.seconds:.1f} s.',
|
|
]
|
|
return lines
|