diora-web/podcasts/antennapod.py

"""Import an AntennaPod database backup into one user's podcasts.

AntennaPod's "Datenbank exportieren" writes its whole SQLite database. From
it come the subscriptions with their settings, the state of every episode
that has one (played, new, started, queued, favourite), the queue and which
episodes were on the phone (the app offers to download those again). See
the import_antennapod command for how to run it.

The backup is only read. Episodes without any state (and not on the phone)
are not imported: the feed's current episodes come with its next refresh
anyway. Those that diora already has, though, are taken out of the inbox
(AntennaPod: not new).
"""

import re
import sqlite3
import time
from contextlib import contextmanager
from dataclasses import dataclass, field
from datetime import datetime, timezone as dt_timezone
from itertools import groupby
from pathlib import Path
from urllib.parse import urlsplit

from django.db import transaction
from django.db.models import Q

from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings
from .views import MAX_DESCRIPTION, _keep_sorted, _leave_inbox, _queue_order, _write_queue_order

CHUNK = 500  # ids per IN (...) query and rows per bulk write

# Feeds.feed_volume_adaption -> diora volume factor (0 = off = 1.0)
VOLUME_CODES = {1: 0.5, 2: 0.2, 3: 1.5, 4: 2.0, 5: 2.5}
# Feeds.new_episodes_action; 0 = the global setting (not in the backup)
NEW_EPISODE_ACTIONS = {1: PodcastFeed.NEW_INBOX, 2: PodcastFeed.NEW_IGNORE, 3: PodcastFeed.NEW_QUEUE}
# Feeds.auto_download; 1 = the global setting (not in the backup)
AUTO_DOWNLOAD_OFF, AUTO_DOWNLOAD_GLOBAL, AUTO_DOWNLOAD_ON = 0, 1, 2
# FeedItems.read
NEW, UNPLAYED, PLAYED = -1, 0, 1
SUBSCRIBED = 0  # Feeds.state
# FeedMedia.filesize above this is garbage (seen: 10^13); taken as unknown.
MAX_FILESIZE = 4_000_000_000

SETTING_FIELDS = ('playback_speed', 'skip_intro', 'skip_outro', 'volume', 'include_words',
                  'exclude_words', 'min_duration', 'new_episodes', 'auto_download')

# Columns read from the backup, with the value to assume if an older
# AntennaPod didn't have the column yet.
FEED_COLUMNS = {
    'id': None, 'title': '', 'custom_title': None, 'download_url': '', 'link': '', 'author': '',
    'image_url': '', 'keep_updated': 1, 'include_filter': '', 'exclude_filter': '',
    'minimal_duration_filter': -1, 'new_episodes_action': 0, 'feed_playback_speed': -1,
    'feed_skip_intro': 0, 'feed_skip_ending': 0, 'feed_volume_adaption': 0, 'auto_download': 1,
    'state': SUBSCRIBED,
}


class AntennaPodImportError(Exception):
    """The file isn't an AntennaPod backup (or the user doesn't exist)."""


# ---------------------------------------------------------------------------
# Small helpers
# ---------------------------------------------------------------------------

def normalize_url(url):
    """A URL reduced to what identifies it: no scheme (http = https), host in
    lower case without a leading www., no default port, no trailing slash, no
    fragment. None for anything but http(s)."""
    url = (url or '').strip()
    try:
        parts = urlsplit(url)
        port = parts.port
    except ValueError:
        return None
    if parts.scheme.lower() not in ('http', 'https') or not parts.hostname:
        return None
    host = parts.hostname.lower()
    if host.startswith('www.'):
        host = host[4:]
    if port and port not in (80, 443):
        host = f'{host}:{port}'
    path = parts.path.rstrip('/')
    return host + path + (f'?{parts.query}' if parts.query else '')


def _web_url(url, max_length=1000):
    """url if it is a usable http(s) URL for a URLField, else ''."""
    url = (url or '').strip()
    if normalize_url(url) is None or len(url) > max_length:
        return ''
    return url


# AntennaPod's filter syntax: words separated by spaces, "quoted phrases".
_TERM = re.compile(r'([^"]\S*|".+?")\s*')


def parse_filter_terms(raw):
    """AntennaPod filter text -> its terms, e.g. 'a "b c"' -> ['a', 'b c']."""
    terms = []
    for m in _TERM.finditer(raw or ''):
        # diora separates its words by commas, so a comma can't be in one.
        term = ' '.join(m.group(1).replace('"', '').replace(',', ' ').split())
        if term and term.casefold() not in {t.casefold() for t in terms}:
            terms.append(term)
    return terms


def filter_words(raw, max_length=500):
    """AntennaPod filter text -> diora's comma-separated words ('' if none);
    as many whole terms as fit into the field."""
    out = ''
    for term in parse_filter_terms(raw):
        candidate = f'{out}, {term}' if out else term
        if len(candidate) > max_length:
            break
        out = candidate
    return out


def _ms(value):
    """A millisecond timestamp -> aware datetime; None for 0/empty/absurd."""
    try:
        value = int(value or 0)
    except (TypeError, ValueError):
        return None
    if value <= 0:
        return None
    try:
        return datetime.fromtimestamp(value / 1000, tz=dt_timezone.utc)
    except (OverflowError, OSError, ValueError):
        return None


def _chunks(seq, size=CHUNK):
    seq = list(seq)
    for i in range(0, len(seq), size):
        yield seq[i:i + size]


@contextmanager
def _given_updated_at():
    """Let bulk_create keep the EpisodeProgress.updated_at we set (auto_now
    would put now everywhere, and the history and the statistics go by it)."""
    f = EpisodeProgress._meta.get_field('updated_at')
    f.auto_now = False
    try:
        yield
    finally:
        f.auto_now = True


# ---------------------------------------------------------------------------
# Feed settings
# ---------------------------------------------------------------------------

def feed_settings(ap, global_auto_download=False):
    """diora settings for one AntennaPod feed row: only those AntennaPod sets
    to something other than its default. Returns (settings, not_kept_updated)."""
    s = {}
    speed = ap['feed_playback_speed']
    if speed is not None and speed > 0:  # -1 = the global speed
        s['playback_speed'] = min(max(round(float(speed), 2), 0.5), 3.0)
    for src, dst in (('feed_skip_intro', 'skip_intro'), ('feed_skip_ending', 'skip_outro')):
        seconds = int(ap[src] or 0)
        if seconds > 0:
            s[dst] = min(seconds, 600)
    volume = VOLUME_CODES.get(ap['feed_volume_adaption'] or 0)
    if volume:
        s['volume'] = volume
    for src, dst in (('include_filter', 'include_words'), ('exclude_filter', 'exclude_words')):
        words = filter_words(ap[src])
        if words:
            s[dst] = words
    minutes = int(ap['minimal_duration_filter'] or 0) // 60  # seconds, -1 = none
    if minutes > 0:
        s['min_duration'] = min(minutes, 600)
    action = NEW_EPISODE_ACTIONS.get(ap['new_episodes_action'] or 0)
    # "Don't update this podcast": AntennaPod never shows anything new of it.
    not_kept_updated = not ap['keep_updated'] and action != PodcastFeed.NEW_QUEUE
    if not_kept_updated:
        action = PodcastFeed.NEW_IGNORE
    if action:
        s['new_episodes'] = action
    code = ap['auto_download']
    if code == AUTO_DOWNLOAD_ON:
        s['auto_download'] = True
    elif code == AUTO_DOWNLOAD_GLOBAL:
        s['auto_download'] = bool(global_auto_download)
    elif code == AUTO_DOWNLOAD_OFF:
        s['auto_download'] = False
    return s, not_kept_updated


# ---------------------------------------------------------------------------
# The import
# ---------------------------------------------------------------------------

@dataclass
class ItemState:
    """What one AntennaPod episode says about its diora episode."""
    played: bool
    dismissed: bool
    position: int          # seconds
    favorite: bool
    activity: datetime | None  # last played or finished, if known
    when: datetime         # what updated_at becomes: activity, else pub date, else the backup date

    def merge(self, other):
        """Two AntennaPod episodes (duplicate feeds) for one diora episode:
        the one listened to last, a star from either."""
        a = self if self.when >= other.when else other
        return ItemState(a.played, a.dismissed, a.position, self.favorite or other.favorite, a.activity, a.when)


@dataclass
class Stats:
    backup_feeds: int = 0
    not_subscribed: int = 0
    matched: int = 0
    created: int = 0
    duplicates: int = 0
    created_titles: list = field(default_factory=list)
    skipped_feeds: list = field(default_factory=list)  # (title, url)
    settings: dict = field(default_factory=lambda: {name: 0 for name in SETTING_FIELDS})
    not_kept_updated: int = 0
    auto_download_on: int = 0
    auto_download_explicit: int = 0
    auto_download_global: int = 0
    auto_download_default: bool = False
    auto_download_default_changed: bool = False
    stateful: int = 0
    by_guid: int = 0
    by_url: int = 0
    episodes_created: int = 0
    no_audio: int = 0
    in_skipped_feeds: int = 0
    stateless_matched: int = 0
    progress_new: int = 0
    progress_updated: int = 0
    progress_unchanged: int = 0
    progress_kept: int = 0
    left_inbox: int = 0
    favorites_backup: int = 0
    favorites_before: int = 0
    favorites_after: int = 0
    queue_backup: int = 0
    queue_missing: int = 0
    queue_before: int = 0
    queue_after: int = 0
    queue_appended: int = 0
    keep_sorted: str = ''
    inbox_before: int = 0
    inbox_after: int = 0
    downloads_backup: int = 0
    downloads: int = 0
    downloads_bytes: int = 0
    downloads_created: int = 0
    downloads_missing: int = 0
    downloads_offered: bool = True
    ignored_left_inbox: int = 0
    queue_removed: int = 0
    removed_titles: list = field(default_factory=list)
    diora_only_left_inbox: int = 0
    backup_date: datetime | None = None
    seconds: float = 0.0


def _inbox_count(user):
    hidden = EpisodeProgress.objects.filter(user=user).filter(Q(played=True) | Q(dismissed=True))
    return PodcastEpisode.objects.filter(feed__user=user).exclude(id__in=hidden.values('episode_id')).count()


def _open_backup(path):
    uri = Path(path).resolve().as_uri() + '?mode=ro'
    db = None
    try:
        db = sqlite3.connect(uri, uri=True)
        tables = {r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
    except sqlite3.DatabaseError as e:
        if db is not None:
            db.close()
        raise AntennaPodImportError(f'{path} is not an SQLite database: {e}')
    missing = {'Feeds', 'FeedItems', 'FeedMedia', 'Queue', 'Favorites'} - tables
    if missing:
        db.close()
        raise AntennaPodImportError(
            f'{path} is not an AntennaPod backup (missing tables: {", ".join(sorted(missing))})')
    return db


def _columns(db, table):
    return {r[1] for r in db.execute(f'PRAGMA table_info({table})')}


class AntennaPodImport:
    def __init__(self, user, path, backup_date, global_auto_download=False, offer_downloads=True,
                 quiet_diora_only=False, replace=False):
        self.user = user
        self.path = path
        self.backup_date = backup_date
        self.global_auto_download = global_auto_download
        # False: the app doesn't offer AntennaPod's downloads (they stay in AntennaPod).
        self.offer_downloads = offer_downloads
        # True: subscriptions only diora has keep their episodes, but the
        # ones out before the backup leave the inbox.
        self.quiet_diora_only = quiet_diora_only
        # A clean slate: AntennaPod's state is the new start. Podcasts only
        # diora has are unsubscribed, the queue is AntennaPod's alone, and
        # AntennaPod wins also over later changes in diora.
        self.replace = replace
        self.stats = Stats(backup_date=backup_date)

    def run(self, dry_run=False):
        started = time.monotonic()
        db = _open_backup(self.path)
        try:
            with transaction.atomic():
                self._run(db)
                if dry_run:
                    transaction.set_rollback(True)
        finally:
            db.close()
        self.stats.seconds = time.monotonic() - started
        return self.stats

    def _run(self, db):
        st = self.stats
        user = self.user
        st.queue_before = len(_queue_order(user))
        st.inbox_before = _inbox_count(user)
        st.favorites_before = EpisodeProgress.objects.filter(user=user, favorite=True).count()

        # AntennaPod downloads new episodes of every podcast by default: so
        # do diora's future subscriptions.
        settings = PodcastSettings.for_user(user)
        if self.global_auto_download and not settings.auto_download_default:
            settings.auto_download_default = True
            settings.save(update_fields=['auto_download_default'])
            st.auto_download_default_changed = True
        st.auto_download_default = settings.auto_download_default
        self.new_feed = PodcastSettings.new_feed_defaults(user)

        feed_map = self._import_feeds(db)                # AntennaPod feed id -> diora feed id
        if self.replace:
            self._remove_diora_only(feed_map)
        queue_items = [r[0] for r in db.execute('SELECT feeditem FROM Queue ORDER BY id')]
        favorite_items = {r[0] for r in db.execute('SELECT feeditem FROM Favorites')}
        st.queue_backup = len(queue_items)
        st.favorites_backup = len(favorite_items)

        states, not_new, item_episode, downloads = self._import_episodes(
            db, feed_map, set(queue_items), favorite_items)
        self._import_progress(states, not_new)
        self._quiet_inbox(feed_map, states)
        self._import_queue(queue_items, item_episode)
        self._import_downloads(settings, downloads, item_episode)

        st.queue_after = len(_queue_order(user))
        st.inbox_after = _inbox_count(user)
        st.favorites_after = EpisodeProgress.objects.filter(user=user, favorite=True).count()

    # -- feeds --------------------------------------------------------------

    def _import_feeds(self, db):
        st = self.stats
        have = _columns(db, 'Feeds')
        select = ', '.join(c if c in have else f'{default!r} AS {c}' if default is not None else f'NULL AS {c}'
                           for c, default in FEED_COLUMNS.items())
        db.row_factory = sqlite3.Row
        try:
            rows = db.execute(f'SELECT {select} FROM Feeds ORDER BY id').fetchall()
        finally:
            db.row_factory = None
        st.backup_feeds = len(rows)

        mine = {}
        for f in PodcastFeed.objects.filter(user=self.user).order_by('id'):
            mine.setdefault(normalize_url(f.rss_url) or f.rss_url, f)

        feed_map = {}       # AntennaPod id -> PodcastFeed (saved or still to create)
        to_create = {}      # normalised URL -> new PodcastFeed
        changed = {}        # diora id -> PodcastFeed with changed settings
        for ap in rows:
            if ap['state'] != SUBSCRIBED:
                st.not_subscribed += 1
                continue
            url = (ap['download_url'] or '').strip()
            title = (ap['custom_title'] or ap['title'] or url).strip()
            key = normalize_url(url)
            if key is None or len(url) > 1000:
                st.skipped_feeds.append((title, url))
                continue
            feed = mine.get(key) or to_create.get(key)
            if feed is None:
                feed = PodcastFeed(
                    user=self.user, rss_url=url, title=title[:300], author=(ap['author'] or '')[:300],
                    link=_web_url(ap['link']), artwork_url=_web_url(ap['image_url']), **self.new_feed,
                )
                to_create[key] = feed
                st.created += 1
                st.created_titles.append(feed.title)
            elif any(f is feed for f in feed_map.values()):
                st.duplicates += 1
            elif feed.pk:
                st.matched += 1
            feed_map[ap['id']] = feed

            settings, not_kept_updated = feed_settings(ap, self.global_auto_download)
            st.not_kept_updated += not_kept_updated
            st.auto_download_explicit += ap['auto_download'] == AUTO_DOWNLOAD_ON
            st.auto_download_global += ap['auto_download'] == AUTO_DOWNLOAD_GLOBAL
            for name, value in settings.items():
                if getattr(feed, name) != value:
                    setattr(feed, name, value)
                    st.settings[name] += 1
                    if feed.pk:
                        changed[feed.pk] = feed
            feed.auto_queue = feed.new_episodes == PodcastFeed.NEW_QUEUE

        if changed:
            PodcastFeed.objects.bulk_update(list(changed.values()), [*SETTING_FIELDS, 'auto_queue'],
                                            batch_size=CHUNK)
        if to_create:
            # last_refreshed_at stays empty: the next refresh fetches them,
            # with the first-fetch rule (only the newest episode is new).
            PodcastFeed.objects.bulk_create(list(to_create.values()), batch_size=CHUNK)
            ids = {}
            for urls in _chunks(f.rss_url for f in to_create.values()):
                ids.update(PodcastFeed.objects.filter(user=self.user, rss_url__in=urls).values_list('rss_url', 'id'))
            for f in to_create.values():
                f.pk = ids[f.rss_url]
        st.auto_download_on = sum(1 for f in {id(f): f for f in feed_map.values()}.values() if f.auto_download)
        return {ap_id: f.pk for ap_id, f in feed_map.items()}

    # -- episodes -----------------------------------------------------------

    def _import_episodes(self, db, feed_map, queued, favorites):
        """Match AntennaPod's episodes to diora's, create the missing ones
        that have a state (or were downloaded). Returns ({episode id: ItemState}
        for those with a state, {episode ids} of matched ones without (not new),
        {AntennaPod item id: episode id}, [(pub date ms, AntennaPod item id,
        size)] of the downloaded ones)."""
        st = self.stats
        backup = self.backup_date
        states = {}
        not_new = set()
        item_episode = {}
        downloads = []
        to_create = {}  # (diora feed id, guid) -> [ItemState, AntennaPod item id, audio url, duration, pub_date]

        media = _columns(db, 'FeedMedia')
        cur = db.execute(
            'SELECT i.feed, i.id, i.item_identifier, i.read, i.pubDate, m.download_url, m.duration, '
            'm.position, m.last_played_time, m.playback_completion_date, '
            + ('m.downloaded, ' if 'downloaded' in media else '0, ')
            + ('m.filesize ' if 'filesize' in media else '0 ')
            + 'FROM FeedItems i LEFT JOIN FeedMedia m ON m.feeditem = i.id ORDER BY i.feed, i.id'
        )

        def rows():
            while True:
                batch = cur.fetchmany(5000)
                if not batch:
                    return
                yield from batch

        for ap_feed, items in groupby(rows(), key=lambda r: r[0]):
            feed_id = feed_map.get(ap_feed)
            by_guid = by_url = None
            for (_, item_id, ident, read, pub_ms, audio, duration, position, last_played, completed,
                 downloaded, size) in items:
                position = max(int(position or 0), 0) // 1000
                activity = max(filter(None, (_ms(last_played), _ms(completed))), default=None)
                in_queue = item_id in queued
                favorite = item_id in favorites
                stateful = read in (NEW, PLAYED) or position > 0 or activity is not None or in_queue or favorite
                # FeedMedia.downloaded: when the file was downloaded (ms; 1 from
                # older versions), 0 = not on the phone.
                downloaded = (downloaded or 0) > 0
                if downloaded:
                    size = int(size or 0)
                    downloads.append((int(pub_ms or 0), item_id, size if 0 < size <= MAX_FILESIZE else 0))
                if feed_id is None:
                    st.in_skipped_feeds += stateful
                    continue
                if by_guid is None:
                    by_guid, by_url = {}, {}
                    for ep_id, guid, url in (PodcastEpisode.objects.filter(feed_id=feed_id)
                                             .order_by('id').values_list('id', 'guid', 'audio_url')):
                        by_guid.setdefault(guid, ep_id)
                        by_url.setdefault(normalize_url(url), ep_id)
                ident = (ident or '').strip()
                audio = (audio or '').strip()
                guid = (ident or audio)[:1000]
                ep_id = by_guid.get(guid) if guid else None
                how = 'guid'
                if ep_id is None and audio:
                    ep_id = by_url.get(normalize_url(audio))
                    how = 'url'
                if not stateful:
                    if ep_id is not None:
                        not_new.add(ep_id)
                        item_episode[item_id] = ep_id
                        continue
                    if not downloaded:
                        continue
                    # On the phone in AntennaPod but not in diora yet (a feed
                    # not fetched so far): created as not new, so the app can
                    # offer to download it again.
                else:
                    st.stateful += 1
                played = read == PLAYED
                pub_date = _ms(pub_ms)
                state = ItemState(
                    played=played,
                    # Only AntennaPod's "new" stays in the inbox; queued is never new.
                    dismissed=read != NEW or in_queue,
                    position=0 if played else position,
                    favorite=favorite,
                    activity=activity,
                    when=activity or (min(pub_date, backup) if pub_date else backup),
                )
                if ep_id is not None:
                    st.by_guid += how == 'guid'
                    st.by_url += how == 'url'
                    states[ep_id] = state.merge(states[ep_id]) if ep_id in states else state
                    item_episode[item_id] = ep_id
                    continue
                if normalize_url(audio) is None or len(audio) > 1000:
                    st.no_audio += stateful
                    continue
                key = (feed_id, guid)
                if key in to_create:
                    pending = to_create[key]
                    pending[0] = state.merge(pending[0])
                    pending[1].append(item_id)
                else:
                    to_create[key] = [state, [item_id], audio, max(int(duration or 0), 0) // 1000, pub_date]
                    st.downloads_created += not stateful

        if to_create:
            self._create_episodes(db, to_create, states, item_episode)
        not_new -= states.keys()
        st.stateless_matched = len(not_new)
        return states, not_new, item_episode, downloads

    def _create_episodes(self, db, to_create, states, item_episode):
        st = self.stats
        # A few at a time: shownotes can be large.
        for keys in _chunks(to_create, 200):
            first_item = {to_create[key][1][0]: key for key in keys}
            marks = ','.join('?' * len(first_item))
            details = {
                first_item[item_id]: (title, description, image)
                for item_id, title, description, image in db.execute(
                    f'SELECT id, title, description, image_url FROM FeedItems WHERE id IN ({marks})',
                    list(first_item))
            }
            batch = []
            for key in keys:
                _, _, audio, duration, pub_date = to_create[key]
                title, description, image = details.get(key, ('', '', ''))
                batch.append(PodcastEpisode(
                    feed_id=key[0], guid=key[1], title=(title or 'Untitled')[:500],
                    description=(description or '')[:MAX_DESCRIPTION], audio_url=audio,
                    duration_seconds=duration, pub_date=pub_date, artwork_url=_web_url(image),
                ))
            PodcastEpisode.objects.bulk_create(batch)
        st.episodes_created = len(to_create) - st.downloads_created

        by_feed = {}
        for feed_id, guid in to_create:
            by_feed.setdefault(feed_id, []).append(guid)
        for feed_id, guids in by_feed.items():
            for chunk in _chunks(guids):
                created = PodcastEpisode.objects.filter(feed_id=feed_id, guid__in=chunk).values_list('guid', 'id')
                for guid, ep_id in created:
                    state, item_ids = to_create[(feed_id, guid)][:2]
                    states[ep_id] = state
                    for item_id in item_ids:
                        item_episode[item_id] = ep_id

    # -- progress -----------------------------------------------------------

    def _import_progress(self, states, not_new):
        st = self.stats
        user = self.user
        backup = self.backup_date
        existing = {}
        for ids in _chunks([*states, *not_new]):
            for p in EpisodeProgress.objects.filter(user=user, episode_id__in=ids):
                existing[p.episode_id] = p

        def diora_newer(p, activity):
            # Changed in diora after the backup was made (and after the
            # episode was last played in AntennaPod): diora's state stays.
            return not self.replace and p.updated_at > backup and (activity is None or p.updated_at > activity)

        new_rows, changed, favorite_only, dismiss_only = [], [], [], []
        for ep_id, s in states.items():
            p = existing.get(ep_id)
            if p is None:
                new_rows.append(EpisodeProgress(
                    user=user, episode_id=ep_id, position_seconds=s.position, played=s.played,
                    dismissed=s.dismissed, favorite=s.favorite, updated_at=s.when,
                ))
                st.progress_new += 1
                continue
            favorite = p.favorite or s.favorite  # stars from both
            if diora_newer(p, s.activity):
                st.progress_kept += 1
                if favorite != p.favorite:
                    favorite_only.append(p.pk)
                continue
            wanted = (s.position, s.played, s.dismissed, favorite, s.when)
            if (p.position_seconds, p.played, p.dismissed, p.favorite, p.updated_at) == wanted:
                st.progress_unchanged += 1
                continue
            p.position_seconds, p.played, p.dismissed, p.favorite, p.updated_at = wanted
            changed.append(p)
            st.progress_updated += 1

        # Known to AntennaPod, but not new there: out of the inbox.
        for ep_id in not_new:
            p = existing.get(ep_id)
            if p is None:
                new_rows.append(EpisodeProgress(user=user, episode_id=ep_id, dismissed=True, updated_at=backup))
                st.left_inbox += 1
            elif not p.played and not p.dismissed and not diora_newer(p, None):
                dismiss_only.append(p.pk)
                st.left_inbox += 1

        with _given_updated_at():
            EpisodeProgress.objects.bulk_create(new_rows, batch_size=CHUNK)
        # bulk_update writes updated_at as given (no auto_now there).
        EpisodeProgress.objects.bulk_update(
            changed, ['position_seconds', 'played', 'dismissed', 'favorite', 'updated_at'], batch_size=CHUNK)
        # A star or leaving the inbox isn't listening: updated_at stays.
        for ids in _chunks(favorite_only):
            EpisodeProgress.objects.filter(pk__in=ids).update(favorite=True)
        for ids in _chunks(dismiss_only):
            EpisodeProgress.objects.filter(pk__in=ids).update(dismissed=True)

    def _quiet_inbox(self, feed_map, states):
        """Episodes AntennaPod never showed as new, though diora has them in
        the inbox: those of podcasts whose new episodes are ignored (AntennaPod
        stopped fetching archived ones, so it never saw them), and with
        quiet_diora_only those of podcasts AntennaPod doesn't have. Only ones
        out before the backup; AntennaPod's own new ones stay."""
        st = self.stats
        user = self.user
        backup = self.backup_date
        imported = set(feed_map.values())
        feeds = set(PodcastFeed.objects.filter(pk__in=imported, new_episodes=PodcastFeed.NEW_IGNORE)
                    .values_list('pk', flat=True))
        diora_only = set()
        if self.quiet_diora_only:
            diora_only = set(PodcastFeed.objects.filter(user=user).exclude(pk__in=imported).values_list('pk', flat=True))
        if not feeds and not diora_only:
            return
        keep = {ep_id for ep_id, s in states.items() if not s.dismissed}
        hidden = EpisodeProgress.objects.filter(user=user).filter(Q(played=True) | Q(dismissed=True))
        rows = (PodcastEpisode.objects.filter(feed_id__in=feeds | diora_only)
                .filter(Q(pub_date__lte=backup) | Q(pub_date__isnull=True))
                .exclude(id__in=hidden.values('episode_id'))
                .values_list('id', 'feed_id'))
        ids = []
        for ep_id, feed_id in rows:
            if ep_id in keep:
                continue
            ids.append(ep_id)
            if feed_id in feeds:
                st.ignored_left_inbox += 1
            else:
                st.diora_only_left_inbox += 1
        # Changed in diora after the backup: diora's state stays.
        newer = set()
        for chunk in ([] if self.replace else _chunks(ids)):
            newer.update(EpisodeProgress.objects.filter(user=user, episode_id__in=chunk, updated_at__gt=backup)
                         .values_list('episode_id', flat=True))
        ids = [i for i in ids if i not in newer]
        for chunk in _chunks(ids):
            EpisodeProgress.objects.filter(user=user, episode_id__in=chunk, dismissed=False).update(dismissed=True)
            have = set(EpisodeProgress.objects.filter(user=user, episode_id__in=chunk)
                       .values_list('episode_id', flat=True))
            with _given_updated_at():
                EpisodeProgress.objects.bulk_create(
                    [EpisodeProgress(user=user, episode_id=i, dismissed=True, updated_at=backup)
                     for i in chunk if i not in have],
                    ignore_conflicts=True,
                )

    def _remove_diora_only(self, feed_map):
        """Unsubscribe the podcasts the backup doesn't have (their episodes,
        states and queue entries go with them)."""
        st = self.stats
        gone = PodcastFeed.objects.filter(user=self.user).exclude(pk__in=set(feed_map.values()))
        st.removed_titles = sorted(gone.values_list('title', flat=True))
        gone.delete()

    # -- queue --------------------------------------------------------------

    def _import_queue(self, queue_items, item_episode):
        """AntennaPod's queue first, then what else was in diora's."""
        st = self.stats
        user = self.user
        before = _queue_order(user)
        front = []
        for item_id in queue_items:
            ep_id = item_episode.get(item_id)
            if ep_id is None:
                st.queue_missing += 1
            elif ep_id not in front:
                front.append(ep_id)
        if not front:
            return
        in_front = set(front)
        rest = [ep_id for ep_id in before if ep_id not in in_front]
        if self.replace and rest:
            for chunk in _chunks(rest):
                PodcastQueue.objects.filter(user=user, episode_id__in=chunk).delete()
            st.queue_removed = len(rest)
            before = [ep_id for ep_id in before if ep_id in in_front]
            rest = []
        st.queue_appended = len(rest)
        present = set(before)
        PodcastQueue.objects.bulk_create(
            [PodcastQueue(user=user, episode_id=ep_id, position=len(before) + i)
             for i, ep_id in enumerate(e for e in front if e not in present)],
            batch_size=CHUNK,
        )
        _write_queue_order(user, front + rest)
        _leave_inbox(user, front)  # queued = not new
        if _keep_sorted(user):
            st.keep_sorted = PodcastSettings.objects.get(user=user).keep_sorted

    # -- downloads ----------------------------------------------------------

    def _import_downloads(self, settings, downloads, item_episode):
        """The episodes AntennaPod had on the phone, newest first: the app
        can't take its files over, so it offers to download them again.
        Replaces what an earlier import left."""
        st = self.stats
        st.downloads_backup = len(downloads)
        entries, seen = [], set()
        for _, item_id, size in sorted(downloads, key=lambda d: (-d[0], -d[1])):
            ep_id = item_episode.get(item_id)
            if ep_id is None:
                st.downloads_missing += 1
                continue
            if ep_id in seen:
                continue
            seen.add(ep_id)
            entries.append({'episode_id': ep_id, 'size': size})
        st.downloads = len(entries)
        st.downloads_bytes = sum(e['size'] for e in entries)
        st.downloads_offered = self.offer_downloads
        settings.import_downloads = entries if self.offer_downloads else []
        settings.save(update_fields=['import_downloads'])


# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------

SETTING_LABELS = {
    'playback_speed': 'speed', 'skip_intro': 'skip intro', 'skip_outro': 'skip outro', 'volume': 'volume',
    'include_words': 'include words', 'exclude_words': 'exclude words', 'min_duration': 'min duration',
    'new_episodes': 'new episodes', 'auto_download': 'auto-download',
}


def _on(flag):
    return 'on' if flag else 'off'


def summary_lines(st, username, path, dry_run, global_auto_download):
    when = st.backup_date.strftime('%Y-%m-%d %H:%M UTC') if st.backup_date else '?'
    subscribed = st.backup_feeds - st.not_subscribed
    settings = ', '.join(f'{SETTING_LABELS[k]} {v}' for k, v in st.settings.items())
    lines = [
        f"AntennaPod import for user '{username}' from {Path(path).name} (backup date {when})"
        + (' -- DRY RUN, nothing was written' if dry_run else ''),
        f'Feeds:      {st.backup_feeds} in backup, {subscribed} subscribed ({st.not_subscribed} not subscribed '
        f'ignored); {st.matched} matched, {st.created} created, {len(st.skipped_feeds)} skipped (not http/https), '
        f'{st.duplicates} duplicates'
        + (f', {len(st.removed_titles)} only in diora unsubscribed (--replace)' if st.removed_titles else ''),
        f'Settings:   changed: {settings}',
        f'            {st.not_kept_updated} feeds not kept updated in AntennaPod -> new episodes "ignore"',
        f'Auto-download: on for {st.auto_download_on} feeds ({st.auto_download_explicit} on in AntennaPod, '
        f"{st.auto_download_global} use AntennaPod's global setting -> {_on(global_auto_download)} "
        f'by --global-auto-download); default for new subscriptions: {_on(st.auto_download_default)}'
        + (' (set by this import)' if st.auto_download_default_changed else ''),
        f'Episodes:   {st.stateful} with a state: {st.by_guid} matched by guid, {st.by_url} by URL, '
        f'{st.episodes_created} created, {st.no_audio} skipped (no audio URL), {st.in_skipped_feeds} in skipped feeds',
        f'            {st.stateless_matched} without a state matched to diora episodes (not new in AntennaPod)',
        f'Progress:   {st.progress_new} new, {st.progress_updated} updated, '
        f'{st.progress_unchanged} unchanged, {st.progress_kept} kept (diora newer); '
        f'{st.left_inbox} taken out of the inbox (not new in AntennaPod)',
        f'Favourites: {st.favorites_backup} in backup; {st.favorites_before} before, {st.favorites_after} after',
        f'Queue:      {st.queue_backup} in backup ({st.queue_missing} not found); {st.queue_before} before, '
        f'{st.queue_after} after ({st.queue_appended} diora entries after AntennaPod\'s'
        + (f', {st.queue_removed} removed (--replace)' if st.queue_removed else '') + ')'
        + (f'; kept sorted by "{st.keep_sorted}"' if st.keep_sorted else ''),
        f'Inbox:      {st.inbox_before} before, {st.inbox_after} after; out of it: {st.ignored_left_inbox} of podcasts '
        f'whose new episodes are ignored (AntennaPod never showed them)'
        + (f', {st.diora_only_left_inbox} of podcasts only diora has' if st.diora_only_left_inbox else ''),
        f'Downloads:  {st.downloads} on the phone in AntennaPod ({st.downloads_bytes / 1_000_000:.0f} MB) '
        + ('-> offered to the app for download' if st.downloads_offered else '-> not offered (--no-downloads)')
        + (f' ({st.downloads_created} episodes created for them)' if st.downloads_created else '')
        + (f'; {st.downloads_missing} not found (feeds not imported, no audio URL)' if st.downloads_missing else ''),
        f'Created feeds ({len(st.created_titles)}):',
        *(f'  - {t}' for t in st.created_titles),
        *([f'Unsubscribed feeds ({len(st.removed_titles)}):'] if st.removed_titles else []),
        *(f'  - {t}' for t in st.removed_titles),
        f'Skipped feeds ({len(st.skipped_feeds)}):',
        *(f'  - {t} ({u})' for t, u in st.skipped_feeds),
        f'Done in {st.seconds:.1f} s.',
    ]
    return lines