diora-web/podcasts/views.py

import json
import logging
import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from datetime import timedelta
import xml.etree.ElementTree as ET

import feedparser
import requests
from django.conf import settings
from django.db import IntegrityError, transaction
from django.db.models import (
    Case, Count, Exists, F, FilteredRelation, IntegerField, Max, OuterRef, Q, Sum, Value, When,
)
from django.db.models.functions import Coalesce
from django.http import HttpResponse, JsonResponse
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods

from . import chapters as chapters_mod
from . import feedext
from . import transcripts as transcripts_mod
from .models import (
    EpisodeProgress, EpisodeTranscript, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings, counts_as_played,
)
from .shownotes import extract_timestamps, sanitize_html

feedext.install()

log = logging.getLogger(__name__)

FEED_TIMEOUT = 20      # seconds per feed download
NEW_ON_SUBSCRIBE = 1   # episodes of a new subscription that count as new
FEED_WORKERS = 8       # feeds downloaded at the same time
MAX_DESCRIPTION = 50_000  # characters of shownotes HTML kept per episode
MAX_TRANSCRIPTS = 10


# ---------------------------------------------------------------------------
# Queue positions
# ---------------------------------------------------------------------------

QUEUE_POSITIONS = {key for key, _ in PodcastSettings.POSITIONS}
KEEP_SORTED = {key for key, _ in PodcastSettings.SORT_ORDERS}


def _queue_order(user):
    """Episode ids in the user's queue, first to last."""
    return list(
        PodcastQueue.objects.filter(user=user)
        .order_by('position', 'added_at', 'id')
        .values_list('episode_id', flat=True)
    )


def _write_queue_order(user, order):
    """Renumber the queue 0..n-1 in the given order."""
    rows = {q.episode_id: q for q in PodcastQueue.objects.filter(user=user)}
    changed = []
    for pos, ep_id in enumerate(order):
        row = rows.get(ep_id)
        if row is not None and row.position != pos:
            row.position = pos
            changed.append(row)
    if changed:
        PodcastQueue.objects.bulk_update(changed, ['position'])


def _current_episode_id(user, queued, hint=None):
    """The episode being listened to: the client's word for it, else the
    queued episode most recently listened to and not finished."""
    if hint in queued:
        return hint
    p = (
        EpisodeProgress.objects
        .filter(user=user, episode_id__in=queued, played=False, position_seconds__gt=0)
        .order_by('-updated_at')
        .values_list('episode_id', flat=True)
        .first()
    )
    return p


def _enqueue(user, episode_ids, where, current_hint=None):
    """Put episodes that aren't queued yet into the queue at `where`
    (back, front or after_current), keeping their given order.
    Returns the ids that were added."""
    queued = _queue_order(user)
    present = set(queued)
    new = []
    for ep_id in episode_ids:
        if ep_id not in present and ep_id not in new:
            new.append(ep_id)
    if not new:
        return []
    if where == PodcastSettings.FRONT:
        at = 0
    elif where == PodcastSettings.AFTER_CURRENT:
        current = _current_episode_id(user, queued, current_hint)
        at = queued.index(current) + 1 if current in queued else 0
    else:
        at = len(queued)
    PodcastQueue.objects.bulk_create(
        [PodcastQueue(user=user, episode_id=ep_id, position=len(queued) + i) for i, ep_id in enumerate(new)],
        ignore_conflicts=True,
    )
    _write_queue_order(user, queued[:at] + new + queued[at:])
    _leave_inbox(user, new)
    _keep_sorted(user)
    return new


def _sort_key(rule):
    """Sort key for one queue row ({pub_date, duration, position, feed_title})
    by a PodcastSettings.SORT_ORDERS rule. Ties keep their current order
    (the sort is stable)."""
    def ts(row):
        return row['pub_date'].timestamp() if row['pub_date'] else None

    def remaining(row):
        return max((row['duration'] or 0) - (row['position'] or 0), 0)

    if rule == 'new':  # no date: last
        return lambda r: (ts(r) is None, -(ts(r) or 0))
    if rule == 'old':  # no date: last as well
        return lambda r: (ts(r) is None, ts(r) or 0)
    if rule == 'short':
        return remaining
    if rule == 'long':
        return lambda r: -remaining(r)
    if rule == 'feed':  # by podcast title, then newest first
        return lambda r: ((r['feed_title'] or '').casefold(), ts(r) is None, -(ts(r) or 0))
    return None


def _keep_sorted(user, rule=None):
    """Sort the user's whole queue by their keep_sorted rule (or `rule`), if
    there is one. Returns True if it is kept sorted."""
    if rule is None:
        rule = PodcastSettings.objects.filter(user=user).values_list('keep_sorted', flat=True).first() or ''
    key = _sort_key(rule)
    if key is None:
        return False
    rows = list(
        PodcastQueue.objects.filter(user=user)
        .order_by('position', 'added_at', 'id')
        .values('episode_id', pub_date=F('episode__pub_date'), duration=F('episode__duration_seconds'),
                feed_title=F('episode__feed__title'))
    )
    positions = dict(
        EpisodeProgress.objects.filter(user=user, episode_id__in=[r['episode_id'] for r in rows])
        .values_list('episode_id', 'position_seconds')
    )
    for r in rows:
        r['position'] = positions.get(r['episode_id'], 0)
    rows.sort(key=key)
    _write_queue_order(user, [r['episode_id'] for r in rows])
    return True


def _leave_inbox(user, episode_ids):
    """Queued or started episodes are no longer new (AntennaPod's rule): out
    of the inbox, without counting as played."""
    EpisodeProgress.objects.filter(user=user, episode_id__in=episode_ids, dismissed=False).update(dismissed=True)
    have = set(EpisodeProgress.objects.filter(user=user, episode_id__in=episode_ids).values_list('episode_id', flat=True))
    EpisodeProgress.objects.bulk_create(
        [EpisodeProgress(user=user, episode_id=ep_id, dismissed=True) for ep_id in episode_ids if ep_id not in have],
        ignore_conflicts=True,
    )


# ---------------------------------------------------------------------------
# Duration helper
# ---------------------------------------------------------------------------

def _parse_duration(raw):
    """Return duration in seconds from itunes:duration string or int."""
    if not raw:
        return 0
    if isinstance(raw, int):
        return raw
    raw = str(raw).strip()
    parts = raw.split(':')
    try:
        if len(parts) == 3:
            return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
        if len(parts) == 2:
            return int(parts[0]) * 60 + int(parts[1])
        return int(raw)
    except (ValueError, TypeError):
        return 0


# ---------------------------------------------------------------------------
# Shownotes, chapters and transcripts of a feed entry
# ---------------------------------------------------------------------------

def _entry_description(entry):
    """Shownotes HTML: content:encoded when the feed has it (that's where the
    full notes usually are), else the description/summary."""
    for c in entry.get('content') or []:
        value = (c.get('value') or '').strip()
        if value:
            return value[:MAX_DESCRIPTION]
    return (entry.get('summary') or entry.get('description') or '')[:MAX_DESCRIPTION]


def _entry_chapters_url(entry):
    url = ((entry.get('podcast_chapters') or {}).get('url') or '').strip()
    return url[:1000] if url.lower().startswith(('http://', 'https://')) else ''


def _entry_transcripts(entry):
    raw = entry.get('podcast_transcripts')
    if raw is None and entry.get('podcast_transcript'):
        raw = [entry['podcast_transcript']]  # feedparser without feedext
    transcripts = []
    for t in raw or []:
        url = (t.get('url') or '').strip()
        if not url.lower().startswith(('http://', 'https://')) or len(url) > 1000:
            continue
        transcripts.append({
            'url': url,
            'type': (t.get('type') or '')[:100],
            'language': (t.get('language') or '')[:35],
        })
        if len(transcripts) >= MAX_TRANSCRIPTS:
            break
    return transcripts


def _apply_extras(episode, description, chapters_url, psc_chapters, transcripts):
    """Copy what the feed says now onto `episode`; True if anything changed.

    Inline (Podlove) chapters win. Without them, fetched chapters (from
    chapters_url, or from the audio file when there's no URL) stay cached for
    as long as the URL stays the same; inline chapters going away means a
    fresh fetch.
    """
    changed = []
    if episode.description != description:
        episode.description = description
        changed.append('description')
    if episode.transcripts != transcripts:
        episode.transcripts = transcripts
        changed.append('transcripts')
    if psc_chapters:
        chapters, fetched_at = psc_chapters, None
    elif chapters_url == episode.chapters_url and episode.chapters_fetched_at:
        chapters, fetched_at = episode.chapters, episode.chapters_fetched_at
    else:
        chapters, fetched_at = [], None
    if episode.chapters_url != chapters_url:
        episode.chapters_url = chapters_url
        changed.append('chapters_url')
    if episode.chapters != chapters:
        episode.chapters = chapters
        changed.append('chapters')
    if episode.chapters_fetched_at != fetched_at:
        episode.chapters_fetched_at = fetched_at
        changed.append('chapters_fetched_at')
    return bool(changed)


EXTRA_FIELDS = ['description', 'transcripts', 'chapters_url', 'chapters', 'chapters_fetched_at']


# ---------------------------------------------------------------------------
# Shared feed refresh helper
# ---------------------------------------------------------------------------

class FeedFetchError(Exception):
    """A feed that could not be fetched or is not a feed; str() is the short
    message stored in PodcastFeed.last_error ("HTTP 404", "timeout", …)."""


@dataclass
class FeedFetch:
    """What _fetch_feed got. status 200: `parsed` is the feedparser result and
    etag/last_modified are the new validators ('' when the server sent none).
    status 304: unchanged since the validators that were sent, parsed is None.
    final_url is where the content came from after all redirects;
    permanent_url is the requested URL after following only the 301/308 hops
    at the start of the redirect chain (== the requested URL without one)."""
    status: int
    parsed: object = None
    etag: str = ''
    last_modified: str = ''
    final_url: str = ''
    permanent_url: str = ''


PERMANENT_REDIRECTS = (301, 308)


def _fetch_feed(url, etag='', last_modified=''):
    """Download and parse a feed (with a timeout, unlike feedparser's own
    fetch). With etag/last_modified it is a conditional GET. Returns a
    FeedFetch; raises FeedFetchError when there is nothing usable."""
    headers = {'User-Agent': 'diora podcast refresh'}
    if etag:
        headers['If-None-Match'] = etag
    if last_modified:
        headers['If-Modified-Since'] = last_modified
    try:
        r = requests.get(url, timeout=FEED_TIMEOUT, headers=headers)
    except requests.Timeout:
        raise FeedFetchError('timeout')
    except requests.TooManyRedirects:
        raise FeedFetchError('too many redirects')
    except requests.ConnectionError:
        raise FeedFetchError('connection failed')
    except requests.RequestException as e:
        raise FeedFetchError(f'request failed ({type(e).__name__})')

    # Each hop's target is the next hop's URL (the last one's is r.url).
    permanent_url = url
    targets = [h.url for h in r.history[1:]] + [r.url]
    for hop, target in zip(r.history, targets):
        if hop.status_code not in PERMANENT_REDIRECTS:
            break
        permanent_url = target

    if r.status_code == 304:
        if not (etag or last_modified):
            raise FeedFetchError('HTTP 304')  # nothing to be unchanged from
        return FeedFetch(304, None, etag, last_modified, r.url, permanent_url)
    if r.status_code >= 400:
        raise FeedFetchError(f'HTTP {r.status_code}')

    parsed = feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
    if parsed.get('bozo') and not parsed.entries and not parsed.feed.get('title'):
        raise FeedFetchError('not a feed')
    new_etag = r.headers.get('ETag', '')
    new_modified = r.headers.get('Last-Modified', '')
    return FeedFetch(
        r.status_code, parsed,
        # A validator too long to store is dropped (a truncated one never matches).
        new_etag if len(new_etag) <= 500 else '',
        new_modified if len(new_modified) <= 100 else '',
        r.url, permanent_url,
    )


def _validators(feeds):
    """If-None-Match / If-Modified-Since for one download that serves all
    `feeds` (several users' subscriptions of the same URL): only when every
    one of them was fetched before and stored the same validators, because a
    304 then skips all of them. Otherwise {} (a full response)."""
    pairs = {(f.http_etag, f.http_last_modified) for f in feeds}
    if len(pairs) != 1 or any(f.last_refreshed_at is None for f in feeds):
        return {}
    etag, last_modified = pairs.pop()
    if not (etag or last_modified):
        return {}
    return {'etag': etag, 'last_modified': last_modified}


def _error_text(e):
    if isinstance(e, FeedFetchError):
        return str(e)[:500]
    return (str(e) or type(e).__name__)[:500]


def _record_error(feed_obj, e):
    feed_obj.last_error = _error_text(e)
    feed_obj.last_error_at = timezone.now()
    feed_obj.error_count += 1
    try:
        feed_obj.save(update_fields=['last_error', 'last_error_at', 'error_count'])
    except Exception:
        log.exception('could not store the refresh error of feed %s', feed_obj.pk)


ERROR_FIELDS = ['last_error', 'last_error_at', 'error_count']


def _clear_error(feed_obj):
    feed_obj.last_error = ''
    feed_obj.last_error_at = None
    feed_obj.error_count = 0


BACKOFF_AFTER = 3        # failures in a row before the hourly job backs off
BACKOFF_MAX_HOURS = 24


def _backoff_hours(error_count):
    """Hours between hourly-job attempts for a feed failing error_count
    times in a row: 2, 4, 8, 16, 24 … from the third failure on."""
    if error_count < BACKOFF_AFTER:
        return 0
    return min(2 ** (error_count - 2), BACKOFF_MAX_HOURS)


def _in_backoff_q(now=None):
    """Q for feeds that failed too often to be tried again yet."""
    now = now or timezone.now()
    n = BACKOFF_AFTER
    while _backoff_hours(n) < BACKOFF_MAX_HOURS:
        n += 1
    # From n failures on the wait is capped; below, one term per count.
    q = Q(error_count__gte=n, last_error_at__gt=now - timedelta(hours=BACKOFF_MAX_HOURS))
    for count in range(BACKOFF_AFTER, n):
        q |= Q(error_count=count, last_error_at__gt=now - timedelta(hours=_backoff_hours(count)))
    return q


def _refresh_feeds(feeds, chunk=32):
    """Refresh many feeds at once: the downloads run in parallel and each
    feed URL is fetched once, even when several users subscribed to it; the
    database writes then happen one by one. Returns (refreshed, failed, new)."""
    feeds = list(feeds)
    refreshed = failed = new = 0
    for start in range(0, len(feeds), chunk):
        part = feeds[start:start + chunk]
        by_url = {}
        for f in part:
            by_url.setdefault(f.rss_url, []).append(f)
        results = {}

        def fetch(url):
            try:
                return url, _fetch_feed(url, **_validators(by_url[url])), None
            except Exception as e:  # network, HTTP or parser trouble: report and go on
                return url, None, e

        urls = sorted(by_url)
        with ThreadPoolExecutor(max_workers=min(FEED_WORKERS, len(urls) or 1)) as pool:
            for url, fetched, err in pool.map(fetch, urls):
                results[url] = (fetched, err)

        for feed in part:
            fetched, err = results[feed.rss_url]
            if err is not None:
                failed += 1
                log.warning('refreshing %s failed: %s', feed.rss_url, err)
                _record_error(feed, err)
                continue
            try:
                new += _refresh_feed(feed, fetched)
                refreshed += 1
            except Exception as e:
                failed += 1
                log.warning('updating %s failed: %s', feed.rss_url, e)
    return refreshed, failed, new


def _refresh_feed(feed_obj, fetched=None):
    """Fetch rss_url (unless `fetched`, a FeedFetch, is given) and upsert
    episodes. Returns count of new episodes. A failure is stored on the feed
    (last_error, error_count) and raised again."""
    if fetched is None:
        try:
            fetched = _fetch_feed(feed_obj.rss_url, **_validators([feed_obj]))
        except Exception as e:
            _record_error(feed_obj, e)
            raise
    try:
        return _apply_feed(feed_obj, fetched)
    except Exception as e:
        _record_error(feed_obj, e)
        raise


def _move_to_permanent_url(feed_obj, new_url):
    """The feed answered with a permanent redirect: subscribe to the new URL,
    unless the user already has that one too (then nothing changes)."""
    if not new_url or new_url == feed_obj.rss_url or len(new_url) > 1000:
        return
    try:
        with transaction.atomic():
            PodcastFeed.objects.filter(pk=feed_obj.pk).update(rss_url=new_url)
    except IntegrityError:  # the user's other subscription has it
        return
    log.info('feed %s moved permanently: %s -> %s', feed_obj.pk, feed_obj.rss_url, new_url)
    feed_obj.rss_url = new_url


def _apply_feed(feed_obj, fetched):
    now = timezone.now()
    if fetched.status == 304:
        # Unchanged since the last full download: nothing to parse or write.
        feed_obj.last_refreshed_at = now
        _clear_error(feed_obj)
        feed_obj.save(update_fields=['last_refreshed_at', *ERROR_FIELDS])
        return 0

    parsed = fetched.parsed
    channel = parsed.feed
    # The first fetch of a new subscription brings the whole back catalogue.
    first_fetch = feed_obj.last_refreshed_at is None

    feed_obj.title = channel.get('title', feed_obj.title or feed_obj.rss_url)[:300]
    feed_obj.description = channel.get('subtitle', channel.get('description', ''))[:2000]
    feed_obj.author = channel.get('author', channel.get('itunes_author', ''))[:300]
    feed_obj.link = channel.get('link', '')[:1000]

    # Artwork
    image = channel.get('image', {})
    feed_obj.artwork_url = (
        channel.get('itunes_image', {}).get('href', '')
        or image.get('href', '')
        or image.get('url', '')
    )[:1000]

    feed_obj.last_refreshed_at = now
    feed_obj.save()

    max_ep = getattr(settings, 'PODCAST_MAX_EPISODES_PER_FEED', 200)
    new_count = 0
    new_episodes_list = []
    entries = parsed.entries[:max_ep]

    # Episodes we already have, in one query, so shownotes/chapters/transcripts
    # that changed in the feed can be brought up to date (only those rows).
    guids = [str(e.get('id') or e.get('guid') or '')[:1000] for e in entries]
    existing = {
        ep.guid: ep
        for ep in feed_obj.episodes.filter(guid__in=[g for g in guids if g]).only('id', 'guid', *EXTRA_FIELDS)
    }
    changed_episodes = {}

    for entry in entries:
        # Find audio enclosure
        audio_url = ''
        for enc in entry.get('enclosures', []):
            mime = enc.get('type', '')
            if mime.startswith('audio/') or enc.get('url', '').endswith(('.mp3', '.m4a', '.ogg', '.opus', '.aac')):
                audio_url = enc.get('url', '')
                break
        if not audio_url:
            continue

        guid = entry.get('id') or entry.get('guid') or audio_url
        guid = str(guid)[:1000]
        title = entry.get('title', 'Untitled')[:500]
        description = _entry_description(entry)
        chapters_url = _entry_chapters_url(entry)
        psc_chapters = chapters_mod.from_psc(entry)
        transcripts = _entry_transcripts(entry)

        known = existing.get(guid)
        if known is not None:
            if _apply_extras(known, description, chapters_url, psc_chapters, transcripts):
                changed_episodes[known.id] = known
            continue

        duration_raw = entry.get('itunes_duration', 0)
        duration_seconds = _parse_duration(duration_raw)

        pub_date = None
        if entry.get('published_parsed'):
            import datetime as dt
            t = entry.published_parsed
            pub_date = dt.datetime(*t[:6], tzinfo=dt.timezone.utc)

        ep_number = None
        try:
            ep_number = int(entry.get('itunes_episode', ''))
        except (ValueError, TypeError):
            pass

        season_number = None
        try:
            season_number = int(entry.get('itunes_season', ''))
        except (ValueError, TypeError):
            pass

        artwork_url = entry.get('itunes_image', {}).get('href', '')[:1000]

        episode_obj, created = PodcastEpisode.objects.get_or_create(
            feed=feed_obj,
            guid=guid,
            defaults={
                'title': title,
                'description': description,
                'audio_url': audio_url[:1000],
                'duration_seconds': duration_seconds,
                'pub_date': pub_date,
                'episode_number': ep_number,
                'season_number': season_number,
                'artwork_url': artwork_url,
                'chapters_url': chapters_url,
                'chapters': psc_chapters,
                'transcripts': transcripts,
            },
        )
        existing[guid] = episode_obj  # the same guid twice in one feed
        if created:
            new_count += 1
            new_episodes_list.append(episode_obj)
        elif _apply_extras(episode_obj, description, chapters_url, psc_chapters, transcripts):
            changed_episodes[episode_obj.id] = episode_obj

    if changed_episodes:
        PodcastEpisode.objects.bulk_update(list(changed_episodes.values()), EXTRA_FIELDS, batch_size=100)

    if first_fetch and len(new_episodes_list) > NEW_ON_SUBSCRIBE:
        # Only the newest episodes count as new; the rest stay in the
        # podcast's episode list, out of the inbox (dismissed, not played).
        newest_first = sorted(new_episodes_list, key=lambda e: e.pub_date or e.discovered_at, reverse=True)
        older = newest_first[NEW_ON_SUBSCRIBE:]
        EpisodeProgress.objects.bulk_create(
            [EpisodeProgress(user=feed_obj.user, episode=e, dismissed=True) for e in older],
            ignore_conflicts=True,
        )
        new_episodes_list = newest_first[:NEW_ON_SUBSCRIBE]

    # The feed's own rules for its new episodes: those its filter turns down,
    # and all of them for 'ignore', go straight out of the inbox (not played);
    # 'queue' puts them in the queue (which also takes them out of the inbox);
    # 'inbox' leaves them there.
    if new_episodes_list:
        unwanted = [e for e in new_episodes_list if not feed_obj.matches_filter(e.title, e.duration_seconds)]
        if feed_obj.new_episodes == PodcastFeed.NEW_IGNORE:
            unwanted = new_episodes_list
        if unwanted:
            gone = {e.id for e in unwanted}
            _leave_inbox(feed_obj.user, list(gone))
            new_episodes_list = [e for e in new_episodes_list if e.id not in gone]

    if feed_obj.new_episodes == PodcastFeed.NEW_QUEUE and new_episodes_list:
        where = PodcastSettings.for_user(feed_obj.user).auto_queue_position
        # Newest first in front of what's there; oldest first when appended,
        # so the end of the queue plays in the order the episodes came out.
        newest_first = sorted(
            new_episodes_list,
            key=lambda e: e.pub_date or e.discovered_at,
            reverse=True,
        )
        ordered = newest_first if where != PodcastSettings.BACK else newest_first[::-1]
        _enqueue(feed_obj.user, [e.id for e in ordered], where)

    # Only now, with every episode written: the validators (a 304 next time
    # skips all of the above) and a clean error state.
    feed_obj.http_etag = fetched.etag
    feed_obj.http_last_modified = fetched.last_modified
    _clear_error(feed_obj)
    feed_obj.save(update_fields=['http_etag', 'http_last_modified', *ERROR_FIELDS])
    _move_to_permanent_url(feed_obj, fetched.permanent_url)
    return new_count


# ---------------------------------------------------------------------------
# Search
# ---------------------------------------------------------------------------

def podcast_search(request):
    q = request.GET.get('q', '').strip()
    if not q:
        return JsonResponse({'results': []})

    try:
        url = f'https://itunes.apple.com/search?term={urllib.parse.quote(q)}&media=podcast&limit=20'
        resp = requests.get(url, timeout=getattr(settings, 'ITUNES_TIMEOUT', 6))
        resp.raise_for_status()
        raw = resp.json().get('results', [])
    except Exception as e:
        return JsonResponse({'error': str(e)}, status=502)

    results = []
    for r in raw:
        results.append({
            'id': r.get('collectionId'),
            'title': r.get('collectionName', ''),
            'author': r.get('artistName', ''),
            'artwork_url': r.get('artworkUrl100', ''),
            'rss_url': r.get('feedUrl', ''),
            'genre': r.get('primaryGenreName', ''),
        })

    return JsonResponse({'results': results})


# ---------------------------------------------------------------------------
# Feed list
# ---------------------------------------------------------------------------

@csrf_exempt
def feed_list(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    return JsonResponse({'feeds': _feeds_data(request.user)})


FEED_SETTING_FIELDS = (
    'playback_speed', 'skip_intro', 'skip_outro', 'volume',
    'new_episodes', 'include_words', 'exclude_words', 'min_duration', 'auto_queue', 'auto_download',
)


def _feed_settings(feed):
    return {'feed_id': feed.id, **{name: getattr(feed, name) for name in FEED_SETTING_FIELDS}}


def _feeds_data(user):
    feeds = list(
        user.podcast_feeds
        .annotate(latest_episode_at=Max('episodes__pub_date'))
        .values('id', 'title', 'artwork_url', 'rss_url', 'last_refreshed_at', 'author', 'added_at',
                'latest_episode_at', 'last_error', 'last_error_at', 'error_count', *FEED_SETTING_FIELDS)
    )

    for f in feeds:
        f['feed_id'] = f['id']  # as in feeds/<id>/settings/
        if f['last_refreshed_at']:
            f['last_refreshed_at'] = f['last_refreshed_at'].isoformat()
        if f['last_error_at']:
            f['last_error_at'] = f['last_error_at'].isoformat()
        if f['added_at']:
            f['added_at'] = f['added_at'].isoformat()
        if f['latest_episode_at']:
            f['latest_episode_at'] = f['latest_episode_at'].isoformat()
    return feeds


# ---------------------------------------------------------------------------
# Add feed
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def add_feed(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    rss_url = body.get('rss_url', '').strip()
    if not rss_url:
        return JsonResponse({'error': 'rss_url required'}, status=400)

    feed, created = PodcastFeed.objects.get_or_create(
        user=request.user,
        rss_url=rss_url,
        defaults={'title': body.get('title', rss_url)[:300], **PodcastSettings.new_feed_defaults(request.user)},
    )

    new_episodes = 0
    if created:
        try:
            new_episodes = _refresh_feed(feed)
        except Exception as e:
            feed.delete()
            return JsonResponse({'error': f'Failed to parse feed: {e}'}, status=400)

    return JsonResponse({
        'ok': True,
        'created': created,
        'feed_id': feed.id,
        'title': feed.title,
        'artwork_url': feed.artwork_url,
        'new_episodes': new_episodes,
    })


# ---------------------------------------------------------------------------
# Remove feed
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def remove_feed(request, pk):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        feed = PodcastFeed.objects.get(pk=pk, user=request.user)
        feed.delete()
        return JsonResponse({'ok': True})
    except PodcastFeed.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)


# ---------------------------------------------------------------------------
# Set auto-queue
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def set_auto_queue(request, pk):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    try:
        feed = PodcastFeed.objects.get(pk=pk, user=request.user)
    except PodcastFeed.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)
    # On: new episodes go to the queue; off: back to the inbox (a feed set
    # to 'ignore' stays so). PodcastFeed.save() sets new_episodes to match.
    feed.auto_queue = bool(body.get('auto_queue', not feed.auto_queue))
    feed.save(update_fields=['auto_queue'])
    return JsonResponse({'ok': True, 'auto_queue': feed.auto_queue})


# ---------------------------------------------------------------------------
# Per-feed settings
# ---------------------------------------------------------------------------

def _number(value):
    return isinstance(value, (int, float)) and not isinstance(value, bool)


def _whole(value, low, high):
    if not _number(value) or value != int(value) or not low <= value <= high:
        raise ValueError
    return int(value)


def _speed(value):
    if value is None:
        return None
    if not _number(value) or not 0.5 <= value <= 3.0:
        raise ValueError
    return float(value)


def _volume(value):
    if not _number(value) or float(value) not in PodcastFeed.VOLUMES:
        raise ValueError
    return float(value)


def _new_episodes(value):
    if value not in {key for key, _ in PodcastFeed.NEW_EPISODE_ACTIONS}:
        raise ValueError
    return value


def _boolean(value):
    if not isinstance(value, bool):
        raise ValueError
    return value


def _word_list(value):
    if value is None:
        return ''
    if not isinstance(value, str) or len(value.strip()) > 500:
        raise ValueError
    return value.strip()


FEED_SETTING_PARSERS = {
    'playback_speed': _speed,
    'skip_intro': lambda v: _whole(v, 0, 600),
    'skip_outro': lambda v: _whole(v, 0, 600),
    'volume': _volume,
    'new_episodes': _new_episodes,
    'include_words': _word_list,
    'exclude_words': _word_list,
    'min_duration': lambda v: _whole(v, 0, 600),
    'auto_download': _boolean,
}


@csrf_exempt
@require_http_methods(['POST'])
def feed_settings(request, pk):
    """Change some of one feed's settings (any of FEED_SETTING_PARSERS);
    returns all of them. Nothing is changed if one value is invalid."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        body = json.loads(request.body or b'{}')
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    if not isinstance(body, dict):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    try:
        feed = PodcastFeed.objects.get(pk=pk, user=request.user)
    except PodcastFeed.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    changes = {}
    for name, parse in FEED_SETTING_PARSERS.items():
        if name in body:
            try:
                changes[name] = parse(body[name])
            except (ValueError, TypeError, OverflowError):
                return JsonResponse({'error': f'invalid {name}', 'field': name}, status=400)
    if changes:
        for name, value in changes.items():
            setattr(feed, name, value)
        feed.save(update_fields=list(changes))
    return JsonResponse(_feed_settings(feed))


# ---------------------------------------------------------------------------
# Feed episodes
# ---------------------------------------------------------------------------

@csrf_exempt
def feed_episodes(request, pk):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        feed = PodcastFeed.objects.get(pk=pk, user=request.user)
    except PodcastFeed.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    episodes = list(
        feed.episodes.values(
            'id', 'title', 'description', 'audio_url', 'duration_seconds',
            'pub_date', 'artwork_url', 'episode_number', 'season_number',
        )
    )

    # Batch-fetch progress
    progress_map = {
        ep['episode_id']: ep
        for ep in EpisodeProgress.objects.filter(
            user=request.user,
            episode__feed=feed,
        ).values('episode_id', 'position_seconds', 'played')
    }

    # Batch-fetch queue membership
    queued_ids = set(
        PodcastQueue.objects.filter(
            user=request.user,
            episode__feed=feed,
        ).values_list('episode_id', flat=True)
    )

    for ep in episodes:
        if ep['pub_date']:
            ep['pub_date'] = ep['pub_date'].isoformat()
        prog = progress_map.get(ep['id'], {})
        ep['position_seconds'] = prog.get('position_seconds', 0)
        ep['played'] = prog.get('played', False)
        ep['in_queue'] = ep['id'] in queued_ids

    return JsonResponse({
        'feed': {
            'id': feed.id,
            'title': feed.title,
            'artwork_url': feed.artwork_url,
            'author': feed.author,
            'auto_queue': feed.auto_queue,
        },
        'episodes': episodes,
    })


# ---------------------------------------------------------------------------
# One episode in detail: shownotes, chapters, transcripts
# ---------------------------------------------------------------------------

def _load_chapters(episode):
    """Fetch the chapters the feed doesn't carry inline, once, and cache them
    on `episode`: the JSON at chapters_url, else (no URL, or nothing there)
    the CHAP frames in the ID3v2 tag at the start of the audio file.

    Problems that may go away (timeouts, server errors) aren't cached, so the
    next request tries again; anything else is remembered as "no chapters".
    """
    url = episode.chapters_url
    chapters = []
    try:
        if url:
            try:
                chapters = chapters_mod.fetch(url)
            except chapters_mod.ChaptersInvalid as e:
                log.info('chapters of episode %s unusable: %s', episode.id, e)
        if not chapters and episode.audio_url:
            try:
                chapters = chapters_mod.fetch_id3(episode.audio_url)
            except chapters_mod.ChaptersInvalid as e:
                log.info('ID3 chapters of episode %s unusable: %s', episode.id, e)
    except chapters_mod.ChaptersUnavailable as e:
        log.info('chapters of episode %s not available: %s', episode.id, e)
        return []
    now = timezone.now()
    # Only if a refresh hasn't changed the URL in the meantime.
    PodcastEpisode.objects.filter(pk=episode.pk, chapters_url=url).update(
        chapters=chapters, chapters_fetched_at=now,
    )
    episode.chapters, episode.chapters_fetched_at = chapters, now
    return chapters


@csrf_exempt
@require_http_methods(['GET'])
def episode_detail(request, pk):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        ep = PodcastEpisode.objects.select_related('feed').get(pk=pk, feed__user=request.user)
    except PodcastEpisode.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    chapters = ep.chapters
    if not chapters and ep.chapters_fetched_at is None:
        chapters = _load_chapters(ep)

    progress = EpisodeProgress.objects.filter(user=request.user, episode=ep).first()
    return JsonResponse({
        'id': ep.id,
        'title': ep.title,
        'description': sanitize_html(ep.description),
        'timestamps': extract_timestamps(ep.description, ep.duration_seconds),
        'chapters': chapters,
        'transcripts': ep.transcripts,
        'has_transcript': _has_transcript(ep),
        'feed': {'id': ep.feed.id, 'title': ep.feed.title, 'artwork_url': ep.feed.artwork_url},
        'audio_url': ep.audio_url,
        'artwork_url': ep.artwork_url,
        'duration_seconds': ep.duration_seconds,
        'pub_date': ep.pub_date.isoformat() if ep.pub_date else None,
        'episode_number': ep.episode_number,
        'season_number': ep.season_number,
        'position_seconds': progress.position_seconds if progress else 0,
        'played': progress.played if progress else False,
        'favorite': progress.favorite if progress else False,
        'last_played_at': progress.updated_at.isoformat() if progress else None,
        'in_queue': PodcastQueue.objects.filter(user=request.user, episode=ep).exists(),
    })


def _transcript_sources(ep):
    return [t['url'] for t in transcripts_mod.candidates(ep.transcripts)]


def _has_transcript(ep):
    """True if the episode offers a transcript we can read, and it hasn't
    turned out to be unusable."""
    sources = _transcript_sources(ep)
    if not sources:
        return False
    cached = EpisodeTranscript.objects.filter(episode=ep).values_list('sources', 'error').first()
    return not (cached and cached[0] == sources and cached[1])


@csrf_exempt
@require_http_methods(['GET'])
def episode_transcript(request, pk):
    """The episode's transcript as {"segments": [{start, end, speaker, text}],
    "source": {url, type, language}}, fetched and parsed on first use, then
    kept. 404 {"error": "no transcript" | "unsupported" | "invalid"} when
    there's none to show; 502 {"error": "unavailable"} when it couldn't be
    loaded right now."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        ep = PodcastEpisode.objects.get(pk=pk, feed__user=request.user)
    except PodcastEpisode.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)
    if not ep.transcripts:
        return JsonResponse({'error': 'no transcript'}, status=404)
    sources = _transcript_sources(ep)
    if not sources:
        return JsonResponse({'error': 'unsupported'}, status=404)

    cached = EpisodeTranscript.objects.filter(episode=ep).first()
    if cached is not None and cached.sources == sources:
        data, error = cached.data, cached.error
    else:
        try:
            data, final = transcripts_mod.load(ep.transcripts)
            error = ''
        except chapters_mod.FetchUnavailable as e:
            log.info('transcript of episode %s not available: %s', ep.id, e)
            return JsonResponse({'error': 'unavailable'}, status=502)
        except transcripts_mod.Unsupported as e:
            log.info('transcript of episode %s unsupported: %s', ep.id, e)
            data, error, final = None, 'unsupported', True
        except chapters_mod.FetchInvalid as e:
            log.info('transcript of episode %s unusable: %s', ep.id, e)
            data, error, final = None, 'invalid', True
        if final:
            EpisodeTranscript.objects.update_or_create(
                episode=ep, defaults={'sources': sources, 'data': data, 'error': error},
            )
    if error:
        return JsonResponse({'error': error}, status=404)
    return JsonResponse(data)


# ---------------------------------------------------------------------------
# Favorites
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def set_favorite(request, pk):
    """Star or unstar an episode: {"favorite": true|false}, no value toggles."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        body = json.loads(request.body or b'{}')
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    if not isinstance(body, dict):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    try:
        episode = PodcastEpisode.objects.get(pk=pk, feed__user=request.user)
    except PodcastEpisode.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    progress, _ = EpisodeProgress.objects.get_or_create(user=request.user, episode=episode)
    wanted = body.get('favorite')
    favorite = (not progress.favorite) if wanted is None else bool(wanted)
    if favorite != progress.favorite:
        # update() leaves updated_at alone: a star isn't listening, and the
        # history and the statistics go by updated_at.
        EpisodeProgress.objects.filter(pk=progress.pk).update(favorite=favorite)
    return JsonResponse({'ok': True, 'favorite': favorite})


# ---------------------------------------------------------------------------
# Episodes of all feeds: filters, history, search
# ---------------------------------------------------------------------------

EPISODE_FILTERS = {
    'all': Q(),
    'unplayed': Q(played=False),
    'in_progress': Q(position_seconds__gt=0, played=False),
    'played': Q(played=True),
    'favorites': Q(favorite=True),
    'history': Q(played=True) | Q(position_seconds__gt=0),
}
EPISODE_LIST_LIMIT = 100
EPISODE_LIST_MAX = 500
EPISODE_LIST_FIELDS = (
    'id', 'title', 'audio_url', 'duration_seconds', 'pub_date', 'artwork_url',
    'feed__id', 'feed__title', 'feed__artwork_url',
    'position_seconds', 'played', 'favorite', 'in_queue', 'last_played_at',
)


def _int_param(request, name, default):
    raw = request.GET.get(name, '').strip()
    return default if raw == '' else int(raw)


def _episode_list_data(user, filter_name='all', feed_id=None, q='', offset=0, limit=EPISODE_LIST_LIMIT):
    """(episodes, total) for the episode list. Two queries, whatever the
    page size: the count and the page. The user's progress row comes in by
    a join, queue membership by a subquery. No shownotes (up to 50 KB each)."""
    qs = (
        PodcastEpisode.objects.filter(feed__user=user)
        .annotate(mine=FilteredRelation('progress', condition=Q(progress__user=user)))
        .annotate(
            position_seconds=Coalesce('mine__position_seconds', Value(0)),
            played=Coalesce('mine__played', Value(False)),
            favorite=Coalesce('mine__favorite', Value(False)),
            last_played_at=F('mine__updated_at'),
            in_queue=Exists(PodcastQueue.objects.filter(user=user, episode=OuterRef('pk'))),
        )
        .filter(EPISODE_FILTERS[filter_name])
    )
    if feed_id is not None:
        qs = qs.filter(feed_id=feed_id)
    if q:
        qs = qs.filter(title__icontains=q)
    if filter_name == 'history':
        qs = qs.order_by(F('last_played_at').desc(nulls_last=True), '-id')
    else:
        qs = qs.order_by(F('pub_date').desc(nulls_last=True), '-id')

    total = qs.count()
    episodes = list(qs.values(*EPISODE_LIST_FIELDS)[offset:offset + limit]) if limit else []
    for ep in episodes:
        ep['pub_date'] = ep['pub_date'].isoformat() if ep['pub_date'] else None
        ep['last_played_at'] = ep['last_played_at'].isoformat() if ep['last_played_at'] else None
        ep['played'] = bool(ep['played'])
        ep['favorite'] = bool(ep['favorite'])
        ep['in_queue'] = bool(ep['in_queue'])
    return episodes, total


@csrf_exempt
@require_http_methods(['GET'])
def episode_list(request):
    """Episodes of all the user's feeds, filtered (all, unplayed, in_progress,
    played, favorites, history), optionally of one feed or matching `q`."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    filter_name = request.GET.get('filter', '').strip() or 'all'
    if filter_name not in EPISODE_FILTERS:
        return JsonResponse({'error': 'unknown filter'}, status=400)
    try:
        offset = max(_int_param(request, 'offset', 0), 0)
        limit = min(max(_int_param(request, 'limit', EPISODE_LIST_LIMIT), 0), EPISODE_LIST_MAX)
        feed_id = _int_param(request, 'feed_id', None)
    except ValueError:
        return JsonResponse({'error': 'offset, limit and feed_id must be numbers'}, status=400)
    episodes, total = _episode_list_data(
        request.user, filter_name, feed_id, request.GET.get('q', '').strip(), offset, limit,
    )
    return JsonResponse({'episodes': episodes, 'offset': offset, 'limit': limit, 'total': total})


# ---------------------------------------------------------------------------
# Listening statistics
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['GET'])
def podcast_stats(request):
    """How much the user listened, in total and per feed; `days` limits it to
    progress rows updated within the last N days."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    rows = EpisodeProgress.objects.filter(user=request.user, episode__feed__user=request.user)
    since = None
    raw_days = request.GET.get('days', '').strip()
    if raw_days:
        try:
            days = int(raw_days)
        except ValueError:
            return JsonResponse({'error': 'days must be a number'}, status=400)
        if days < 0:
            return JsonResponse({'error': 'days must not be negative'}, status=400)
        since = timezone.now() - timedelta(days=days)
        rows = rows.filter(updated_at__gte=since)

    # A played episode counts with its full length (or as far as it got, if
    # the feed gave no length); anything else as far as it got.
    listened = Case(
        When(played=True, episode__duration_seconds__gt=0, then=F('episode__duration_seconds')),
        default=F('position_seconds'),
        output_field=IntegerField(),
    )
    per_feed = (
        rows.order_by()
        .values('episode__feed_id', 'episode__feed__title', 'episode__feed__artwork_url')
        .annotate(
            seconds=Sum(listened),
            played_count=Count('id', filter=Q(played=True)),
            started_count=Count('id', filter=Q(played=False, position_seconds__gt=0)),
        )
    )
    total_seconds = episodes_played = episodes_started = 0
    feeds = []
    for row in per_feed:
        seconds = row['seconds'] or 0
        total_seconds += seconds
        episodes_played += row['played_count']
        episodes_started += row['started_count']
        if seconds > 0:
            feeds.append({
                'feed_id': row['episode__feed_id'],
                'title': row['episode__feed__title'],
                'artwork_url': row['episode__feed__artwork_url'],
                'seconds': seconds,
                'episodes_played': row['played_count'],
            })
    feeds.sort(key=lambda f: (-f['seconds'], f['title'].lower(), f['feed_id']))
    return JsonResponse({
        'total_seconds': total_seconds,
        'episodes_played': episodes_played,
        'episodes_started': episodes_started,
        'since': since.isoformat() if since else None,
        'feeds': feeds,
    })


# ---------------------------------------------------------------------------
# Import OPML
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def import_opml(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    f = request.FILES.get('file')
    if not f:
        return JsonResponse({'error': 'no file uploaded'}, status=400)

    try:
        content = f.read().decode('utf-8', errors='replace')
        root = ET.fromstring(content)
    except Exception as e:
        return JsonResponse({'error': f'invalid OPML: {e}'}, status=400)

    added = 0
    skipped = 0
    new_feed = PodcastSettings.new_feed_defaults(request.user)
    for outline in root.iter('outline'):
        rss_url = outline.get('xmlUrl', '').strip()
        if not rss_url:
            continue
        title = outline.get('title', outline.get('text', rss_url))[:300]
        _, created = PodcastFeed.objects.get_or_create(
            user=request.user,
            rss_url=rss_url,
            defaults={'title': title, **new_feed},
        )
        if created:
            added += 1
        else:
            skipped += 1

    return JsonResponse({'ok': True, 'added': added, 'skipped': skipped})


# ---------------------------------------------------------------------------
# Export OPML
# ---------------------------------------------------------------------------

def _xml_text(s):
    """Drop characters XML 1.0 can't carry at all (escaping is ElementTree's job)."""
    return ''.join(c for c in s if c in '\t\n\r' or (' ' <= c <= '퟿') or ('' <= c <= '<EFBFBD>')
                   or ord(c) >= 0x10000)


@csrf_exempt
@require_http_methods(['GET'])
def export_opml(request):
    """The user's subscriptions as an OPML 2.0 file, for other podcast apps
    (and for import_opml)."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    root = ET.Element('opml', version='2.0')
    head = ET.SubElement(root, 'head')
    ET.SubElement(head, 'title').text = 'diora Podcasts'
    body = ET.SubElement(root, 'body')
    for feed in request.user.podcast_feeds.order_by('title', 'id').only('title', 'rss_url', 'link'):
        title = _xml_text(feed.title or feed.rss_url)
        ET.SubElement(body, 'outline', {
            'type': 'rss',
            'text': title,
            'title': title,
            'xmlUrl': _xml_text(feed.rss_url),
            'htmlUrl': _xml_text(feed.link),
        })
    ET.indent(root)
    xml = '<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(root, encoding='unicode') + '\n'
    response = HttpResponse(xml.encode('utf-8'), content_type='text/x-opml; charset=utf-8')
    response['Content-Disposition'] = 'attachment; filename="diora-podcasts.opml"'
    return response


# ---------------------------------------------------------------------------
# Refresh feed now
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def refresh_feed_now(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    feed_id = body.get('feed_id')
    if not feed_id:
        return JsonResponse({'error': 'feed_id required'}, status=400)

    try:
        feed = PodcastFeed.objects.get(pk=feed_id, user=request.user)
    except PodcastFeed.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    try:
        new_episodes = _refresh_feed(feed)
    except Exception as e:
        return JsonResponse({'error': str(e)}, status=502)

    return JsonResponse({'ok': True, 'new_episodes': new_episodes})


@csrf_exempt
@require_http_methods(['POST'])
def refresh_all_feeds(request):
    """Refresh all of the user's feeds on the server in one go. Feeds the
    hourly job (or an earlier call) refreshed within `max_age` seconds are
    skipped. Clients then fetch the result once (state/, inbox/ …).
    The user asked, so feeds the hourly job backs off from are tried too."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        body = json.loads(request.body or b'{}')
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    try:
        max_age = max(int(body.get('max_age', 300)), 0)
    except (TypeError, ValueError):
        return JsonResponse({'error': 'max_age must be a number'}, status=400)
    cutoff = timezone.now() - timedelta(seconds=max_age)
    feeds = request.user.podcast_feeds.filter(Q(last_refreshed_at__isnull=True) | Q(last_refreshed_at__lt=cutoff))
    refreshed, failed, new = _refresh_feeds(feeds)
    return JsonResponse({'ok': True, 'refreshed': refreshed, 'failed': failed, 'new_episodes': new})


@csrf_exempt
def podcast_state(request):
    """Everything a client shows first, in one response: feeds, queue, the
    first page of the inbox and the settings."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        limit = min(max(int(request.GET.get('inbox_limit', 100)), 0), 1000)
    except ValueError:
        return JsonResponse({'error': 'inbox_limit must be a number'}, status=400)
    s = PodcastSettings.for_user(request.user)
    return JsonResponse({
        'feeds': _feeds_data(request.user),
        'queue': _queue_data(request.user),
        'inbox': _inbox_data(request.user, 0, limit, with_description=False),
        'settings': {'enqueue_position': s.enqueue_position, 'auto_queue_position': s.auto_queue_position,
                     'keep_sorted': s.keep_sorted, 'auto_download_default': s.auto_download_default},
        'server_time': timezone.now().isoformat(),
    })


# ---------------------------------------------------------------------------
# Queue
# ---------------------------------------------------------------------------

@csrf_exempt
def queue_get(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    return JsonResponse({'queue': _queue_data(request.user)})


def _queue_data(user):
    items = list(
        PodcastQueue.objects
        .filter(user=user)
        .order_by('position', 'added_at', 'id')
        .select_related('episode__feed')
        .values(
            'id', 'position',
            'episode__id', 'episode__title', 'episode__audio_url',
            'episode__duration_seconds', 'episode__artwork_url',
            'episode__feed__id', 'episode__feed__title', 'episode__feed__artwork_url',
            'episode__pub_date',
        )
    )

    ep_ids = [item['episode__id'] for item in items]
    progress_map = {
        p['episode_id']: p['position_seconds']
        for p in EpisodeProgress.objects.filter(
            user=user, episode_id__in=ep_ids,
        ).values('episode_id', 'position_seconds')
    }
    for item in items:
        item['position_seconds'] = progress_map.get(item['episode__id'], 0)
        if item['episode__pub_date']:
            item['episode__pub_date'] = item['episode__pub_date'].isoformat()
    return items


@csrf_exempt
@require_http_methods(['POST'])
def queue_add(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    # One episode (episode_id) or several in order (episode_ids). Where they
    # go: `where` if given, else the user's setting. `current_episode_id` is
    # what the client is playing, for after_current.
    ids = body.get('episode_ids')
    if ids is None:
        ids = [body.get('episode_id')]
    if not isinstance(ids, list):
        return JsonResponse({'error': 'episode_ids must be a list'}, status=400)
    where = body.get('where') or PodcastSettings.for_user(request.user).enqueue_position
    if where not in QUEUE_POSITIONS:
        return JsonResponse({'error': 'unknown position'}, status=400)

    found = set(
        PodcastEpisode.objects.filter(pk__in=[i for i in ids if isinstance(i, int)], feed__user=request.user)
        .values_list('id', flat=True)
    )
    if not found:
        return JsonResponse({'error': 'not found'}, status=404)
    added = _enqueue(
        request.user, [i for i in ids if i in found], where,
        current_hint=body.get('current_episode_id'),
    )
    return JsonResponse({'ok': True, 'created': bool(added), 'added': added})


@csrf_exempt
@require_http_methods(['POST'])
def queue_remove(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    episode_id = body.get('episode_id')
    PodcastQueue.objects.filter(user=request.user, episode_id=episode_id).delete()
    return JsonResponse({'ok': True})


@csrf_exempt
@require_http_methods(['POST'])
def queue_reorder(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    order = body.get('order', [])  # list of episode ids
    for pos, ep_id in enumerate(order):
        PodcastQueue.objects.filter(user=request.user, episode_id=ep_id).update(position=pos)
    # A queue that is kept sorted goes straight back into its order.
    _keep_sorted(request.user)

    return JsonResponse({'ok': True})


@csrf_exempt
def podcast_settings(request):
    """GET the user's podcast settings; POST changes some of them."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    s = PodcastSettings.for_user(request.user)
    if request.method == 'POST':
        try:
            body = json.loads(request.body)
        except (json.JSONDecodeError, ValueError):
            return JsonResponse({'error': 'invalid JSON'}, status=400)
        fields = []
        for name in ('enqueue_position', 'auto_queue_position'):
            if name in body:
                if body[name] not in QUEUE_POSITIONS:
                    return JsonResponse({'error': f'unknown {name}'}, status=400)
                setattr(s, name, body[name])
                fields.append(name)
        if 'keep_sorted' in body:
            if body['keep_sorted'] not in KEEP_SORTED:
                return JsonResponse({'error': 'unknown keep_sorted'}, status=400)
            s.keep_sorted = body['keep_sorted']
            fields.append('keep_sorted')
        if 'auto_download_default' in body:
            if not isinstance(body['auto_download_default'], bool):
                return JsonResponse({'error': 'invalid auto_download_default'}, status=400)
            s.auto_download_default = body['auto_download_default']
            fields.append('auto_download_default')
        if fields:
            s.save(update_fields=fields)
        if 'keep_sorted' in fields:
            _keep_sorted(request.user, s.keep_sorted)  # sorted right away
    elif request.method != 'GET':
        return JsonResponse({'error': 'method not allowed'}, status=405)
    return JsonResponse({
        'enqueue_position': s.enqueue_position,
        'auto_queue_position': s.auto_queue_position,
        'keep_sorted': s.keep_sorted,
        'auto_download_default': s.auto_download_default,
        'positions':[{'value': k, 'label': v} for k, v in PodcastSettings.POSITIONS],
    })


# ---------------------------------------------------------------------------
# Progress
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def save_progress(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    episode_id = body.get('episode_id')
    position = int(body.get('position_seconds', 0))

    try:
        episode = PodcastEpisode.objects.get(pk=episode_id, feed__user=request.user)
    except PodcastEpisode.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    progress, _ = EpisodeProgress.objects.get_or_create(
        user=request.user,
        episode=episode,
    )
    progress.position_seconds = position
    # Started: no longer new.
    if position > 0:
        progress.dismissed = True
    # Within the last seconds: played (same rule as the apps).
    if counts_as_played(position, episode.duration_seconds):
        progress.played = True

    progress.save()
    return JsonResponse({'ok': True, 'played': progress.played})


@csrf_exempt
@require_http_methods(['POST'])
def mark_played(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    episode_id = body.get('episode_id')
    played = bool(body.get('played', True))

    try:
        episode = PodcastEpisode.objects.get(pk=episode_id, feed__user=request.user)
    except PodcastEpisode.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    progress, _ = EpisodeProgress.objects.get_or_create(
        user=request.user,
        episode=episode,
    )
    progress.played = played
    progress.save(update_fields=['played', 'updated_at'])
    return JsonResponse({'ok': True, 'played': played})


# ---------------------------------------------------------------------------
# Dismiss episodes (hide from inbox without marking played)
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def dismiss_episodes(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    episode_ids = body.get('episode_ids', [])
    dismissed = bool(body.get('dismissed', True))

    if not isinstance(episode_ids, list) or not episode_ids:
        return JsonResponse({'error': 'episode_ids required'}, status=400)

    # Only allow dismissing episodes that belong to this user
    valid_ids = list(
        PodcastEpisode.objects.filter(
            id__in=episode_ids, feed__user=request.user
        ).values_list('id', flat=True)
    )

    for ep_id in valid_ids:
        progress, _ = EpisodeProgress.objects.get_or_create(
            user=request.user,
            episode_id=ep_id,
        )
        progress.dismissed = dismissed
        progress.save(update_fields=['dismissed', 'updated_at'])

    return JsonResponse({'ok': True, 'count': len(valid_ids)})


# ---------------------------------------------------------------------------
# Inbox
# ---------------------------------------------------------------------------

@csrf_exempt
def inbox(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    limit = min(int(request.GET.get('limit', getattr(settings, 'PODCAST_INBOX_PAGE_SIZE', 200))), 1000)
    offset = max(int(request.GET.get('offset', 0)), 0)
    return JsonResponse({'episodes': _inbox_data(request.user, offset, limit), 'offset': offset, 'limit': limit})


def _inbox_data(user, offset, limit, with_description=True):
    """Inbox episodes. Shownotes can be large (up to 50 KB each); the apps'
    state/ leaves them out and loads one episode's at a time."""
    fields = ['id', 'title', 'audio_url', 'duration_seconds', 'pub_date', 'artwork_url',
              'feed__id', 'feed__title', 'feed__artwork_url']
    if with_description:
        fields.insert(2, 'description')
    hidden_ids = set(
        EpisodeProgress.objects.filter(
            user=user,
        ).filter(
            Q(played=True) | Q(dismissed=True)
        ).values_list('episode_id', flat=True)
    )

    episodes = list(
        PodcastEpisode.objects.filter(feed__user=user)
        .exclude(id__in=hidden_ids)
        .select_related('feed')
        .order_by('-pub_date')
        .values(*fields)[offset:offset + limit]
    )

    progress_map = {
        ep['episode_id']: ep
        for ep in EpisodeProgress.objects.filter(
            user=user,
            episode_id__in=[e['id'] for e in episodes],
        ).values('episode_id', 'position_seconds')
    }
    queued_ids = set(
        PodcastQueue.objects.filter(
            user=user,
            episode_id__in=[e['id'] for e in episodes],
        ).values_list('episode_id', flat=True)
    )
    for ep in episodes:
        if ep['pub_date']:
            ep['pub_date'] = ep['pub_date'].isoformat()
        prog = progress_map.get(ep['id'], {})
        ep['position_seconds'] = prog.get('position_seconds', 0)
        ep['in_queue'] = ep['id'] in queued_ids
    return episodes