diora-web/podcasts/views.py

import json
import logging
import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
import xml.etree.ElementTree as ET

import feedparser
import requests
from django.conf import settings
from django.db.models import (
    Case, Count, Exists, F, FilteredRelation, IntegerField, Max, OuterRef, Q, Sum, Value, When,
)
from django.db.models.functions import Coalesce
from django.http import HttpResponse, JsonResponse
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods

from . import chapters as chapters_mod
from . import feedext
from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings, counts_as_played
from .shownotes import extract_timestamps, sanitize_html

feedext.install()

log = logging.getLogger(__name__)

FEED_TIMEOUT = 20      # seconds per feed download
NEW_ON_SUBSCRIBE = 1   # episodes of a new subscription that count as new
FEED_WORKERS = 8       # feeds downloaded at the same time
MAX_DESCRIPTION = 50_000  # characters of shownotes HTML kept per episode
MAX_TRANSCRIPTS = 10


# ---------------------------------------------------------------------------
# Queue positions
# ---------------------------------------------------------------------------

QUEUE_POSITIONS = {key for key, _ in PodcastSettings.POSITIONS}


def _queue_order(user):
    """Episode ids in the user's queue, first to last."""
    return list(
        PodcastQueue.objects.filter(user=user)
        .order_by('position', 'added_at', 'id')
        .values_list('episode_id', flat=True)
    )


def _write_queue_order(user, order):
    """Renumber the queue 0..n-1 in the given order."""
    rows = {q.episode_id: q for q in PodcastQueue.objects.filter(user=user)}
    changed = []
    for pos, ep_id in enumerate(order):
        row = rows.get(ep_id)
        if row is not None and row.position != pos:
            row.position = pos
            changed.append(row)
    if changed:
        PodcastQueue.objects.bulk_update(changed, ['position'])


def _current_episode_id(user, queued, hint=None):
    """The episode being listened to: the client's word for it, else the
    queued episode most recently listened to and not finished."""
    if hint in queued:
        return hint
    p = (
        EpisodeProgress.objects
        .filter(user=user, episode_id__in=queued, played=False, position_seconds__gt=0)
        .order_by('-updated_at')
        .values_list('episode_id', flat=True)
        .first()
    )
    return p


def _enqueue(user, episode_ids, where, current_hint=None):
    """Put episodes that aren't queued yet into the queue at `where`
    (back, front or after_current), keeping their given order.
    Returns the ids that were added."""
    queued = _queue_order(user)
    present = set(queued)
    new = []
    for ep_id in episode_ids:
        if ep_id not in present and ep_id not in new:
            new.append(ep_id)
    if not new:
        return []
    if where == PodcastSettings.FRONT:
        at = 0
    elif where == PodcastSettings.AFTER_CURRENT:
        current = _current_episode_id(user, queued, current_hint)
        at = queued.index(current) + 1 if current in queued else 0
    else:
        at = len(queued)
    PodcastQueue.objects.bulk_create(
        [PodcastQueue(user=user, episode_id=ep_id, position=len(queued) + i) for i, ep_id in enumerate(new)],
        ignore_conflicts=True,
    )
    _write_queue_order(user, queued[:at] + new + queued[at:])
    _leave_inbox(user, new)
    return new


def _leave_inbox(user, episode_ids):
    """Queued or started episodes are no longer new (AntennaPod's rule): out
    of the inbox, without counting as played."""
    EpisodeProgress.objects.filter(user=user, episode_id__in=episode_ids, dismissed=False).update(dismissed=True)
    have = set(EpisodeProgress.objects.filter(user=user, episode_id__in=episode_ids).values_list('episode_id', flat=True))
    EpisodeProgress.objects.bulk_create(
        [EpisodeProgress(user=user, episode_id=ep_id, dismissed=True) for ep_id in episode_ids if ep_id not in have],
        ignore_conflicts=True,
    )


# ---------------------------------------------------------------------------
# Duration helper
# ---------------------------------------------------------------------------

def _parse_duration(raw):
    """Return duration in seconds from itunes:duration string or int."""
    if not raw:
        return 0
    if isinstance(raw, int):
        return raw
    raw = str(raw).strip()
    parts = raw.split(':')
    try:
        if len(parts) == 3:
            return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
        if len(parts) == 2:
            return int(parts[0]) * 60 + int(parts[1])
        return int(raw)
    except (ValueError, TypeError):
        return 0


# ---------------------------------------------------------------------------
# Shownotes, chapters and transcripts of a feed entry
# ---------------------------------------------------------------------------

def _entry_description(entry):
    """Shownotes HTML: content:encoded when the feed has it (that's where the
    full notes usually are), else the description/summary."""
    for c in entry.get('content') or []:
        value = (c.get('value') or '').strip()
        if value:
            return value[:MAX_DESCRIPTION]
    return (entry.get('summary') or entry.get('description') or '')[:MAX_DESCRIPTION]


def _entry_chapters_url(entry):
    url = ((entry.get('podcast_chapters') or {}).get('url') or '').strip()
    return url[:1000] if url.lower().startswith(('http://', 'https://')) else ''


def _entry_transcripts(entry):
    raw = entry.get('podcast_transcripts')
    if raw is None and entry.get('podcast_transcript'):
        raw = [entry['podcast_transcript']]  # feedparser without feedext
    transcripts = []
    for t in raw or []:
        url = (t.get('url') or '').strip()
        if not url.lower().startswith(('http://', 'https://')) or len(url) > 1000:
            continue
        transcripts.append({
            'url': url,
            'type': (t.get('type') or '')[:100],
            'language': (t.get('language') or '')[:35],
        })
        if len(transcripts) >= MAX_TRANSCRIPTS:
            break
    return transcripts


def _apply_extras(episode, description, chapters_url, psc_chapters, transcripts):
    """Copy what the feed says now onto `episode`; True if anything changed.

    Inline (Podlove) chapters win. Without them, chapters fetched from
    chapters_url stay cached for as long as the URL stays the same.
    """
    changed = []
    if episode.description != description:
        episode.description = description
        changed.append('description')
    if episode.transcripts != transcripts:
        episode.transcripts = transcripts
        changed.append('transcripts')
    if psc_chapters:
        chapters, fetched_at = psc_chapters, episode.chapters_fetched_at
    elif chapters_url and chapters_url == episode.chapters_url and episode.chapters_fetched_at:
        chapters, fetched_at = episode.chapters, episode.chapters_fetched_at
    else:
        chapters, fetched_at = [], None
    if episode.chapters_url != chapters_url:
        episode.chapters_url = chapters_url
        changed.append('chapters_url')
    if episode.chapters != chapters:
        episode.chapters = chapters
        changed.append('chapters')
    if episode.chapters_fetched_at != fetched_at:
        episode.chapters_fetched_at = fetched_at
        changed.append('chapters_fetched_at')
    return bool(changed)


EXTRA_FIELDS = ['description', 'transcripts', 'chapters_url', 'chapters', 'chapters_fetched_at']


# ---------------------------------------------------------------------------
# Shared feed refresh helper
# ---------------------------------------------------------------------------

def _fetch_feed(url):
    """Download and parse a feed (with a timeout, unlike feedparser's own fetch)."""
    r = requests.get(url, timeout=FEED_TIMEOUT, headers={'User-Agent': 'diora podcast refresh'})
    r.raise_for_status()
    return feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})


def _refresh_feeds(feeds, chunk=32):
    """Refresh many feeds at once: the downloads run in parallel and each
    feed URL is fetched once, even when several users subscribed to it; the
    database writes then happen one by one. Returns (refreshed, failed, new)."""
    feeds = list(feeds)
    refreshed = failed = new = 0
    for start in range(0, len(feeds), chunk):
        part = feeds[start:start + chunk]
        urls = sorted({f.rss_url for f in part})
        results = {}

        def fetch(url):
            try:
                return url, _fetch_feed(url), None
            except Exception as e:  # network, HTTP or parser trouble: report and go on
                return url, None, e

        with ThreadPoolExecutor(max_workers=min(FEED_WORKERS, len(urls) or 1)) as pool:
            for url, parsed, err in pool.map(fetch, urls):
                results[url] = (parsed, err)

        for feed in part:
            parsed, err = results[feed.rss_url]
            if err is not None:
                failed += 1
                log.warning('refreshing %s failed: %s', feed.rss_url, err)
                continue
            try:
                new += _refresh_feed(feed, parsed)
                refreshed += 1
            except Exception as e:
                failed += 1
                log.warning('updating %s failed: %s', feed.rss_url, e)
    return refreshed, failed, new


def _refresh_feed(feed_obj, parsed=None):
    """Parse rss_url and upsert episodes. Returns count of new episodes."""
    if parsed is None:
        parsed = _fetch_feed(feed_obj.rss_url)
    channel = parsed.feed
    # The first fetch of a new subscription brings the whole back catalogue.
    first_fetch = feed_obj.last_refreshed_at is None

    feed_obj.title = channel.get('title', feed_obj.title or feed_obj.rss_url)[:300]
    feed_obj.description = channel.get('subtitle', channel.get('description', ''))[:2000]
    feed_obj.author = channel.get('author', channel.get('itunes_author', ''))[:300]
    feed_obj.link = channel.get('link', '')[:1000]

    # Artwork
    image = channel.get('image', {})
    feed_obj.artwork_url = (
        channel.get('itunes_image', {}).get('href', '')
        or image.get('href', '')
        or image.get('url', '')
    )[:1000]

    feed_obj.last_refreshed_at = timezone.now()
    feed_obj.save()

    max_ep = getattr(settings, 'PODCAST_MAX_EPISODES_PER_FEED', 200)
    new_count = 0
    new_episodes_list = []
    entries = parsed.entries[:max_ep]

    # Episodes we already have, in one query, so shownotes/chapters/transcripts
    # that changed in the feed can be brought up to date (only those rows).
    guids = [str(e.get('id') or e.get('guid') or '')[:1000] for e in entries]
    existing = {
        ep.guid: ep
        for ep in feed_obj.episodes.filter(guid__in=[g for g in guids if g]).only('id', 'guid', *EXTRA_FIELDS)
    }
    changed_episodes = {}

    for entry in entries:
        # Find audio enclosure
        audio_url = ''
        for enc in entry.get('enclosures', []):
            mime = enc.get('type', '')
            if mime.startswith('audio/') or enc.get('url', '').endswith(('.mp3', '.m4a', '.ogg', '.opus', '.aac')):
                audio_url = enc.get('url', '')
                break
        if not audio_url:
            continue

        guid = entry.get('id') or entry.get('guid') or audio_url
        guid = str(guid)[:1000]
        title = entry.get('title', 'Untitled')[:500]
        description = _entry_description(entry)
        chapters_url = _entry_chapters_url(entry)
        psc_chapters = chapters_mod.from_psc(entry)
        transcripts = _entry_transcripts(entry)

        known = existing.get(guid)
        if known is not None:
            if _apply_extras(known, description, chapters_url, psc_chapters, transcripts):
                changed_episodes[known.id] = known
            continue

        duration_raw = entry.get('itunes_duration', 0)
        duration_seconds = _parse_duration(duration_raw)

        pub_date = None
        if entry.get('published_parsed'):
            import datetime as dt
            t = entry.published_parsed
            pub_date = dt.datetime(*t[:6], tzinfo=dt.timezone.utc)

        ep_number = None
        try:
            ep_number = int(entry.get('itunes_episode', ''))
        except (ValueError, TypeError):
            pass

        season_number = None
        try:
            season_number = int(entry.get('itunes_season', ''))
        except (ValueError, TypeError):
            pass

        artwork_url = entry.get('itunes_image', {}).get('href', '')[:1000]

        episode_obj, created = PodcastEpisode.objects.get_or_create(
            feed=feed_obj,
            guid=guid,
            defaults={
                'title': title,
                'description': description,
                'audio_url': audio_url[:1000],
                'duration_seconds': duration_seconds,
                'pub_date': pub_date,
                'episode_number': ep_number,
                'season_number': season_number,
                'artwork_url': artwork_url,
                'chapters_url': chapters_url,
                'chapters': psc_chapters,
                'transcripts': transcripts,
            },
        )
        existing[guid] = episode_obj  # the same guid twice in one feed
        if created:
            new_count += 1
            new_episodes_list.append(episode_obj)
        elif _apply_extras(episode_obj, description, chapters_url, psc_chapters, transcripts):
            changed_episodes[episode_obj.id] = episode_obj

    if changed_episodes:
        PodcastEpisode.objects.bulk_update(list(changed_episodes.values()), EXTRA_FIELDS, batch_size=100)

    if first_fetch and len(new_episodes_list) > NEW_ON_SUBSCRIBE:
        # Only the newest episodes count as new; the rest stay in the
        # podcast's episode list, out of the inbox (dismissed, not played).
        newest_first = sorted(new_episodes_list, key=lambda e: e.pub_date or e.discovered_at, reverse=True)
        older = newest_first[NEW_ON_SUBSCRIBE:]
        EpisodeProgress.objects.bulk_create(
            [EpisodeProgress(user=feed_obj.user, episode=e, dismissed=True) for e in older],
            ignore_conflicts=True,
        )
        new_episodes_list = newest_first[:NEW_ON_SUBSCRIBE]

    if feed_obj.auto_queue and new_episodes_list:
        where = PodcastSettings.for_user(feed_obj.user).auto_queue_position
        # Newest first in front of what's there; oldest first when appended,
        # so the end of the queue plays in the order the episodes came out.
        newest_first = sorted(
            new_episodes_list,
            key=lambda e: e.pub_date or e.discovered_at,
            reverse=True,
        )
        ordered = newest_first if where != PodcastSettings.BACK else newest_first[::-1]
        _enqueue(feed_obj.user, [e.id for e in ordered], where)

    return new_count


# ---------------------------------------------------------------------------
# Search
# ---------------------------------------------------------------------------

def podcast_search(request):
    q = request.GET.get('q', '').strip()
    if not q:
        return JsonResponse({'results': []})

    try:
        url = f'https://itunes.apple.com/search?term={urllib.parse.quote(q)}&media=podcast&limit=20'
        resp = requests.get(url, timeout=getattr(settings, 'ITUNES_TIMEOUT', 6))
        resp.raise_for_status()
        raw = resp.json().get('results', [])
    except Exception as e:
        return JsonResponse({'error': str(e)}, status=502)

    results = []
    for r in raw:
        results.append({
            'id': r.get('collectionId'),
            'title': r.get('collectionName', ''),
            'author': r.get('artistName', ''),
            'artwork_url': r.get('artworkUrl100', ''),
            'rss_url': r.get('feedUrl', ''),
            'genre': r.get('primaryGenreName', ''),
        })

    return JsonResponse({'results': results})


# ---------------------------------------------------------------------------
# Feed list
# ---------------------------------------------------------------------------

@csrf_exempt
def feed_list(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    return JsonResponse({'feeds': _feeds_data(request.user)})


def _feeds_data(user):
    feeds = list(
        user.podcast_feeds
        .annotate(latest_episode_at=Max('episodes__pub_date'))
        .values('id', 'title', 'artwork_url', 'rss_url', 'last_refreshed_at', 'author', 'added_at', 'auto_queue', 'latest_episode_at')
    )

    for f in feeds:
        if f['last_refreshed_at']:
            f['last_refreshed_at'] = f['last_refreshed_at'].isoformat()
        if f['added_at']:
            f['added_at'] = f['added_at'].isoformat()
        if f['latest_episode_at']:
            f['latest_episode_at'] = f['latest_episode_at'].isoformat()
    return feeds


# ---------------------------------------------------------------------------
# Add feed
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def add_feed(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    rss_url = body.get('rss_url', '').strip()
    if not rss_url:
        return JsonResponse({'error': 'rss_url required'}, status=400)

    feed, created = PodcastFeed.objects.get_or_create(
        user=request.user,
        rss_url=rss_url,
        defaults={'title': body.get('title', rss_url)[:300]},
    )

    new_episodes = 0
    if created:
        try:
            new_episodes = _refresh_feed(feed)
        except Exception as e:
            feed.delete()
            return JsonResponse({'error': f'Failed to parse feed: {e}'}, status=400)

    return JsonResponse({
        'ok': True,
        'created': created,
        'feed_id': feed.id,
        'title': feed.title,
        'artwork_url': feed.artwork_url,
        'new_episodes': new_episodes,
    })


# ---------------------------------------------------------------------------
# Remove feed
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def remove_feed(request, pk):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        feed = PodcastFeed.objects.get(pk=pk, user=request.user)
        feed.delete()
        return JsonResponse({'ok': True})
    except PodcastFeed.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)


# ---------------------------------------------------------------------------
# Set auto-queue
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def set_auto_queue(request, pk):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    try:
        feed = PodcastFeed.objects.get(pk=pk, user=request.user)
    except PodcastFeed.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)
    feed.auto_queue = bool(body.get('auto_queue', not feed.auto_queue))
    feed.save(update_fields=['auto_queue'])
    return JsonResponse({'ok': True, 'auto_queue': feed.auto_queue})


# ---------------------------------------------------------------------------
# Feed episodes
# ---------------------------------------------------------------------------

@csrf_exempt
def feed_episodes(request, pk):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        feed = PodcastFeed.objects.get(pk=pk, user=request.user)
    except PodcastFeed.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    episodes = list(
        feed.episodes.values(
            'id', 'title', 'description', 'audio_url', 'duration_seconds',
            'pub_date', 'artwork_url', 'episode_number', 'season_number',
        )
    )

    # Batch-fetch progress
    progress_map = {
        ep['episode_id']: ep
        for ep in EpisodeProgress.objects.filter(
            user=request.user,
            episode__feed=feed,
        ).values('episode_id', 'position_seconds', 'played')
    }

    # Batch-fetch queue membership
    queued_ids = set(
        PodcastQueue.objects.filter(
            user=request.user,
            episode__feed=feed,
        ).values_list('episode_id', flat=True)
    )

    for ep in episodes:
        if ep['pub_date']:
            ep['pub_date'] = ep['pub_date'].isoformat()
        prog = progress_map.get(ep['id'], {})
        ep['position_seconds'] = prog.get('position_seconds', 0)
        ep['played'] = prog.get('played', False)
        ep['in_queue'] = ep['id'] in queued_ids

    return JsonResponse({
        'feed': {
            'id': feed.id,
            'title': feed.title,
            'artwork_url': feed.artwork_url,
            'author': feed.author,
            'auto_queue': feed.auto_queue,
        },
        'episodes': episodes,
    })


# ---------------------------------------------------------------------------
# One episode in detail: shownotes, chapters, transcripts
# ---------------------------------------------------------------------------

def _load_chapters(episode):
    """Fetch the JSON chapters of `episode` once and cache them on it.

    Problems that may go away (timeouts, server errors) aren't cached, so the
    next request tries again; anything else is remembered as "no chapters".
    """
    url = episode.chapters_url
    try:
        chapters = chapters_mod.fetch(url)
    except chapters_mod.ChaptersUnavailable as e:
        log.info('chapters of episode %s not available: %s', episode.id, e)
        return []
    except chapters_mod.ChaptersInvalid as e:
        log.info('chapters of episode %s unusable: %s', episode.id, e)
        chapters = []
    now = timezone.now()
    # Only if a refresh hasn't changed the URL in the meantime.
    PodcastEpisode.objects.filter(pk=episode.pk, chapters_url=url).update(
        chapters=chapters, chapters_fetched_at=now,
    )
    episode.chapters, episode.chapters_fetched_at = chapters, now
    return chapters


@csrf_exempt
@require_http_methods(['GET'])
def episode_detail(request, pk):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        ep = PodcastEpisode.objects.select_related('feed').get(pk=pk, feed__user=request.user)
    except PodcastEpisode.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    chapters = ep.chapters
    if not chapters and ep.chapters_url and ep.chapters_fetched_at is None:
        chapters = _load_chapters(ep)

    progress = EpisodeProgress.objects.filter(user=request.user, episode=ep).first()
    return JsonResponse({
        'id': ep.id,
        'title': ep.title,
        'description': sanitize_html(ep.description),
        'timestamps': extract_timestamps(ep.description, ep.duration_seconds),
        'chapters': chapters,
        'transcripts': ep.transcripts,
        'feed': {'id': ep.feed.id, 'title': ep.feed.title, 'artwork_url': ep.feed.artwork_url},
        'audio_url': ep.audio_url,
        'artwork_url': ep.artwork_url,
        'duration_seconds': ep.duration_seconds,
        'pub_date': ep.pub_date.isoformat() if ep.pub_date else None,
        'episode_number': ep.episode_number,
        'season_number': ep.season_number,
        'position_seconds': progress.position_seconds if progress else 0,
        'played': progress.played if progress else False,
        'favorite': progress.favorite if progress else False,
        'last_played_at': progress.updated_at.isoformat() if progress else None,
        'in_queue': PodcastQueue.objects.filter(user=request.user, episode=ep).exists(),
    })


# ---------------------------------------------------------------------------
# Favorites
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def set_favorite(request, pk):
    """Star or unstar an episode: {"favorite": true|false}, no value toggles."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        body = json.loads(request.body or b'{}')
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    if not isinstance(body, dict):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    try:
        episode = PodcastEpisode.objects.get(pk=pk, feed__user=request.user)
    except PodcastEpisode.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    progress, _ = EpisodeProgress.objects.get_or_create(user=request.user, episode=episode)
    wanted = body.get('favorite')
    favorite = (not progress.favorite) if wanted is None else bool(wanted)
    if favorite != progress.favorite:
        # update() leaves updated_at alone: a star isn't listening, and the
        # history and the statistics go by updated_at.
        EpisodeProgress.objects.filter(pk=progress.pk).update(favorite=favorite)
    return JsonResponse({'ok': True, 'favorite': favorite})


# ---------------------------------------------------------------------------
# Episodes of all feeds: filters, history, search
# ---------------------------------------------------------------------------

EPISODE_FILTERS = {
    'all': Q(),
    'unplayed': Q(played=False),
    'in_progress': Q(position_seconds__gt=0, played=False),
    'played': Q(played=True),
    'favorites': Q(favorite=True),
    'history': Q(played=True) | Q(position_seconds__gt=0),
}
EPISODE_LIST_LIMIT = 100
EPISODE_LIST_MAX = 500
EPISODE_LIST_FIELDS = (
    'id', 'title', 'audio_url', 'duration_seconds', 'pub_date', 'artwork_url',
    'feed__id', 'feed__title', 'feed__artwork_url',
    'position_seconds', 'played', 'favorite', 'in_queue', 'last_played_at',
)


def _int_param(request, name, default):
    raw = request.GET.get(name, '').strip()
    return default if raw == '' else int(raw)


def _episode_list_data(user, filter_name='all', feed_id=None, q='', offset=0, limit=EPISODE_LIST_LIMIT):
    """(episodes, total) for the episode list. Two queries, whatever the
    page size: the count and the page. The user's progress row comes in by
    a join, queue membership by a subquery. No shownotes (up to 50 KB each)."""
    qs = (
        PodcastEpisode.objects.filter(feed__user=user)
        .annotate(mine=FilteredRelation('progress', condition=Q(progress__user=user)))
        .annotate(
            position_seconds=Coalesce('mine__position_seconds', Value(0)),
            played=Coalesce('mine__played', Value(False)),
            favorite=Coalesce('mine__favorite', Value(False)),
            last_played_at=F('mine__updated_at'),
            in_queue=Exists(PodcastQueue.objects.filter(user=user, episode=OuterRef('pk'))),
        )
        .filter(EPISODE_FILTERS[filter_name])
    )
    if feed_id is not None:
        qs = qs.filter(feed_id=feed_id)
    if q:
        qs = qs.filter(title__icontains=q)
    if filter_name == 'history':
        qs = qs.order_by(F('last_played_at').desc(nulls_last=True), '-id')
    else:
        qs = qs.order_by(F('pub_date').desc(nulls_last=True), '-id')

    total = qs.count()
    episodes = list(qs.values(*EPISODE_LIST_FIELDS)[offset:offset + limit]) if limit else []
    for ep in episodes:
        ep['pub_date'] = ep['pub_date'].isoformat() if ep['pub_date'] else None
        ep['last_played_at'] = ep['last_played_at'].isoformat() if ep['last_played_at'] else None
        ep['played'] = bool(ep['played'])
        ep['favorite'] = bool(ep['favorite'])
        ep['in_queue'] = bool(ep['in_queue'])
    return episodes, total


@csrf_exempt
@require_http_methods(['GET'])
def episode_list(request):
    """Episodes of all the user's feeds, filtered (all, unplayed, in_progress,
    played, favorites, history), optionally of one feed or matching `q`."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    filter_name = request.GET.get('filter', '').strip() or 'all'
    if filter_name not in EPISODE_FILTERS:
        return JsonResponse({'error': 'unknown filter'}, status=400)
    try:
        offset = max(_int_param(request, 'offset', 0), 0)
        limit = min(max(_int_param(request, 'limit', EPISODE_LIST_LIMIT), 0), EPISODE_LIST_MAX)
        feed_id = _int_param(request, 'feed_id', None)
    except ValueError:
        return JsonResponse({'error': 'offset, limit and feed_id must be numbers'}, status=400)
    episodes, total = _episode_list_data(
        request.user, filter_name, feed_id, request.GET.get('q', '').strip(), offset, limit,
    )
    return JsonResponse({'episodes': episodes, 'offset': offset, 'limit': limit, 'total': total})


# ---------------------------------------------------------------------------
# Listening statistics
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['GET'])
def podcast_stats(request):
    """How much the user listened, in total and per feed; `days` limits it to
    progress rows updated within the last N days."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    rows = EpisodeProgress.objects.filter(user=request.user, episode__feed__user=request.user)
    since = None
    raw_days = request.GET.get('days', '').strip()
    if raw_days:
        try:
            days = int(raw_days)
        except ValueError:
            return JsonResponse({'error': 'days must be a number'}, status=400)
        if days < 0:
            return JsonResponse({'error': 'days must not be negative'}, status=400)
        since = timezone.now() - timedelta(days=days)
        rows = rows.filter(updated_at__gte=since)

    # A played episode counts with its full length (or as far as it got, if
    # the feed gave no length); anything else as far as it got.
    listened = Case(
        When(played=True, episode__duration_seconds__gt=0, then=F('episode__duration_seconds')),
        default=F('position_seconds'),
        output_field=IntegerField(),
    )
    per_feed = (
        rows.order_by()
        .values('episode__feed_id', 'episode__feed__title', 'episode__feed__artwork_url')
        .annotate(
            seconds=Sum(listened),
            played_count=Count('id', filter=Q(played=True)),
            started_count=Count('id', filter=Q(played=False, position_seconds__gt=0)),
        )
    )
    total_seconds = episodes_played = episodes_started = 0
    feeds = []
    for row in per_feed:
        seconds = row['seconds'] or 0
        total_seconds += seconds
        episodes_played += row['played_count']
        episodes_started += row['started_count']
        if seconds > 0:
            feeds.append({
                'feed_id': row['episode__feed_id'],
                'title': row['episode__feed__title'],
                'artwork_url': row['episode__feed__artwork_url'],
                'seconds': seconds,
                'episodes_played': row['played_count'],
            })
    feeds.sort(key=lambda f: (-f['seconds'], f['title'].lower(), f['feed_id']))
    return JsonResponse({
        'total_seconds': total_seconds,
        'episodes_played': episodes_played,
        'episodes_started': episodes_started,
        'since': since.isoformat() if since else None,
        'feeds': feeds,
    })


# ---------------------------------------------------------------------------
# Import OPML
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def import_opml(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    f = request.FILES.get('file')
    if not f:
        return JsonResponse({'error': 'no file uploaded'}, status=400)

    try:
        content = f.read().decode('utf-8', errors='replace')
        root = ET.fromstring(content)
    except Exception as e:
        return JsonResponse({'error': f'invalid OPML: {e}'}, status=400)

    added = 0
    skipped = 0
    for outline in root.iter('outline'):
        rss_url = outline.get('xmlUrl', '').strip()
        if not rss_url:
            continue
        title = outline.get('title', outline.get('text', rss_url))[:300]
        _, created = PodcastFeed.objects.get_or_create(
            user=request.user,
            rss_url=rss_url,
            defaults={'title': title},
        )
        if created:
            added += 1
        else:
            skipped += 1

    return JsonResponse({'ok': True, 'added': added, 'skipped': skipped})


# ---------------------------------------------------------------------------
# Export OPML
# ---------------------------------------------------------------------------

def _xml_text(s):
    """Drop characters XML 1.0 can't carry at all (escaping is ElementTree's job)."""
    return ''.join(c for c in s if c in '\t\n\r' or (' ' <= c <= '퟿') or ('' <= c <= '<EFBFBD>')
                   or ord(c) >= 0x10000)


@csrf_exempt
@require_http_methods(['GET'])
def export_opml(request):
    """The user's subscriptions as an OPML 2.0 file, for other podcast apps
    (and for import_opml)."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    root = ET.Element('opml', version='2.0')
    head = ET.SubElement(root, 'head')
    ET.SubElement(head, 'title').text = 'diora Podcasts'
    body = ET.SubElement(root, 'body')
    for feed in request.user.podcast_feeds.order_by('title', 'id').only('title', 'rss_url', 'link'):
        title = _xml_text(feed.title or feed.rss_url)
        ET.SubElement(body, 'outline', {
            'type': 'rss',
            'text': title,
            'title': title,
            'xmlUrl': _xml_text(feed.rss_url),
            'htmlUrl': _xml_text(feed.link),
        })
    ET.indent(root)
    xml = '<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(root, encoding='unicode') + '\n'
    response = HttpResponse(xml.encode('utf-8'), content_type='text/x-opml; charset=utf-8')
    response['Content-Disposition'] = 'attachment; filename="diora-podcasts.opml"'
    return response


# ---------------------------------------------------------------------------
# Refresh feed now
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def refresh_feed_now(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    feed_id = body.get('feed_id')
    if not feed_id:
        return JsonResponse({'error': 'feed_id required'}, status=400)

    try:
        feed = PodcastFeed.objects.get(pk=feed_id, user=request.user)
    except PodcastFeed.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    try:
        new_episodes = _refresh_feed(feed)
    except Exception as e:
        return JsonResponse({'error': str(e)}, status=502)

    return JsonResponse({'ok': True, 'new_episodes': new_episodes})


@csrf_exempt
@require_http_methods(['POST'])
def refresh_all_feeds(request):
    """Refresh all of the user's feeds on the server in one go. Feeds the
    hourly job (or an earlier call) refreshed within `max_age` seconds are
    skipped. Clients then fetch the result once (state/, inbox/ …)."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        body = json.loads(request.body or b'{}')
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    try:
        max_age = max(int(body.get('max_age', 300)), 0)
    except (TypeError, ValueError):
        return JsonResponse({'error': 'max_age must be a number'}, status=400)
    cutoff = timezone.now() - timedelta(seconds=max_age)
    feeds = request.user.podcast_feeds.filter(Q(last_refreshed_at__isnull=True) | Q(last_refreshed_at__lt=cutoff))
    refreshed, failed, new = _refresh_feeds(feeds)
    return JsonResponse({'ok': True, 'refreshed': refreshed, 'failed': failed, 'new_episodes': new})


@csrf_exempt
def podcast_state(request):
    """Everything a client shows first, in one response: feeds, queue, the
    first page of the inbox and the settings."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        limit = min(max(int(request.GET.get('inbox_limit', 100)), 0), 1000)
    except ValueError:
        return JsonResponse({'error': 'inbox_limit must be a number'}, status=400)
    s = PodcastSettings.for_user(request.user)
    return JsonResponse({
        'feeds': _feeds_data(request.user),
        'queue': _queue_data(request.user),
        'inbox': _inbox_data(request.user, 0, limit, with_description=False),
        'settings': {'enqueue_position': s.enqueue_position, 'auto_queue_position': s.auto_queue_position},
        'server_time': timezone.now().isoformat(),
    })


# ---------------------------------------------------------------------------
# Queue
# ---------------------------------------------------------------------------

@csrf_exempt
def queue_get(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    return JsonResponse({'queue': _queue_data(request.user)})


def _queue_data(user):
    items = list(
        PodcastQueue.objects
        .filter(user=user)
        .order_by('position', 'added_at', 'id')
        .select_related('episode__feed')
        .values(
            'id', 'position',
            'episode__id', 'episode__title', 'episode__audio_url',
            'episode__duration_seconds', 'episode__artwork_url',
            'episode__feed__id', 'episode__feed__title', 'episode__feed__artwork_url',
            'episode__pub_date',
        )
    )

    ep_ids = [item['episode__id'] for item in items]
    progress_map = {
        p['episode_id']: p['position_seconds']
        for p in EpisodeProgress.objects.filter(
            user=user, episode_id__in=ep_ids,
        ).values('episode_id', 'position_seconds')
    }
    for item in items:
        item['position_seconds'] = progress_map.get(item['episode__id'], 0)
        if item['episode__pub_date']:
            item['episode__pub_date'] = item['episode__pub_date'].isoformat()
    return items


@csrf_exempt
@require_http_methods(['POST'])
def queue_add(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    # One episode (episode_id) or several in order (episode_ids). Where they
    # go: `where` if given, else the user's setting. `current_episode_id` is
    # what the client is playing, for after_current.
    ids = body.get('episode_ids')
    if ids is None:
        ids = [body.get('episode_id')]
    if not isinstance(ids, list):
        return JsonResponse({'error': 'episode_ids must be a list'}, status=400)
    where = body.get('where') or PodcastSettings.for_user(request.user).enqueue_position
    if where not in QUEUE_POSITIONS:
        return JsonResponse({'error': 'unknown position'}, status=400)

    found = set(
        PodcastEpisode.objects.filter(pk__in=[i for i in ids if isinstance(i, int)], feed__user=request.user)
        .values_list('id', flat=True)
    )
    if not found:
        return JsonResponse({'error': 'not found'}, status=404)
    added = _enqueue(
        request.user, [i for i in ids if i in found], where,
        current_hint=body.get('current_episode_id'),
    )
    return JsonResponse({'ok': True, 'created': bool(added), 'added': added})


@csrf_exempt
@require_http_methods(['POST'])
def queue_remove(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    episode_id = body.get('episode_id')
    PodcastQueue.objects.filter(user=request.user, episode_id=episode_id).delete()
    return JsonResponse({'ok': True})


@csrf_exempt
@require_http_methods(['POST'])
def queue_reorder(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    order = body.get('order', [])  # list of episode ids
    for pos, ep_id in enumerate(order):
        PodcastQueue.objects.filter(user=request.user, episode_id=ep_id).update(position=pos)

    return JsonResponse({'ok': True})


@csrf_exempt
def podcast_settings(request):
    """GET the user's podcast settings; POST changes some of them."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    s = PodcastSettings.for_user(request.user)
    if request.method == 'POST':
        try:
            body = json.loads(request.body)
        except (json.JSONDecodeError, ValueError):
            return JsonResponse({'error': 'invalid JSON'}, status=400)
        fields = []
        for name in ('enqueue_position', 'auto_queue_position'):
            if name in body:
                if body[name] not in QUEUE_POSITIONS:
                    return JsonResponse({'error': f'unknown {name}'}, status=400)
                setattr(s, name, body[name])
                fields.append(name)
        if fields:
            s.save(update_fields=fields)
    elif request.method != 'GET':
        return JsonResponse({'error': 'method not allowed'}, status=405)
    return JsonResponse({
        'enqueue_position': s.enqueue_position,
        'auto_queue_position': s.auto_queue_position,
        'positions': [{'value': k, 'label': v} for k, v in PodcastSettings.POSITIONS],
    })


# ---------------------------------------------------------------------------
# Progress
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def save_progress(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    episode_id = body.get('episode_id')
    position = int(body.get('position_seconds', 0))

    try:
        episode = PodcastEpisode.objects.get(pk=episode_id, feed__user=request.user)
    except PodcastEpisode.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    progress, _ = EpisodeProgress.objects.get_or_create(
        user=request.user,
        episode=episode,
    )
    progress.position_seconds = position
    # Started: no longer new.
    if position > 0:
        progress.dismissed = True
    # Within the last seconds: played (same rule as the apps).
    if counts_as_played(position, episode.duration_seconds):
        progress.played = True

    progress.save()
    return JsonResponse({'ok': True, 'played': progress.played})


@csrf_exempt
@require_http_methods(['POST'])
def mark_played(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    episode_id = body.get('episode_id')
    played = bool(body.get('played', True))

    try:
        episode = PodcastEpisode.objects.get(pk=episode_id, feed__user=request.user)
    except PodcastEpisode.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)

    progress, _ = EpisodeProgress.objects.get_or_create(
        user=request.user,
        episode=episode,
    )
    progress.played = played
    progress.save(update_fields=['played', 'updated_at'])
    return JsonResponse({'ok': True, 'played': played})


# ---------------------------------------------------------------------------
# Dismiss episodes (hide from inbox without marking played)
# ---------------------------------------------------------------------------

@csrf_exempt
@require_http_methods(['POST'])
def dismiss_episodes(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)

    episode_ids = body.get('episode_ids', [])
    dismissed = bool(body.get('dismissed', True))

    if not isinstance(episode_ids, list) or not episode_ids:
        return JsonResponse({'error': 'episode_ids required'}, status=400)

    # Only allow dismissing episodes that belong to this user
    valid_ids = list(
        PodcastEpisode.objects.filter(
            id__in=episode_ids, feed__user=request.user
        ).values_list('id', flat=True)
    )

    for ep_id in valid_ids:
        progress, _ = EpisodeProgress.objects.get_or_create(
            user=request.user,
            episode_id=ep_id,
        )
        progress.dismissed = dismissed
        progress.save(update_fields=['dismissed', 'updated_at'])

    return JsonResponse({'ok': True, 'count': len(valid_ids)})


# ---------------------------------------------------------------------------
# Inbox
# ---------------------------------------------------------------------------

@csrf_exempt
def inbox(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)

    limit = min(int(request.GET.get('limit', getattr(settings, 'PODCAST_INBOX_PAGE_SIZE', 200))), 1000)
    offset = max(int(request.GET.get('offset', 0)), 0)
    return JsonResponse({'episodes': _inbox_data(request.user, offset, limit), 'offset': offset, 'limit': limit})


def _inbox_data(user, offset, limit, with_description=True):
    """Inbox episodes. Shownotes can be large (up to 50 KB each); the apps'
    state/ leaves them out and loads one episode's at a time."""
    fields = ['id', 'title', 'audio_url', 'duration_seconds', 'pub_date', 'artwork_url',
              'feed__id', 'feed__title', 'feed__artwork_url']
    if with_description:
        fields.insert(2, 'description')
    hidden_ids = set(
        EpisodeProgress.objects.filter(
            user=user,
        ).filter(
            Q(played=True) | Q(dismissed=True)
        ).values_list('episode_id', flat=True)
    )

    episodes = list(
        PodcastEpisode.objects.filter(feed__user=user)
        .exclude(id__in=hidden_ids)
        .select_related('feed')
        .order_by('-pub_date')
        .values(*fields)[offset:offset + limit]
    )

    progress_map = {
        ep['episode_id']: ep
        for ep in EpisodeProgress.objects.filter(
            user=user,
            episode_id__in=[e['id'] for e in episodes],
        ).values('episode_id', 'position_seconds')
    }
    queued_ids = set(
        PodcastQueue.objects.filter(
            user=user,
            episode_id__in=[e['id'] for e in episodes],
        ).values_list('episode_id', flat=True)
    )
    for ep in episodes:
        if ep['pub_date']:
            ep['pub_date'] = ep['pub_date'].isoformat()
        prog = progress_map.get(ep['id'], {})
        ep['position_seconds'] = prog.get('position_seconds', 0)
        ep['in_queue'] = ep['id'] in queued_ids
    return episodes