"""The feed reader's JSON API (session or Bearer token, like the podcast API).

Sources:   GET  /feeds/state/                 sources, unread counts, settings
           POST /feeds/discover/              {url} -> the feeds a page or URL offers
           POST /feeds/sources/add/           {url, kind, category, title?}
           POST /feeds/sources/bulk/          {sources: [{url, title, kind, category}]} (OPML import)
           POST /feeds/sources/<id>/          {title, kind, category (news), folders (blogs), fulltext, fav, hidden, notify}
           POST /feeds/folders/rename/        {name, new}   (all blogs in it)
           POST /feeds/folders/dissolve/      {name}        (the blogs stay)
           POST /feeds/sources/<id>/delete/
           GET  /feeds/export.opml
           POST /feeds/refresh/               refresh the account's sources now
News:      GET  /feeds/briefing/              what's new since the briefing was last seen
           POST /feeds/briefing/seen/         seen: everything up to now counts as read
           GET  /feeds/news/?before=&category=&ids=  stories, newest first (ids: just these, a topic's);
                                              the first page also has the day's topics
           GET  /feeds/stories/<id>/          one story with all its items
           GET  /feeds/stories/<id>/overview/ summary, what only one report says, each source's
                                              words, the story in the other language (feeds/textproc.py)
Blogs:     GET  /feeds/blogs/?folder=&source=&unread=1&fav=1&before=   (folder= alone: blogs without one;
                                              without folder/source the hidden blogs stay out)
Items:     GET  /feeds/items/<id>/            the article (fetches the page if the feed had a teaser)
           POST /feeds/items/read/            {ids, read}
           POST /feeds/items/<id>/star/       {starred}
           POST /feeds/items/<id>/progress/   {progress}   0-1; from 0.9 on it's read
           GET  /feeds/starred/
Settings:  POST /feeds/settings/              {news_keep_days}
Notices:   GET  /feeds/notifications/?after=<item id>   new posts of blogs with the bell on (devices
                                              remember the last id; without after: only that id)
Logins:    POST /feeds/logins/                {source, username, password}  subscriber login for its site
           POST /feeds/logins/delete/         {source}
"""

import json
import time
from datetime import timedelta
from xml.sax.saxutils import quoteattr

import feedparser
import requests
from django.core.cache import cache
from django.db.models import Count
from django.http import HttpResponse, JsonResponse
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods

from . import fetch, logins, refresh, textproc
from .models import Item, SiteLogin, Source, Story

PAGE = 40
BLOG_PAGE = 150        # blog posts per page (all unread ones come at once, up to BLOG_UNREAD_MAX)
BLOG_UNREAD_MAX = 600
NEW_HOURS = 24         # a blog post this young and unread is "new"
RARE_POSTS = 10        # how often a blog posts: the mean gap over its last this many posts
FOLDER_MAX = 40        # characters of a folder name
READ_AT = 0.9          # read this far, a post counts as read
BRIEFING_TOP = 8
BRIEFING_PER_CATEGORY = 5
REFRESH_WAIT = 40  # seconds an explicit refresh may take
CATEGORY_NONE = 'Allgemein'


def _auth(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    return None


def _body(request):
    try:
        return json.loads(request.body or b'{}')
    except (ValueError, UnicodeDecodeError):
        return None


def _when(value):
    """A datetime parameter; None when there's none or it's garbled (an unencoded '+' arrives as
    a space)."""
    try:
        return parse_datetime((value or '').strip().replace(' ', '+'))
    except ValueError:
        return None


def _iso(dt):
    return dt.isoformat() if dt else None


def _source_data(s, unread=0, new=0, interval=None):
    return {
        'id': s.id, 'url': s.url, 'title': s.title, 'site_url': s.site_url, 'icon_url': s.icon_url,
        'kind': s.kind, 'category': s.category, 'folders': s.folders, 'fulltext': s.fulltext,
        'unread': unread, 'new': new,
        'fav': s.fav, 'hidden': s.hidden, 'notify': s.notify, 'synthetic': s.synthetic,
        # Days between posts (blogs, for "Seltene zuerst"); a blog with one post counts as rare.
        'interval_days': interval,
        'last_refreshed_at': _iso(s.last_refreshed_at), 'last_error': s.last_error, 'error_count': s.error_count,
        'login': logins.info(s.user_id, s.url),
    }


def _item_data(i, content=False):
    d = {
        'id': i.id, 'source': i.source_id, 'source_title': i.source.title, 'kind': i.kind,
        'url': i.url, 'title': i.title, 'author': i.author, 'published': _iso(i.published), 'words': i.words,
        'summary': i.summary, 'image_url': i.image_url, 'read': i.read_at is not None, 'progress': i.progress,
        'starred': i.starred, 'story': i.story_id,
    }
    if i.source.synthetic:
        # The article of a podcast episode (docs/transcription.md); the lists fetch it with the item.
        art = getattr(i, 'episode_article', None)
        if art is not None:
            d['episode_id'] = art.episode_id
    if content:
        d['content'] = i.content
        d['content_full'] = i.content_full
        d['paywalled'] = i.paywalled
    return d


def _story_data(s, items):
    items = sorted(items, key=lambda i: i.published)
    lead = next((i for i in items if i.id == s.lead_id), items[0] if items else None)
    return {
        'id': s.id, 'first_at': _iso(s.first_at), 'last_at': _iso(s.last_at),
        'size': len(items), 'sources': len({i.source_id for i in items}),
        'category': (lead.source.category if lead else '') or CATEGORY_NONE,
        'unread': sum(1 for i in items if i.read_at is None),
        'lead': _item_data(lead) if lead else None,
        'items': [_item_data(i) for i in items],
    }


def _stories(user, qs):
    """Story dicts for a Story queryset (items fetched in one go)."""
    stories = list(qs)
    items = {}
    for i in Item.objects.filter(story__in=stories, gone=False).select_related('source'):
        items.setdefault(i.story_id, []).append(i)
    return [_story_data(s, items[s.id]) for s in stories if s.id in items]


# ---- sources

@csrf_exempt
@require_http_methods(['GET'])
def state(request):
    if (err := _auth(request)):
        return err
    unread = dict(Item.objects.filter(user=request.user, gone=False, read_at=None)
                  .values_list('source').annotate(n=Count('id')))
    now = timezone.now()
    new = dict(Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False, read_at=None,
                                   published__gte=now - timedelta(hours=NEW_HOURS))
               .values_list('source').annotate(n=Count('id')))
    sources = list(Source.objects.filter(user=request.user))
    intervals = _intervals(request.user, [x for x in sources if x.kind == Source.BLOG], now)
    s = refresh.settings_for(request.user)
    since = s.news_seen_at or (timezone.now() - timedelta(days=1))
    new_news = Story.objects.filter(user=request.user, last_at__gt=since,
                                    items__gone=False, items__read_at=None).distinct().count()
    return JsonResponse({
        'sources': [_source_data(x, unread.get(x.id, 0), new.get(x.id, 0), intervals.get(x.id)) for x in sources],
        'categories': sorted({x.category for x in sources if x.kind == Source.NEWS and x.category}),
        'folders': _ordered(_all_folders(sources), s.folder_order),
        'news_new': new_news,
        'blogs_unread': sum(unread.get(x.id, 0) for x in sources if x.kind == Source.BLOG and not x.hidden),
        'blogs_new': sum(new.get(x.id, 0) for x in sources if x.kind == Source.BLOG and not x.hidden),
        'settings': {'news_keep_days': s.news_keep_days, 'news_seen_at': _iso(s.news_seen_at)},
    })


def _clean_folder(name):
    return ' '.join(str(name or '').split())[:FOLDER_MAX]


def _all_folders(sources):
    """The folders the blogs carry, each once (case doesn't matter), sorted."""
    seen = {}
    for s in sources:
        if s.kind == Source.BLOG:
            for f in s.folders or []:
                seen.setdefault(f.casefold(), f)
    return sorted(seen.values(), key=str.casefold)


def _ordered(names, order):
    """The folders in the order the user gave them (docs/library.md), new ones after."""
    rank = {n.casefold(): i for i, n in enumerate(order or [])}
    return sorted(names, key=lambda n: (rank.get(n.casefold(), len(rank)), n.casefold()))


@csrf_exempt
@require_http_methods(['POST'])
def folder_order(request):
    """{order: [names]}: the blog folders' chip order."""
    if (err := _auth(request)):
        return err
    order = (_body(request) or {}).get('order')
    if not isinstance(order, list) or not all(isinstance(n, str) for n in order):
        return JsonResponse({'error': 'order: eine Liste von Namen'}, status=400)
    s = refresh.settings_for(request.user)
    s.folder_order = [_clean_folder(n) for n in order if _clean_folder(n)][:100]
    s.save(update_fields=['folder_order'])
    sources = Source.objects.filter(user=request.user, kind=Source.BLOG)
    return JsonResponse({'folders': _ordered(_all_folders(sources), s.folder_order)})


def _folders_for(user, names):
    """Folder names cleaned, without doubles, spelled as an existing folder is."""
    existing = {f.casefold(): f for f in _all_folders(Source.objects.filter(user=user, kind=Source.BLOG))}
    out = []
    for n in names or []:
        n = _clean_folder(n)
        if n and n.casefold() not in {o.casefold() for o in out}:
            out.append(existing.get(n.casefold(), n))
    return out


def _in_folder(source, name):
    if name == '':
        return not source.folders
    return name.casefold() in {f.casefold() for f in source.folders or []}


def _intervals(user, blogs, now):
    """Mean days between the last RARE_POSTS posts of each blog, up to now:
    (now - the oldest of them) / their number; one post counts as 30 days."""
    out = {}
    for s in blogs:
        dates = list(s.items.filter(gone=False).order_by('-published').values_list('published', flat=True)[:RARE_POSTS])
        if not dates:
            continue
        days = (now - dates[-1]).total_seconds() / 86400 / len(dates)
        out[s.id] = round(max(days, 30.0) if len(dates) == 1 else days, 2)
    return out


def _probe(url):
    """(feed url, title, entries) if `url` is a feed, else ([candidate feeds], icon)."""
    r = requests.get(url, timeout=fetch.PAGE_TIMEOUT, headers={'User-Agent': fetch.USER_AGENT})
    r.raise_for_status()
    ctype = r.headers.get('Content-Type', '')
    if 'html' in ctype and b'<rss' not in r.content[:2000] and b'<feed' not in r.content[:2000]:
        return None, fetch.discover(r.text, r.url)
    parsed = feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
    if parsed.get('bozo') and not parsed.entries and not parsed.feed.get('title'):
        return None, ([], '')
    return (r.url, fetch.short_title(fetch.plain(parsed.feed.get('title') or '')) or r.url, parsed), None


@csrf_exempt
@require_http_methods(['POST'])
def discover(request):
    """The feeds behind a URL: itself if it's a feed, else the ones the page links to."""
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    url = (body.get('url') or '').strip()
    if not url:
        return JsonResponse({'error': 'Adresse fehlt'}, status=400)
    if not url.startswith(('http://', 'https://')):
        url = 'https://' + url
    try:
        feed, page = _probe(url)
        if feed is None:
            candidates, _icon = page
            found = []
            for u, title in candidates[:6]:
                try:
                    f, _ = _probe(u)
                except requests.RequestException:
                    continue
                if f:
                    found.append(f)
            if not found:
                return JsonResponse({'error': 'Dort ist kein Feed zu finden.'}, status=404)
        else:
            found = [feed]
    except requests.RequestException:
        return JsonResponse({'error': 'Die Adresse ist nicht erreichbar.'}, status=502)
    from podcasts.models import PodcastFeed
    out = []
    for u, title, parsed in found:
        entries = parsed.entries[:5]
        teaser = sum(1 for e in entries if fetch.is_teaser(fetch.parse_entry(e, u)['content']))
        # Most entries with a sound or video file: a podcast (Abos & Quellen offers it as one).
        media = sum(1 for e in parsed.entries[:10] if _has_media(e))
        podcast = media and media * 2 >= min(len(parsed.entries), 10)
        out.append({
            'url': u, 'title': title,
            'subscribed': Source.objects.filter(user=request.user, url=u).exists(),
            'podcast_subscribed': PodcastFeed.objects.filter(user=request.user, rss_url=u).exists(),
            'samples': [fetch.plain(e.get('title') or '') for e in entries],
            # Many entries a day suggests news, few long ones a blog.
            'suggested_kind': 'podcast' if podcast
            else Source.NEWS if len(parsed.entries) >= 25 or teaser >= 3 else Source.BLOG,
        })
    return JsonResponse({'feeds': out})


def _has_media(entry):
    for enc in entry.get('enclosures') or []:
        if (enc.get('type') or '').lower().startswith(('audio/', 'video/')):
            return True
    return False


def _place(user, kind, category, folders, **more):
    """Where a new source goes: news into a section (category), a blog into
    folders (the category of an OPML folder or the add dialog is its folder)."""
    category = (category or '').strip()[:100]
    if kind == Source.NEWS:
        return dict(kind=kind, category=category, **more)
    names = folders if isinstance(folders, list) else ([category] if category else [])
    return dict(kind=kind, category='', folders=_folders_for(user, names), **more)


def _kind(v):
    return v if v in (Source.NEWS, Source.BLOG) else None


@csrf_exempt
@require_http_methods(['POST'])
def add_source(request):
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    url = (body.get('url') or '').strip()
    kind = _kind(body.get('kind'))
    if not url.startswith(('http://', 'https://')) or not kind:
        return JsonResponse({'error': 'Adresse und Art (Nachrichten/Blog) nötig'}, status=400)
    source, created = Source.objects.get_or_create(
        user=request.user, url=url[:1000],
        defaults=_place(request.user, kind, body.get('category'), body.get('folders'),
                        title=(body.get('title') or url)[:300]))
    if created:
        refresh.refresh([source], deadline=time.monotonic() + 25)
        source.refresh_from_db()
    return JsonResponse({'source': _source_data(source, source.items.filter(read_at=None, gone=False).count()),
                         'created': created})


@csrf_exempt
@require_http_methods(['POST'])
def add_sources(request):
    """Many at once (an OPML file the page read); refreshed by the next run."""
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    added = 0
    for x in body.get('sources') or []:
        url = (x.get('url') or '').strip()
        kind = _kind(x.get('kind'))
        if not url.startswith(('http://', 'https://')) or not kind:
            continue
        _, created = Source.objects.get_or_create(
            user=request.user, url=url[:1000],
            defaults=_place(request.user, kind, x.get('category'), x.get('folders'),
                            title=(x.get('title') or url)[:300]))
        added += created
    sources = Source.objects.filter(user=request.user, last_refreshed_at=None, synthetic=False)
    refresh.refresh(sources, deadline=time.monotonic() + REFRESH_WAIT)
    return JsonResponse({'added': added})


@csrf_exempt
@require_http_methods(['POST'])
def edit_source(request, pk):
    if (err := _auth(request)):
        return err
    source = Source.objects.filter(user=request.user, pk=pk).first()
    body = _body(request)
    if source is None or body is None:
        return JsonResponse({'error': 'not found'}, status=404)
    if 'title' in body and (body['title'] or '').strip():
        source.title = body['title'].strip()[:300]
    for flag in ('fav', 'hidden', 'notify'):
        if isinstance(body.get(flag), bool):
            setattr(source, flag, body[flag])
    if isinstance(body.get('folders'), list):
        source.folders = _folders_for(request.user, body['folders'])
    elif 'category' in body and source.kind == Source.BLOG and _kind(body.get('kind')) != Source.NEWS:
        # Older clients: one folder in the category field.
        cat = _clean_folder(body['category'])
        source.folders = _folders_for(request.user, [cat]) if cat else []
    if 'category' in body and (source.kind == Source.NEWS or _kind(body.get('kind')) == Source.NEWS):
        source.category = (body['category'] or '').strip()[:100]
    if body.get('fulltext') in dict(Source.FULLTEXTS):
        source.fulltext = body['fulltext']
    new_kind = _kind(body.get('kind'))
    if source.synthetic and new_kind == Source.NEWS:
        return JsonResponse({'error': 'Diese Quelle bleibt ein Blog'}, status=400)
    if new_kind and new_kind != source.kind:
        # A section becomes a folder and back.
        if new_kind == Source.BLOG and source.category and not source.folders:
            source.folders, source.category = _folders_for(request.user, [source.category]), ''
        elif new_kind == Source.NEWS and source.folders and not source.category:
            source.category = source.folders[0][:100]
        source.kind = new_kind
        # Its items follow; as blog articles they leave their stories.
        source.items.update(kind=new_kind, **({'story': None} if new_kind == Source.BLOG else {}))
        Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
    source.save()
    return JsonResponse({'source': _source_data(source)})


@csrf_exempt
@require_http_methods(['POST'])
def delete_source(request, pk):
    if (err := _auth(request)):
        return err
    Source.objects.filter(user=request.user, pk=pk).delete()
    Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
    return JsonResponse({'ok': True})


@csrf_exempt
@require_http_methods(['GET'])
def export_opml(request):
    if (err := _auth(request)):
        return err
    groups = {}
    for s in Source.objects.filter(user=request.user, synthetic=False).order_by('kind', 'category', 'title'):
        kind = 'Nachrichten' if s.kind == Source.NEWS else 'Blogs'
        place = s.category if s.kind == Source.NEWS else ((s.folders or [''])[0])
        label = kind + (f' – {place}' if place else '')
        groups.setdefault(label, []).append(s)
    lines = ['<?xml version="1.0" encoding="UTF-8"?>', '<opml version="2.0">',
             '<head><title>diora Feeds</title></head>', '<body>']
    for label, sources in groups.items():
        lines.append(f'<outline text={quoteattr(label)}>')
        for s in sources:
            lines.append(f'<outline type="rss" text={quoteattr(s.title)} xmlUrl={quoteattr(s.url)}'
                         + (f' htmlUrl={quoteattr(s.site_url)}' if s.site_url else '') + '/>')
        lines.append('</outline>')
    lines += ['</body>', '</opml>']
    resp = HttpResponse('\n'.join(lines), content_type='text/x-opml; charset=utf-8')
    resp['Content-Disposition'] = 'attachment; filename="diora-feeds.opml"'
    return resp


@csrf_exempt
@require_http_methods(['POST'])
def refresh_now(request):
    if (err := _auth(request)):
        return err
    refreshed, failed, new = refresh.refresh(Source.objects.filter(user=request.user, synthetic=False),
                                             deadline=time.monotonic() + REFRESH_WAIT)
    return JsonResponse({'refreshed': refreshed, 'failed': failed, 'new': new})


# ---- news

@csrf_exempt
@require_http_methods(['GET'])
def briefing(request):
    """Since the briefing was last seen (else the last day): the top stories
    (reported by several sources), then the rest by section, newest first."""
    if (err := _auth(request)):
        return err
    s = refresh.settings_for(request.user)
    since = s.news_seen_at or (timezone.now() - timedelta(days=1))
    qs = Story.objects.filter(user=request.user, last_at__gt=since,
                              items__gone=False, items__read_at=None).distinct()
    stories = _stories(request.user, qs)
    stories.sort(key=lambda x: (-x['sources'], -x['size'], x['last_at']), reverse=False)
    top = [x for x in stories if x['sources'] >= 2][:BRIEFING_TOP]
    top_ids = {x['id'] for x in top}
    sections = {}
    for x in sorted(stories, key=lambda x: x['last_at'], reverse=True):
        if x['id'] not in top_ids:
            sections.setdefault(x['category'], []).append(x)
    return JsonResponse({
        'since': _iso(since), 'total': len(stories), 'top': top,
        'sections': [{'category': c, 'count': len(v), 'stories': v[:BRIEFING_PER_CATEGORY]}
                     for c, v in sorted(sections.items(), key=lambda kv: (kv[0] == CATEGORY_NONE, kv[0]))],
    })


@csrf_exempt
@require_http_methods(['POST'])
def briefing_seen(request):
    if (err := _auth(request)):
        return err
    now = timezone.now()
    s = refresh.settings_for(request.user)
    Item.objects.filter(user=request.user, kind=Source.NEWS, read_at=None, fetched_at__lte=now).update(read_at=now)
    s.news_seen_at = now
    s.save(update_fields=['news_seen_at'])
    return JsonResponse({'news_seen_at': _iso(now)})


@csrf_exempt
@require_http_methods(['GET'])
def news(request):
    if (err := _auth(request)):
        return err
    qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at')
    ids = [int(x) for x in request.GET.get('ids', '').split(',') if x.strip().isdigit()]
    if ids:
        stories = _stories(request.user, qs.filter(pk__in=ids[:PAGE]))
        return JsonResponse({'stories': stories, 'more': False})
    before = _when(request.GET.get('before'))
    if before:
        qs = qs.filter(last_at__lt=before)
    category = request.GET.get('category')
    if category:
        qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category)
    stories = _stories(request.user, qs[:PAGE])
    out = {'stories': stories, 'more': len(stories) == PAGE}
    if not before:
        topics = _news_text(request.user)['topics']
        if category:
            # The category's share of each topic.
            mine = set(qs.values_list('pk', flat=True)[:500])
            topics = [t for t in ({**t, 'ids': [i for i in t['ids'] if i in mine]} for t in topics)
                      if len(t['ids']) >= textproc.TOPIC_MIN]
        out['topics'] = topics
    return JsonResponse(out)


# ---- reading help (feeds/textproc.py)

FURNITURE_ITEMS = 40          # a source's recent articles that show its furniture
TEXT_DAYS = 2                 # topics and related stories: of these days
TEXT_CACHE = 6 * 3600


def _furniture(source):
    key = f'feeds:furniture:{source.id}'
    keys = cache.get(key)
    if keys is None:
        texts = [textproc.plain(c) for c in Item.objects.filter(source=source, gone=False)
                 .order_by('-published').values_list('content', flat=True)[:FURNITURE_ITEMS]]
        keys = textproc.furniture(texts)
        cache.set(key, keys, TEXT_CACHE)
    return keys


def _clean_text(i):
    """An item's text without its source's furniture, plain."""
    return textproc.without_furniture(textproc.plain(i.content) or i.summary, _furniture(i.source))


def _news_text(user):
    """The news of the last days, looked at as a whole: topics [{label, ids}] and related
    stories in the other language {story id: [ids]}. Cached until a story changes."""
    since = timezone.now() - timedelta(days=TEXT_DAYS)
    qs = Story.objects.filter(user=user, last_at__gte=since, items__gone=False).distinct()
    latest = qs.order_by('-last_at').values_list('last_at', flat=True).first()
    key = f'feeds:newstext:{user.id}:{qs.count()}:{latest.timestamp() if latest else 0}'
    got = cache.get(key)
    if got is not None:
        return got
    items = {}
    for i in Item.objects.filter(story__in=qs, gone=False).select_related('source'):
        items.setdefault(i.story_id, []).append(i)
    tl, sl = [], []
    for sid, its in items.items():
        tl.append((sid, ' '.join(f'{i.title}. {i.summary}' for i in its), {i.source.title for i in its}))
        text = ' '.join(f'{i.title}. {_clean_text(i)[:4000]}' for i in its)
        sl.append((sid, textproc.language(text), textproc.names(text)))
    got = {
        'topics': textproc.topics(tl),
        'related': {sid: [b for b, _ in lst] for sid, lst in textproc.related(sl).items()},
    }
    cache.set(key, got, TEXT_CACHE)
    return got


@csrf_exempt
@require_http_methods(['GET'])
def story(request, pk):
    if (err := _auth(request)):
        return err
    found = _stories(request.user, Story.objects.filter(user=request.user, pk=pk))
    if not found:
        return JsonResponse({'error': 'not found'}, status=404)
    return JsonResponse({'story': found[0]})


@csrf_exempt
@require_http_methods(['GET'])
def story_overview(request, pk):
    """What a story's reports say together: a summary from all of them, what only one says,
    each source's own words, and the same story in the other language."""
    if (err := _auth(request)):
        return err
    s = Story.objects.filter(user=request.user, pk=pk).first()
    if s is None:
        return JsonResponse({'error': 'not found'}, status=404)
    its = list(Item.objects.filter(story=s, gone=False).select_related('source').order_by('published'))
    key = f'feeds:overview:{s.id}:{len(its)}:{sum(len(i.content) for i in its)}'
    out = cache.get(key)
    if out is None:
        docs = [(i.source.title, _clean_text(i)) for i in its]
        out = {}
        if len({src for src, _ in docs}) >= 2:
            summary = textproc.summarize(docs)
            out['summary'] = [{'source': src, 'text': t} for src, t in summary]
            said = {t for _, t in summary}
            novel = {src: [t for t in ts if t not in said] for src, ts in textproc.novel(docs).items()}
            out['novel'] = {src: ts for src, ts in novel.items() if ts}
            out['words'] = textproc.word_choice(docs)
        cache.set(key, out, TEXT_CACHE)
    rel = _news_text(request.user)['related'].get(s.id, [])
    related = _stories(request.user, Story.objects.filter(user=request.user, pk__in=rel)) if rel else []
    return JsonResponse({**out, 'related': related})


# ---- blogs, items

@csrf_exempt
@require_http_methods(['GET'])
def blogs(request):
    if (err := _auth(request)):
        return err
    qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source', 'episode_article')
    if request.GET.get('source', '').isdigit():
        qs = qs.filter(source_id=int(request.GET['source']))
    elif 'folder' in request.GET:
        # A folder shows all its blogs, the hidden ones too.
        name = request.GET['folder']
        ids = [s.id for s in Source.objects.filter(user=request.user, kind=Source.BLOG) if _in_folder(s, name)]
        qs = qs.filter(source_id__in=ids)
    else:
        qs = qs.filter(source__hidden=False)
        if request.GET.get('fav') == '1':
            qs = qs.filter(source__fav=True)
    unread_only = request.GET.get('unread') == '1'
    if unread_only:
        qs = qs.filter(read_at=None)
    before = _when(request.GET.get('before'))
    if before:
        qs = qs.filter(published__lt=before)
    # Unread: all of them at once (the page sorts them into days, favorites, rare blogs).
    limit = BLOG_UNREAD_MAX if unread_only else BLOG_PAGE
    items = list(qs.order_by('-published')[:limit])
    return JsonResponse({'items': [_item_data(i) for i in items], 'more': len(items) == limit})


@csrf_exempt
@require_http_methods(['GET'])
def starred(request):
    if (err := _auth(request)):
        return err
    items = Item.objects.filter(user=request.user, starred=True).select_related('source', 'episode_article').order_by('-published')
    return JsonResponse({'items': [_item_data(i) for i in items[:200]]})


@csrf_exempt
@require_http_methods(['GET'])
def item(request, pk):
    if (err := _auth(request)):
        return err
    i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source', 'episode_article').first()
    if i is None:
        return JsonResponse({'error': 'not found'}, status=404)
    if refresh.needs_article(i, i.source):
        refresh.load_article(i)
    d = _item_data(i, content=True)
    if not i.source.synthetic:
        d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source))
    return JsonResponse({'item': d})


@csrf_exempt
@require_http_methods(['POST'])
def mark_read(request):
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    ids = [int(x) for x in body.get('ids') or [] if str(x).isdigit()]
    qs = Item.objects.filter(user=request.user, id__in=ids)
    n = qs.update(read_at=timezone.now() if body.get('read', True) else None)
    return JsonResponse({'updated': n})


@csrf_exempt
@require_http_methods(['POST'])
def item_progress(request, pk):
    """How far the article was read (from any device); from READ_AT on it's read."""
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    try:
        progress = min(1.0, max(0.0, float(body.get('progress'))))
    except (TypeError, ValueError):
        return JsonResponse({'error': 'progress 0-1'}, status=400)
    i = Item.objects.filter(user=request.user, pk=pk).first()
    if i is None:
        return JsonResponse({'error': 'not found'}, status=404)
    i.progress = progress
    i.progress_at = timezone.now()
    if progress >= READ_AT and i.read_at is None:
        i.read_at = i.progress_at
    i.save(update_fields=['progress', 'progress_at', 'read_at'])
    return JsonResponse({'progress': i.progress, 'read': i.read_at is not None})


@csrf_exempt
@require_http_methods(['POST'])
def rename_folder(request):
    """A folder's new name, on every blog that carries it."""
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    old, new = _clean_folder(body.get('name')), _clean_folder(body.get('new'))
    if not old or not new:
        return JsonResponse({'error': 'Name fehlt'}, status=400)
    n = 0
    for s in Source.objects.filter(user=request.user, kind=Source.BLOG):
        if _in_folder(s, old):
            kept = [f for f in s.folders if f.casefold() != old.casefold() and f.casefold() != new.casefold()]
            s.folders = kept + [new]
            s.save(update_fields=['folders'])
            n += 1
    return JsonResponse({'blogs': n})


@csrf_exempt
@require_http_methods(['POST'])
def dissolve_folder(request):
    """The folder goes from every blog; the blogs stay subscribed."""
    if (err := _auth(request)):
        return err
    name = _clean_folder((_body(request) or {}).get('name'))
    n = 0
    for s in Source.objects.filter(user=request.user, kind=Source.BLOG):
        if name and _in_folder(s, name):
            s.folders = [f for f in s.folders if f.casefold() != name.casefold()]
            s.save(update_fields=['folders'])
            n += 1
    return JsonResponse({'blogs': n})


@csrf_exempt
@require_http_methods(['POST'])
def star(request, pk):
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    n = Item.objects.filter(user=request.user, pk=pk).update(starred=bool(body.get('starred', True)))
    return JsonResponse({'updated': n})


@csrf_exempt
@require_http_methods(['GET'])
def notifications(request):
    """What a device should announce: posts marked when they came in, still
    unread, of blogs that still have the bell on, after the id the device saw
    last. Without `after` (a device's first time) nothing, only where to start."""
    if (err := _auth(request)):
        return err
    qs = Item.objects.filter(user=request.user, notify=True)
    last = qs.order_by('-id').values_list('id', flat=True).first() or 0
    after = request.GET.get('after', '')
    if not after.isdigit():
        return JsonResponse({'items': [], 'last': last})
    items = (qs.filter(id__gt=int(after), read_at=None, gone=False, source__notify=True)
             .select_related('source').order_by('-published')[:50])
    return JsonResponse({'items': [_item_data(i) for i in items], 'last': last})


@csrf_exempt
@require_http_methods(['POST'])
def save_login(request):
    """Check a subscriber login for the source's site and keep it; then the
    articles that came only in part are fetched again."""
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
    username = (body.get('username') or '').strip()
    password = body.get('password') or ''
    if source is None or not username or not password:
        return JsonResponse({'error': 'Benutzername und Passwort nötig'}, status=400)
    if source.synthetic:
        return JsonResponse({'error': 'Diese Quelle hat keine Adresse'}, status=400)
    try:
        adapter = logins.check_and_store(request.user, source.url, username, password)
    except logins.LoginFailed as e:
        return JsonResponse({'error': str(e)}, status=400)
    # Whatever came in part (or not at all) from this site: fetch again, the newest now.
    items = Item.objects.filter(user=request.user, gone=False, content_full=False, url__icontains=adapter.domain)
    items.update(fulltext_failed=False)
    todo = list(items.select_related('source').order_by('-published')[:40])
    refresh._articles(todo, deadline=time.monotonic() + 30)
    done = Item.objects.filter(pk__in=[i.pk for i in todo], content_full=True).count()
    return JsonResponse({'ok': True, 'login': logins.info(request.user.id, source.url), 'refetched': done})


@csrf_exempt
@require_http_methods(['POST'])
def delete_login(request):
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
    adapter = logins.adapter_for(source.url) if source else None
    if adapter:
        SiteLogin.objects.filter(user=request.user, domain=adapter.domain).delete()
        logins.forget(request.user.id, adapter.domain)
    return JsonResponse({'ok': True})


@csrf_exempt
@require_http_methods(['POST'])
def settings(request):
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    s = refresh.settings_for(request.user)
    days = body.get('news_keep_days')
    if isinstance(days, int) and 1 <= days <= 30:
        s.news_keep_days = days
        s.save(update_fields=['news_keep_days'])
    return JsonResponse({'news_keep_days': s.news_keep_days})