diora-web/feeds/views.py

"""The feed reader's JSON API (session or Bearer token, like the podcast API).

Sources:   GET  /feeds/state/                 sources, unread counts, settings
           POST /feeds/discover/              {url} -> the feeds a page or URL offers
           POST /feeds/sources/add/           {url, kind, category, title?}
           POST /feeds/sources/bulk/          {sources: [{url, title, kind, category}]} (OPML import)
           POST /feeds/sources/<id>/          {title, kind, category, fulltext}
           POST /feeds/sources/<id>/delete/
           GET  /feeds/export.opml
           POST /feeds/refresh/               refresh the account's sources now
News:      GET  /feeds/briefing/              what's new since the briefing was last seen
           POST /feeds/briefing/seen/         seen: everything up to now counts as read
           GET  /feeds/news/?before=&category=  stories, newest first
           GET  /feeds/stories/<id>/          one story with all its items
Blogs:     GET  /feeds/blogs/?folder=&source=&unread=1&before=   (a blog's category is its folder)
Items:     GET  /feeds/items/<id>/            the article (fetches the page if the feed had a teaser)
           POST /feeds/items/read/            {ids, read}
           POST /feeds/items/<id>/star/       {starred}
           GET  /feeds/starred/
Settings:  POST /feeds/settings/              {news_keep_days}
Logins:    POST /feeds/logins/                {source, username, password}  subscriber login for its site
           POST /feeds/logins/delete/         {source}
"""

import json
import time
from datetime import timedelta
from xml.sax.saxutils import quoteattr

import feedparser
import requests
from django.db.models import Count
from django.http import HttpResponse, JsonResponse
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods

from . import fetch, logins, refresh
from .models import Item, SiteLogin, Source, Story

PAGE = 40
BRIEFING_TOP = 8
BRIEFING_PER_CATEGORY = 5
REFRESH_WAIT = 40  # seconds an explicit refresh may take
CATEGORY_NONE = 'Allgemein'


def _auth(request):
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    return None


def _body(request):
    try:
        return json.loads(request.body or b'{}')
    except (ValueError, UnicodeDecodeError):
        return None


def _iso(dt):
    return dt.isoformat() if dt else None


def _source_data(s, unread=0):
    return {
        'id': s.id, 'url': s.url, 'title': s.title, 'site_url': s.site_url, 'icon_url': s.icon_url,
        'kind': s.kind, 'category': s.category, 'fulltext': s.fulltext, 'unread': unread,
        'last_refreshed_at': _iso(s.last_refreshed_at), 'last_error': s.last_error, 'error_count': s.error_count,
        'login': logins.info(s.user_id, s.url),
    }


def _item_data(i, content=False):
    d = {
        'id': i.id, 'source': i.source_id, 'source_title': i.source.title, 'kind': i.kind,
        'url': i.url, 'title': i.title, 'author': i.author, 'published': _iso(i.published),
        'summary': i.summary, 'image_url': i.image_url, 'read': i.read_at is not None,
        'starred': i.starred, 'story': i.story_id,
    }
    if content:
        d['content'] = i.content
        d['content_full'] = i.content_full
        d['paywalled'] = i.paywalled
    return d


def _story_data(s, items):
    items = sorted(items, key=lambda i: i.published)
    lead = next((i for i in items if i.id == s.lead_id), items[0] if items else None)
    return {
        'id': s.id, 'first_at': _iso(s.first_at), 'last_at': _iso(s.last_at),
        'size': len(items), 'sources': len({i.source_id for i in items}),
        'category': (lead.source.category if lead else '') or CATEGORY_NONE,
        'unread': sum(1 for i in items if i.read_at is None),
        'lead': _item_data(lead) if lead else None,
        'items': [_item_data(i) for i in items],
    }


def _stories(user, qs):
    """Story dicts for a Story queryset (items fetched in one go)."""
    stories = list(qs)
    items = {}
    for i in Item.objects.filter(story__in=stories, gone=False).select_related('source'):
        items.setdefault(i.story_id, []).append(i)
    return [_story_data(s, items[s.id]) for s in stories if s.id in items]


# ---- sources

@csrf_exempt
@require_http_methods(['GET'])
def state(request):
    if (err := _auth(request)):
        return err
    unread = dict(Item.objects.filter(user=request.user, gone=False, read_at=None)
                  .values_list('source').annotate(n=Count('id')))
    sources = list(Source.objects.filter(user=request.user))
    s = refresh.settings_for(request.user)
    since = s.news_seen_at or (timezone.now() - timedelta(days=1))
    new_news = Story.objects.filter(user=request.user, last_at__gt=since,
                                    items__gone=False, items__read_at=None).distinct().count()
    return JsonResponse({
        'sources': [_source_data(x, unread.get(x.id, 0)) for x in sources],
        'categories': sorted({x.category for x in sources if x.kind == Source.NEWS and x.category}),
        'folders': sorted({x.category for x in sources if x.kind == Source.BLOG and x.category}),
        'news_new': new_news,
        'blogs_unread': sum(n for sid, n in unread.items() if any(x.id == sid and x.kind == Source.BLOG for x in sources)),
        'settings': {'news_keep_days': s.news_keep_days, 'news_seen_at': _iso(s.news_seen_at)},
    })


def _probe(url):
    """(feed url, title, entries) if `url` is a feed, else ([candidate feeds], icon)."""
    r = requests.get(url, timeout=fetch.PAGE_TIMEOUT, headers={'User-Agent': fetch.USER_AGENT})
    r.raise_for_status()
    ctype = r.headers.get('Content-Type', '')
    if 'html' in ctype and b'<rss' not in r.content[:2000] and b'<feed' not in r.content[:2000]:
        return None, fetch.discover(r.text, r.url)
    parsed = feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
    if parsed.get('bozo') and not parsed.entries and not parsed.feed.get('title'):
        return None, ([], '')
    return (r.url, fetch.short_title(fetch.plain(parsed.feed.get('title') or '')) or r.url, parsed), None


@csrf_exempt
@require_http_methods(['POST'])
def discover(request):
    """The feeds behind a URL: itself if it's a feed, else the ones the page links to."""
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    url = (body.get('url') or '').strip()
    if not url:
        return JsonResponse({'error': 'Adresse fehlt'}, status=400)
    if not url.startswith(('http://', 'https://')):
        url = 'https://' + url
    try:
        feed, page = _probe(url)
        if feed is None:
            candidates, _icon = page
            found = []
            for u, title in candidates[:6]:
                try:
                    f, _ = _probe(u)
                except requests.RequestException:
                    continue
                if f:
                    found.append(f)
            if not found:
                return JsonResponse({'error': 'Dort ist kein Feed zu finden.'}, status=404)
        else:
            found = [feed]
    except requests.RequestException:
        return JsonResponse({'error': 'Die Adresse ist nicht erreichbar.'}, status=502)
    out = []
    for u, title, parsed in found:
        entries = parsed.entries[:5]
        teaser = sum(1 for e in entries if fetch.is_teaser(fetch.parse_entry(e, u)['content']))
        out.append({
            'url': u, 'title': title,
            'subscribed': Source.objects.filter(user=request.user, url=u).exists(),
            'samples': [fetch.plain(e.get('title') or '') for e in entries],
            # Many entries a day suggests news, few long ones a blog.
            'suggested_kind': Source.NEWS if len(parsed.entries) >= 25 or teaser >= 3 else Source.BLOG,
        })
    return JsonResponse({'feeds': out})


def _kind(v):
    return v if v in (Source.NEWS, Source.BLOG) else None


@csrf_exempt
@require_http_methods(['POST'])
def add_source(request):
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    url = (body.get('url') or '').strip()
    kind = _kind(body.get('kind'))
    if not url.startswith(('http://', 'https://')) or not kind:
        return JsonResponse({'error': 'Adresse und Art (Nachrichten/Blog) nötig'}, status=400)
    source, created = Source.objects.get_or_create(
        user=request.user, url=url[:1000],
        defaults={'title': (body.get('title') or url)[:300], 'kind': kind,
                  'category': (body.get('category') or '').strip()[:100]})
    if created:
        refresh.refresh([source], deadline=time.monotonic() + 25)
        source.refresh_from_db()
    return JsonResponse({'source': _source_data(source, source.items.filter(read_at=None, gone=False).count()),
                         'created': created})


@csrf_exempt
@require_http_methods(['POST'])
def add_sources(request):
    """Many at once (an OPML file the page read); refreshed by the next run."""
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    added = 0
    for x in body.get('sources') or []:
        url = (x.get('url') or '').strip()
        kind = _kind(x.get('kind'))
        if not url.startswith(('http://', 'https://')) or not kind:
            continue
        _, created = Source.objects.get_or_create(
            user=request.user, url=url[:1000],
            defaults={'title': (x.get('title') or url)[:300], 'kind': kind,
                      'category': (x.get('category') or '').strip()[:100]})
        added += created
    sources = Source.objects.filter(user=request.user, last_refreshed_at=None)
    refresh.refresh(sources, deadline=time.monotonic() + REFRESH_WAIT)
    return JsonResponse({'added': added})


@csrf_exempt
@require_http_methods(['POST'])
def edit_source(request, pk):
    if (err := _auth(request)):
        return err
    source = Source.objects.filter(user=request.user, pk=pk).first()
    body = _body(request)
    if source is None or body is None:
        return JsonResponse({'error': 'not found'}, status=404)
    if 'title' in body and (body['title'] or '').strip():
        source.title = body['title'].strip()[:300]
    if 'category' in body:
        source.category = (body['category'] or '').strip()[:100]
    if body.get('fulltext') in dict(Source.FULLTEXTS):
        source.fulltext = body['fulltext']
    new_kind = _kind(body.get('kind'))
    if new_kind and new_kind != source.kind:
        source.kind = new_kind
        # Its items follow; as blog articles they leave their stories.
        source.items.update(kind=new_kind, **({'story': None} if new_kind == Source.BLOG else {}))
        Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
    source.save()
    return JsonResponse({'source': _source_data(source)})


@csrf_exempt
@require_http_methods(['POST'])
def delete_source(request, pk):
    if (err := _auth(request)):
        return err
    Source.objects.filter(user=request.user, pk=pk).delete()
    Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
    return JsonResponse({'ok': True})


@csrf_exempt
@require_http_methods(['GET'])
def export_opml(request):
    if (err := _auth(request)):
        return err
    groups = {}
    for s in Source.objects.filter(user=request.user).order_by('kind', 'category', 'title'):
        kind = 'Nachrichten' if s.kind == Source.NEWS else 'Blogs'
        label = kind + (f' – {s.category}' if s.category else '')
        groups.setdefault(label, []).append(s)
    lines = ['<?xml version="1.0" encoding="UTF-8"?>', '<opml version="2.0">',
             '<head><title>diora Feeds</title></head>', '<body>']
    for label, sources in groups.items():
        lines.append(f'<outline text={quoteattr(label)}>')
        for s in sources:
            lines.append(f'<outline type="rss" text={quoteattr(s.title)} xmlUrl={quoteattr(s.url)}'
                         + (f' htmlUrl={quoteattr(s.site_url)}' if s.site_url else '') + '/>')
        lines.append('</outline>')
    lines += ['</body>', '</opml>']
    resp = HttpResponse('\n'.join(lines), content_type='text/x-opml; charset=utf-8')
    resp['Content-Disposition'] = 'attachment; filename="diora-feeds.opml"'
    return resp


@csrf_exempt
@require_http_methods(['POST'])
def refresh_now(request):
    if (err := _auth(request)):
        return err
    refreshed, failed, new = refresh.refresh(Source.objects.filter(user=request.user),
                                             deadline=time.monotonic() + REFRESH_WAIT)
    return JsonResponse({'refreshed': refreshed, 'failed': failed, 'new': new})


# ---- news

@csrf_exempt
@require_http_methods(['GET'])
def briefing(request):
    """Since the briefing was last seen (else the last day): the top stories
    (reported by several sources), then the rest by section, newest first."""
    if (err := _auth(request)):
        return err
    s = refresh.settings_for(request.user)
    since = s.news_seen_at or (timezone.now() - timedelta(days=1))
    qs = Story.objects.filter(user=request.user, last_at__gt=since,
                              items__gone=False, items__read_at=None).distinct()
    stories = _stories(request.user, qs)
    stories.sort(key=lambda x: (-x['sources'], -x['size'], x['last_at']), reverse=False)
    top = [x for x in stories if x['sources'] >= 2][:BRIEFING_TOP]
    top_ids = {x['id'] for x in top}
    sections = {}
    for x in sorted(stories, key=lambda x: x['last_at'], reverse=True):
        if x['id'] not in top_ids:
            sections.setdefault(x['category'], []).append(x)
    return JsonResponse({
        'since': _iso(since), 'total': len(stories), 'top': top,
        'sections': [{'category': c, 'count': len(v), 'stories': v[:BRIEFING_PER_CATEGORY]}
                     for c, v in sorted(sections.items(), key=lambda kv: (kv[0] == CATEGORY_NONE, kv[0]))],
    })


@csrf_exempt
@require_http_methods(['POST'])
def briefing_seen(request):
    if (err := _auth(request)):
        return err
    now = timezone.now()
    s = refresh.settings_for(request.user)
    Item.objects.filter(user=request.user, kind=Source.NEWS, read_at=None, fetched_at__lte=now).update(read_at=now)
    s.news_seen_at = now
    s.save(update_fields=['news_seen_at'])
    return JsonResponse({'news_seen_at': _iso(now)})


@csrf_exempt
@require_http_methods(['GET'])
def news(request):
    if (err := _auth(request)):
        return err
    qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at')
    before = parse_datetime(request.GET.get('before') or '')
    if before:
        qs = qs.filter(last_at__lt=before)
    category = request.GET.get('category')
    if category:
        qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category)
    stories = _stories(request.user, qs[:PAGE])
    return JsonResponse({'stories': stories, 'more': len(stories) == PAGE})


@csrf_exempt
@require_http_methods(['GET'])
def story(request, pk):
    if (err := _auth(request)):
        return err
    found = _stories(request.user, Story.objects.filter(user=request.user, pk=pk))
    if not found:
        return JsonResponse({'error': 'not found'}, status=404)
    return JsonResponse({'story': found[0]})


# ---- blogs, items

@csrf_exempt
@require_http_methods(['GET'])
def blogs(request):
    if (err := _auth(request)):
        return err
    qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source')
    if request.GET.get('source', '').isdigit():
        qs = qs.filter(source_id=int(request.GET['source']))
    elif 'folder' in request.GET:
        qs = qs.filter(source__category=request.GET['folder'])
    if request.GET.get('unread') == '1':
        qs = qs.filter(read_at=None)
    before = parse_datetime(request.GET.get('before') or '')
    if before:
        qs = qs.filter(published__lt=before)
    items = list(qs.order_by('-published')[:PAGE])
    return JsonResponse({'items': [_item_data(i) for i in items], 'more': len(items) == PAGE})


@csrf_exempt
@require_http_methods(['GET'])
def starred(request):
    if (err := _auth(request)):
        return err
    items = Item.objects.filter(user=request.user, starred=True).select_related('source').order_by('-published')
    return JsonResponse({'items': [_item_data(i) for i in items[:200]]})


@csrf_exempt
@require_http_methods(['GET'])
def item(request, pk):
    if (err := _auth(request)):
        return err
    i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source').first()
    if i is None:
        return JsonResponse({'error': 'not found'}, status=404)
    if refresh.needs_article(i, i.source):
        refresh.load_article(i)
    return JsonResponse({'item': _item_data(i, content=True)})


@csrf_exempt
@require_http_methods(['POST'])
def mark_read(request):
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    ids = [int(x) for x in body.get('ids') or [] if str(x).isdigit()]
    qs = Item.objects.filter(user=request.user, id__in=ids)
    n = qs.update(read_at=timezone.now() if body.get('read', True) else None)
    return JsonResponse({'updated': n})


@csrf_exempt
@require_http_methods(['POST'])
def star(request, pk):
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    n = Item.objects.filter(user=request.user, pk=pk).update(starred=bool(body.get('starred', True)))
    return JsonResponse({'updated': n})


@csrf_exempt
@require_http_methods(['POST'])
def save_login(request):
    """Check a subscriber login for the source's site and keep it; then the
    articles that came only in part are fetched again."""
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
    username = (body.get('username') or '').strip()
    password = body.get('password') or ''
    if source is None or not username or not password:
        return JsonResponse({'error': 'Benutzername und Passwort nötig'}, status=400)
    try:
        adapter = logins.check_and_store(request.user, source.url, username, password)
    except logins.LoginFailed as e:
        return JsonResponse({'error': str(e)}, status=400)
    # Whatever came in part (or not at all) from this site: fetch again, the newest now.
    items = Item.objects.filter(user=request.user, gone=False, content_full=False, url__icontains=adapter.domain)
    items.update(fulltext_failed=False)
    todo = list(items.select_related('source').order_by('-published')[:40])
    refresh._articles(todo, deadline=time.monotonic() + 30)
    done = Item.objects.filter(pk__in=[i.pk for i in todo], content_full=True).count()
    return JsonResponse({'ok': True, 'login': logins.info(request.user.id, source.url), 'refetched': done})


@csrf_exempt
@require_http_methods(['POST'])
def delete_login(request):
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
    adapter = logins.adapter_for(source.url) if source else None
    if adapter:
        SiteLogin.objects.filter(user=request.user, domain=adapter.domain).delete()
        logins.forget(request.user.id, adapter.domain)
    return JsonResponse({'ok': True})


@csrf_exempt
@require_http_methods(['POST'])
def settings(request):
    if (err := _auth(request)):
        return err
    body = _body(request) or {}
    s = refresh.settings_for(request.user)
    days = body.get('news_keep_days')
    if isinstance(days, int) and 1 <= days <= 30:
        s.news_keep_days = days
        s.save(update_fields=['news_keep_days'])
    return JsonResponse({'news_keep_days': s.news_keep_days})