mrwnslz
69442f4a4d
TranscriptionSettings (Token Fernet-verschluesselt wie SiteLogin), TranscriptionJob und EpisodeArticle in podcasts; Source.synthetic in feeds, das Refresh, Aufraeumen, OPML-Export, Kommando und Umstellen auf Nachrichten ueberspringt bzw. verweigert. GET/POST /podcasts/transcription/settings/ und POST /podcasts/transcription/test/ (SSRF-Pruefung ueber chapters._open, jetzt auch fuer POST; eingebaute Instanz aus DIORA_WHISPER_URL ausgenommen). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
815 lines
34 KiB
Python
815 lines
34 KiB
Python
"""The feed reader's JSON API (session or Bearer token, like the podcast API).
|
||
|
||
Sources: GET /feeds/state/ sources, unread counts, settings
|
||
POST /feeds/discover/ {url} -> the feeds a page or URL offers
|
||
POST /feeds/sources/add/ {url, kind, category, title?}
|
||
POST /feeds/sources/bulk/ {sources: [{url, title, kind, category}]} (OPML import)
|
||
POST /feeds/sources/<id>/ {title, kind, category (news), folders (blogs), fulltext, fav, hidden, notify}
|
||
POST /feeds/folders/rename/ {name, new} (all blogs in it)
|
||
POST /feeds/folders/dissolve/ {name} (the blogs stay)
|
||
POST /feeds/sources/<id>/delete/
|
||
GET /feeds/export.opml
|
||
POST /feeds/refresh/ refresh the account's sources now
|
||
News: GET /feeds/briefing/ what's new since the briefing was last seen
|
||
POST /feeds/briefing/seen/ seen: everything up to now counts as read
|
||
GET /feeds/news/?before=&category=&ids= stories, newest first (ids: just these, a topic's);
|
||
the first page also has the day's topics
|
||
GET /feeds/stories/<id>/ one story with all its items
|
||
GET /feeds/stories/<id>/overview/ summary, what only one report says, each source's
|
||
words, the story in the other language (feeds/textproc.py)
|
||
Blogs: GET /feeds/blogs/?folder=&source=&unread=1&fav=1&before= (folder= alone: blogs without one;
|
||
without folder/source the hidden blogs stay out)
|
||
Items: GET /feeds/items/<id>/ the article (fetches the page if the feed had a teaser)
|
||
POST /feeds/items/read/ {ids, read}
|
||
POST /feeds/items/<id>/star/ {starred}
|
||
POST /feeds/items/<id>/progress/ {progress} 0-1; from 0.9 on it's read
|
||
GET /feeds/starred/
|
||
Settings: POST /feeds/settings/ {news_keep_days}
|
||
Notices: GET /feeds/notifications/?after=<item id> new posts of blogs with the bell on (devices
|
||
remember the last id; without after: only that id)
|
||
Logins: POST /feeds/logins/ {source, username, password} subscriber login for its site
|
||
POST /feeds/logins/delete/ {source}
|
||
"""
|
||
|
||
import json
|
||
import time
|
||
from datetime import timedelta
|
||
from xml.sax.saxutils import quoteattr
|
||
|
||
import feedparser
|
||
import requests
|
||
from django.core.cache import cache
|
||
from django.db.models import Count
|
||
from django.http import HttpResponse, JsonResponse
|
||
from django.utils import timezone
|
||
from django.utils.dateparse import parse_datetime
|
||
from django.views.decorators.csrf import csrf_exempt
|
||
from django.views.decorators.http import require_http_methods
|
||
|
||
from . import fetch, logins, refresh, textproc
|
||
from .models import Item, SiteLogin, Source, Story
|
||
|
||
PAGE = 40
|
||
BLOG_PAGE = 150 # blog posts per page (all unread ones come at once, up to BLOG_UNREAD_MAX)
|
||
BLOG_UNREAD_MAX = 600
|
||
NEW_HOURS = 24 # a blog post this young and unread is "new"
|
||
RARE_POSTS = 10 # how often a blog posts: the mean gap over its last this many posts
|
||
FOLDER_MAX = 40 # characters of a folder name
|
||
READ_AT = 0.9 # read this far, a post counts as read
|
||
BRIEFING_TOP = 8
|
||
BRIEFING_PER_CATEGORY = 5
|
||
REFRESH_WAIT = 40 # seconds an explicit refresh may take
|
||
CATEGORY_NONE = 'Allgemein'
|
||
|
||
|
||
def _auth(request):
|
||
if not request.user.is_authenticated:
|
||
return JsonResponse({'error': 'authentication required'}, status=401)
|
||
return None
|
||
|
||
|
||
def _body(request):
|
||
try:
|
||
return json.loads(request.body or b'{}')
|
||
except (ValueError, UnicodeDecodeError):
|
||
return None
|
||
|
||
|
||
def _when(value):
|
||
"""A datetime parameter; None when there's none or it's garbled (an unencoded '+' arrives as
|
||
a space)."""
|
||
try:
|
||
return parse_datetime((value or '').strip().replace(' ', '+'))
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def _iso(dt):
|
||
return dt.isoformat() if dt else None
|
||
|
||
|
||
def _source_data(s, unread=0, new=0, interval=None):
|
||
return {
|
||
'id': s.id, 'url': s.url, 'title': s.title, 'site_url': s.site_url, 'icon_url': s.icon_url,
|
||
'kind': s.kind, 'category': s.category, 'folders': s.folders, 'fulltext': s.fulltext,
|
||
'unread': unread, 'new': new,
|
||
'fav': s.fav, 'hidden': s.hidden, 'notify': s.notify, 'synthetic': s.synthetic,
|
||
# Days between posts (blogs, for "Seltene zuerst"); a blog with one post counts as rare.
|
||
'interval_days': interval,
|
||
'last_refreshed_at': _iso(s.last_refreshed_at), 'last_error': s.last_error, 'error_count': s.error_count,
|
||
'login': logins.info(s.user_id, s.url),
|
||
}
|
||
|
||
|
||
def _item_data(i, content=False):
|
||
d = {
|
||
'id': i.id, 'source': i.source_id, 'source_title': i.source.title, 'kind': i.kind,
|
||
'url': i.url, 'title': i.title, 'author': i.author, 'published': _iso(i.published), 'words': i.words,
|
||
'summary': i.summary, 'image_url': i.image_url, 'read': i.read_at is not None, 'progress': i.progress,
|
||
'starred': i.starred, 'story': i.story_id,
|
||
}
|
||
if content:
|
||
d['content'] = i.content
|
||
d['content_full'] = i.content_full
|
||
d['paywalled'] = i.paywalled
|
||
return d
|
||
|
||
|
||
def _story_data(s, items):
|
||
items = sorted(items, key=lambda i: i.published)
|
||
lead = next((i for i in items if i.id == s.lead_id), items[0] if items else None)
|
||
return {
|
||
'id': s.id, 'first_at': _iso(s.first_at), 'last_at': _iso(s.last_at),
|
||
'size': len(items), 'sources': len({i.source_id for i in items}),
|
||
'category': (lead.source.category if lead else '') or CATEGORY_NONE,
|
||
'unread': sum(1 for i in items if i.read_at is None),
|
||
'lead': _item_data(lead) if lead else None,
|
||
'items': [_item_data(i) for i in items],
|
||
}
|
||
|
||
|
||
def _stories(user, qs):
|
||
"""Story dicts for a Story queryset (items fetched in one go)."""
|
||
stories = list(qs)
|
||
items = {}
|
||
for i in Item.objects.filter(story__in=stories, gone=False).select_related('source'):
|
||
items.setdefault(i.story_id, []).append(i)
|
||
return [_story_data(s, items[s.id]) for s in stories if s.id in items]
|
||
|
||
|
||
# ---- sources
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def state(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
unread = dict(Item.objects.filter(user=request.user, gone=False, read_at=None)
|
||
.values_list('source').annotate(n=Count('id')))
|
||
now = timezone.now()
|
||
new = dict(Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False, read_at=None,
|
||
published__gte=now - timedelta(hours=NEW_HOURS))
|
||
.values_list('source').annotate(n=Count('id')))
|
||
sources = list(Source.objects.filter(user=request.user))
|
||
intervals = _intervals(request.user, [x for x in sources if x.kind == Source.BLOG], now)
|
||
s = refresh.settings_for(request.user)
|
||
since = s.news_seen_at or (timezone.now() - timedelta(days=1))
|
||
new_news = Story.objects.filter(user=request.user, last_at__gt=since,
|
||
items__gone=False, items__read_at=None).distinct().count()
|
||
return JsonResponse({
|
||
'sources': [_source_data(x, unread.get(x.id, 0), new.get(x.id, 0), intervals.get(x.id)) for x in sources],
|
||
'categories': sorted({x.category for x in sources if x.kind == Source.NEWS and x.category}),
|
||
'folders': _ordered(_all_folders(sources), s.folder_order),
|
||
'news_new': new_news,
|
||
'blogs_unread': sum(unread.get(x.id, 0) for x in sources if x.kind == Source.BLOG and not x.hidden),
|
||
'blogs_new': sum(new.get(x.id, 0) for x in sources if x.kind == Source.BLOG and not x.hidden),
|
||
'settings': {'news_keep_days': s.news_keep_days, 'news_seen_at': _iso(s.news_seen_at)},
|
||
})
|
||
|
||
|
||
def _clean_folder(name):
|
||
return ' '.join(str(name or '').split())[:FOLDER_MAX]
|
||
|
||
|
||
def _all_folders(sources):
|
||
"""The folders the blogs carry, each once (case doesn't matter), sorted."""
|
||
seen = {}
|
||
for s in sources:
|
||
if s.kind == Source.BLOG:
|
||
for f in s.folders or []:
|
||
seen.setdefault(f.casefold(), f)
|
||
return sorted(seen.values(), key=str.casefold)
|
||
|
||
|
||
def _ordered(names, order):
|
||
"""The folders in the order the user gave them (docs/library.md), new ones after."""
|
||
rank = {n.casefold(): i for i, n in enumerate(order or [])}
|
||
return sorted(names, key=lambda n: (rank.get(n.casefold(), len(rank)), n.casefold()))
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def folder_order(request):
|
||
"""{order: [names]}: the blog folders' chip order."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
order = (_body(request) or {}).get('order')
|
||
if not isinstance(order, list) or not all(isinstance(n, str) for n in order):
|
||
return JsonResponse({'error': 'order: eine Liste von Namen'}, status=400)
|
||
s = refresh.settings_for(request.user)
|
||
s.folder_order = [_clean_folder(n) for n in order if _clean_folder(n)][:100]
|
||
s.save(update_fields=['folder_order'])
|
||
sources = Source.objects.filter(user=request.user, kind=Source.BLOG)
|
||
return JsonResponse({'folders': _ordered(_all_folders(sources), s.folder_order)})
|
||
|
||
|
||
def _folders_for(user, names):
|
||
"""Folder names cleaned, without doubles, spelled as an existing folder is."""
|
||
existing = {f.casefold(): f for f in _all_folders(Source.objects.filter(user=user, kind=Source.BLOG))}
|
||
out = []
|
||
for n in names or []:
|
||
n = _clean_folder(n)
|
||
if n and n.casefold() not in {o.casefold() for o in out}:
|
||
out.append(existing.get(n.casefold(), n))
|
||
return out
|
||
|
||
|
||
def _in_folder(source, name):
|
||
if name == '':
|
||
return not source.folders
|
||
return name.casefold() in {f.casefold() for f in source.folders or []}
|
||
|
||
|
||
def _intervals(user, blogs, now):
|
||
"""Mean days between the last RARE_POSTS posts of each blog, up to now:
|
||
(now - the oldest of them) / their number; one post counts as 30 days."""
|
||
out = {}
|
||
for s in blogs:
|
||
dates = list(s.items.filter(gone=False).order_by('-published').values_list('published', flat=True)[:RARE_POSTS])
|
||
if not dates:
|
||
continue
|
||
days = (now - dates[-1]).total_seconds() / 86400 / len(dates)
|
||
out[s.id] = round(max(days, 30.0) if len(dates) == 1 else days, 2)
|
||
return out
|
||
|
||
|
||
def _probe(url):
|
||
"""(feed url, title, entries) if `url` is a feed, else ([candidate feeds], icon)."""
|
||
r = requests.get(url, timeout=fetch.PAGE_TIMEOUT, headers={'User-Agent': fetch.USER_AGENT})
|
||
r.raise_for_status()
|
||
ctype = r.headers.get('Content-Type', '')
|
||
if 'html' in ctype and b'<rss' not in r.content[:2000] and b'<feed' not in r.content[:2000]:
|
||
return None, fetch.discover(r.text, r.url)
|
||
parsed = feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
|
||
if parsed.get('bozo') and not parsed.entries and not parsed.feed.get('title'):
|
||
return None, ([], '')
|
||
return (r.url, fetch.short_title(fetch.plain(parsed.feed.get('title') or '')) or r.url, parsed), None
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def discover(request):
|
||
"""The feeds behind a URL: itself if it's a feed, else the ones the page links to."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
url = (body.get('url') or '').strip()
|
||
if not url:
|
||
return JsonResponse({'error': 'Adresse fehlt'}, status=400)
|
||
if not url.startswith(('http://', 'https://')):
|
||
url = 'https://' + url
|
||
try:
|
||
feed, page = _probe(url)
|
||
if feed is None:
|
||
candidates, _icon = page
|
||
found = []
|
||
for u, title in candidates[:6]:
|
||
try:
|
||
f, _ = _probe(u)
|
||
except requests.RequestException:
|
||
continue
|
||
if f:
|
||
found.append(f)
|
||
if not found:
|
||
return JsonResponse({'error': 'Dort ist kein Feed zu finden.'}, status=404)
|
||
else:
|
||
found = [feed]
|
||
except requests.RequestException:
|
||
return JsonResponse({'error': 'Die Adresse ist nicht erreichbar.'}, status=502)
|
||
from podcasts.models import PodcastFeed
|
||
out = []
|
||
for u, title, parsed in found:
|
||
entries = parsed.entries[:5]
|
||
teaser = sum(1 for e in entries if fetch.is_teaser(fetch.parse_entry(e, u)['content']))
|
||
# Most entries with a sound or video file: a podcast (Abos & Quellen offers it as one).
|
||
media = sum(1 for e in parsed.entries[:10] if _has_media(e))
|
||
podcast = media and media * 2 >= min(len(parsed.entries), 10)
|
||
out.append({
|
||
'url': u, 'title': title,
|
||
'subscribed': Source.objects.filter(user=request.user, url=u).exists(),
|
||
'podcast_subscribed': PodcastFeed.objects.filter(user=request.user, rss_url=u).exists(),
|
||
'samples': [fetch.plain(e.get('title') or '') for e in entries],
|
||
# Many entries a day suggests news, few long ones a blog.
|
||
'suggested_kind': 'podcast' if podcast
|
||
else Source.NEWS if len(parsed.entries) >= 25 or teaser >= 3 else Source.BLOG,
|
||
})
|
||
return JsonResponse({'feeds': out})
|
||
|
||
|
||
def _has_media(entry):
|
||
for enc in entry.get('enclosures') or []:
|
||
if (enc.get('type') or '').lower().startswith(('audio/', 'video/')):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _place(user, kind, category, folders, **more):
|
||
"""Where a new source goes: news into a section (category), a blog into
|
||
folders (the category of an OPML folder or the add dialog is its folder)."""
|
||
category = (category or '').strip()[:100]
|
||
if kind == Source.NEWS:
|
||
return dict(kind=kind, category=category, **more)
|
||
names = folders if isinstance(folders, list) else ([category] if category else [])
|
||
return dict(kind=kind, category='', folders=_folders_for(user, names), **more)
|
||
|
||
|
||
def _kind(v):
|
||
return v if v in (Source.NEWS, Source.BLOG) else None
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def add_source(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
url = (body.get('url') or '').strip()
|
||
kind = _kind(body.get('kind'))
|
||
if not url.startswith(('http://', 'https://')) or not kind:
|
||
return JsonResponse({'error': 'Adresse und Art (Nachrichten/Blog) nötig'}, status=400)
|
||
source, created = Source.objects.get_or_create(
|
||
user=request.user, url=url[:1000],
|
||
defaults=_place(request.user, kind, body.get('category'), body.get('folders'),
|
||
title=(body.get('title') or url)[:300]))
|
||
if created:
|
||
refresh.refresh([source], deadline=time.monotonic() + 25)
|
||
source.refresh_from_db()
|
||
return JsonResponse({'source': _source_data(source, source.items.filter(read_at=None, gone=False).count()),
|
||
'created': created})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def add_sources(request):
|
||
"""Many at once (an OPML file the page read); refreshed by the next run."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
added = 0
|
||
for x in body.get('sources') or []:
|
||
url = (x.get('url') or '').strip()
|
||
kind = _kind(x.get('kind'))
|
||
if not url.startswith(('http://', 'https://')) or not kind:
|
||
continue
|
||
_, created = Source.objects.get_or_create(
|
||
user=request.user, url=url[:1000],
|
||
defaults=_place(request.user, kind, x.get('category'), x.get('folders'),
|
||
title=(x.get('title') or url)[:300]))
|
||
added += created
|
||
sources = Source.objects.filter(user=request.user, last_refreshed_at=None, synthetic=False)
|
||
refresh.refresh(sources, deadline=time.monotonic() + REFRESH_WAIT)
|
||
return JsonResponse({'added': added})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def edit_source(request, pk):
|
||
if (err := _auth(request)):
|
||
return err
|
||
source = Source.objects.filter(user=request.user, pk=pk).first()
|
||
body = _body(request)
|
||
if source is None or body is None:
|
||
return JsonResponse({'error': 'not found'}, status=404)
|
||
if 'title' in body and (body['title'] or '').strip():
|
||
source.title = body['title'].strip()[:300]
|
||
for flag in ('fav', 'hidden', 'notify'):
|
||
if isinstance(body.get(flag), bool):
|
||
setattr(source, flag, body[flag])
|
||
if isinstance(body.get('folders'), list):
|
||
source.folders = _folders_for(request.user, body['folders'])
|
||
elif 'category' in body and source.kind == Source.BLOG and _kind(body.get('kind')) != Source.NEWS:
|
||
# Older clients: one folder in the category field.
|
||
cat = _clean_folder(body['category'])
|
||
source.folders = _folders_for(request.user, [cat]) if cat else []
|
||
if 'category' in body and (source.kind == Source.NEWS or _kind(body.get('kind')) == Source.NEWS):
|
||
source.category = (body['category'] or '').strip()[:100]
|
||
if body.get('fulltext') in dict(Source.FULLTEXTS):
|
||
source.fulltext = body['fulltext']
|
||
new_kind = _kind(body.get('kind'))
|
||
if source.synthetic and new_kind == Source.NEWS:
|
||
return JsonResponse({'error': 'Diese Quelle bleibt ein Blog'}, status=400)
|
||
if new_kind and new_kind != source.kind:
|
||
# A section becomes a folder and back.
|
||
if new_kind == Source.BLOG and source.category and not source.folders:
|
||
source.folders, source.category = _folders_for(request.user, [source.category]), ''
|
||
elif new_kind == Source.NEWS and source.folders and not source.category:
|
||
source.category = source.folders[0][:100]
|
||
source.kind = new_kind
|
||
# Its items follow; as blog articles they leave their stories.
|
||
source.items.update(kind=new_kind, **({'story': None} if new_kind == Source.BLOG else {}))
|
||
Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
|
||
source.save()
|
||
return JsonResponse({'source': _source_data(source)})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def delete_source(request, pk):
|
||
if (err := _auth(request)):
|
||
return err
|
||
Source.objects.filter(user=request.user, pk=pk).delete()
|
||
Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
|
||
return JsonResponse({'ok': True})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def export_opml(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
groups = {}
|
||
for s in Source.objects.filter(user=request.user, synthetic=False).order_by('kind', 'category', 'title'):
|
||
kind = 'Nachrichten' if s.kind == Source.NEWS else 'Blogs'
|
||
place = s.category if s.kind == Source.NEWS else ((s.folders or [''])[0])
|
||
label = kind + (f' – {place}' if place else '')
|
||
groups.setdefault(label, []).append(s)
|
||
lines = ['<?xml version="1.0" encoding="UTF-8"?>', '<opml version="2.0">',
|
||
'<head><title>diora Feeds</title></head>', '<body>']
|
||
for label, sources in groups.items():
|
||
lines.append(f'<outline text={quoteattr(label)}>')
|
||
for s in sources:
|
||
lines.append(f'<outline type="rss" text={quoteattr(s.title)} xmlUrl={quoteattr(s.url)}'
|
||
+ (f' htmlUrl={quoteattr(s.site_url)}' if s.site_url else '') + '/>')
|
||
lines.append('</outline>')
|
||
lines += ['</body>', '</opml>']
|
||
resp = HttpResponse('\n'.join(lines), content_type='text/x-opml; charset=utf-8')
|
||
resp['Content-Disposition'] = 'attachment; filename="diora-feeds.opml"'
|
||
return resp
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def refresh_now(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
refreshed, failed, new = refresh.refresh(Source.objects.filter(user=request.user, synthetic=False),
|
||
deadline=time.monotonic() + REFRESH_WAIT)
|
||
return JsonResponse({'refreshed': refreshed, 'failed': failed, 'new': new})
|
||
|
||
|
||
# ---- news
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def briefing(request):
|
||
"""Since the briefing was last seen (else the last day): the top stories
|
||
(reported by several sources), then the rest by section, newest first."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
s = refresh.settings_for(request.user)
|
||
since = s.news_seen_at or (timezone.now() - timedelta(days=1))
|
||
qs = Story.objects.filter(user=request.user, last_at__gt=since,
|
||
items__gone=False, items__read_at=None).distinct()
|
||
stories = _stories(request.user, qs)
|
||
stories.sort(key=lambda x: (-x['sources'], -x['size'], x['last_at']), reverse=False)
|
||
top = [x for x in stories if x['sources'] >= 2][:BRIEFING_TOP]
|
||
top_ids = {x['id'] for x in top}
|
||
sections = {}
|
||
for x in sorted(stories, key=lambda x: x['last_at'], reverse=True):
|
||
if x['id'] not in top_ids:
|
||
sections.setdefault(x['category'], []).append(x)
|
||
return JsonResponse({
|
||
'since': _iso(since), 'total': len(stories), 'top': top,
|
||
'sections': [{'category': c, 'count': len(v), 'stories': v[:BRIEFING_PER_CATEGORY]}
|
||
for c, v in sorted(sections.items(), key=lambda kv: (kv[0] == CATEGORY_NONE, kv[0]))],
|
||
})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def briefing_seen(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
now = timezone.now()
|
||
s = refresh.settings_for(request.user)
|
||
Item.objects.filter(user=request.user, kind=Source.NEWS, read_at=None, fetched_at__lte=now).update(read_at=now)
|
||
s.news_seen_at = now
|
||
s.save(update_fields=['news_seen_at'])
|
||
return JsonResponse({'news_seen_at': _iso(now)})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def news(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at')
|
||
ids = [int(x) for x in request.GET.get('ids', '').split(',') if x.strip().isdigit()]
|
||
if ids:
|
||
stories = _stories(request.user, qs.filter(pk__in=ids[:PAGE]))
|
||
return JsonResponse({'stories': stories, 'more': False})
|
||
before = _when(request.GET.get('before'))
|
||
if before:
|
||
qs = qs.filter(last_at__lt=before)
|
||
category = request.GET.get('category')
|
||
if category:
|
||
qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category)
|
||
stories = _stories(request.user, qs[:PAGE])
|
||
out = {'stories': stories, 'more': len(stories) == PAGE}
|
||
if not before:
|
||
topics = _news_text(request.user)['topics']
|
||
if category:
|
||
# The category's share of each topic.
|
||
mine = set(qs.values_list('pk', flat=True)[:500])
|
||
topics = [t for t in ({**t, 'ids': [i for i in t['ids'] if i in mine]} for t in topics)
|
||
if len(t['ids']) >= textproc.TOPIC_MIN]
|
||
out['topics'] = topics
|
||
return JsonResponse(out)
|
||
|
||
|
||
# ---- reading help (feeds/textproc.py)
|
||
|
||
FURNITURE_ITEMS = 40 # a source's recent articles that show its furniture
|
||
TEXT_DAYS = 2 # topics and related stories: of these days
|
||
TEXT_CACHE = 6 * 3600
|
||
|
||
|
||
def _furniture(source):
|
||
key = f'feeds:furniture:{source.id}'
|
||
keys = cache.get(key)
|
||
if keys is None:
|
||
texts = [textproc.plain(c) for c in Item.objects.filter(source=source, gone=False)
|
||
.order_by('-published').values_list('content', flat=True)[:FURNITURE_ITEMS]]
|
||
keys = textproc.furniture(texts)
|
||
cache.set(key, keys, TEXT_CACHE)
|
||
return keys
|
||
|
||
|
||
def _clean_text(i):
|
||
"""An item's text without its source's furniture, plain."""
|
||
return textproc.without_furniture(textproc.plain(i.content) or i.summary, _furniture(i.source))
|
||
|
||
|
||
def _news_text(user):
|
||
"""The news of the last days, looked at as a whole: topics [{label, ids}] and related
|
||
stories in the other language {story id: [ids]}. Cached until a story changes."""
|
||
since = timezone.now() - timedelta(days=TEXT_DAYS)
|
||
qs = Story.objects.filter(user=user, last_at__gte=since, items__gone=False).distinct()
|
||
latest = qs.order_by('-last_at').values_list('last_at', flat=True).first()
|
||
key = f'feeds:newstext:{user.id}:{qs.count()}:{latest.timestamp() if latest else 0}'
|
||
got = cache.get(key)
|
||
if got is not None:
|
||
return got
|
||
items = {}
|
||
for i in Item.objects.filter(story__in=qs, gone=False).select_related('source'):
|
||
items.setdefault(i.story_id, []).append(i)
|
||
tl, sl = [], []
|
||
for sid, its in items.items():
|
||
tl.append((sid, ' '.join(f'{i.title}. {i.summary}' for i in its), {i.source.title for i in its}))
|
||
text = ' '.join(f'{i.title}. {_clean_text(i)[:4000]}' for i in its)
|
||
sl.append((sid, textproc.language(text), textproc.names(text)))
|
||
got = {
|
||
'topics': textproc.topics(tl),
|
||
'related': {sid: [b for b, _ in lst] for sid, lst in textproc.related(sl).items()},
|
||
}
|
||
cache.set(key, got, TEXT_CACHE)
|
||
return got
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def story(request, pk):
|
||
if (err := _auth(request)):
|
||
return err
|
||
found = _stories(request.user, Story.objects.filter(user=request.user, pk=pk))
|
||
if not found:
|
||
return JsonResponse({'error': 'not found'}, status=404)
|
||
return JsonResponse({'story': found[0]})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def story_overview(request, pk):
|
||
"""What a story's reports say together: a summary from all of them, what only one says,
|
||
each source's own words, and the same story in the other language."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
s = Story.objects.filter(user=request.user, pk=pk).first()
|
||
if s is None:
|
||
return JsonResponse({'error': 'not found'}, status=404)
|
||
its = list(Item.objects.filter(story=s, gone=False).select_related('source').order_by('published'))
|
||
key = f'feeds:overview:{s.id}:{len(its)}:{sum(len(i.content) for i in its)}'
|
||
out = cache.get(key)
|
||
if out is None:
|
||
docs = [(i.source.title, _clean_text(i)) for i in its]
|
||
out = {}
|
||
if len({src for src, _ in docs}) >= 2:
|
||
summary = textproc.summarize(docs)
|
||
out['summary'] = [{'source': src, 'text': t} for src, t in summary]
|
||
said = {t for _, t in summary}
|
||
novel = {src: [t for t in ts if t not in said] for src, ts in textproc.novel(docs).items()}
|
||
out['novel'] = {src: ts for src, ts in novel.items() if ts}
|
||
out['words'] = textproc.word_choice(docs)
|
||
cache.set(key, out, TEXT_CACHE)
|
||
rel = _news_text(request.user)['related'].get(s.id, [])
|
||
related = _stories(request.user, Story.objects.filter(user=request.user, pk__in=rel)) if rel else []
|
||
return JsonResponse({**out, 'related': related})
|
||
|
||
|
||
# ---- blogs, items
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def blogs(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source')
|
||
if request.GET.get('source', '').isdigit():
|
||
qs = qs.filter(source_id=int(request.GET['source']))
|
||
elif 'folder' in request.GET:
|
||
# A folder shows all its blogs, the hidden ones too.
|
||
name = request.GET['folder']
|
||
ids = [s.id for s in Source.objects.filter(user=request.user, kind=Source.BLOG) if _in_folder(s, name)]
|
||
qs = qs.filter(source_id__in=ids)
|
||
else:
|
||
qs = qs.filter(source__hidden=False)
|
||
if request.GET.get('fav') == '1':
|
||
qs = qs.filter(source__fav=True)
|
||
unread_only = request.GET.get('unread') == '1'
|
||
if unread_only:
|
||
qs = qs.filter(read_at=None)
|
||
before = _when(request.GET.get('before'))
|
||
if before:
|
||
qs = qs.filter(published__lt=before)
|
||
# Unread: all of them at once (the page sorts them into days, favorites, rare blogs).
|
||
limit = BLOG_UNREAD_MAX if unread_only else BLOG_PAGE
|
||
items = list(qs.order_by('-published')[:limit])
|
||
return JsonResponse({'items': [_item_data(i) for i in items], 'more': len(items) == limit})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def starred(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
items = Item.objects.filter(user=request.user, starred=True).select_related('source').order_by('-published')
|
||
return JsonResponse({'items': [_item_data(i) for i in items[:200]]})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def item(request, pk):
|
||
if (err := _auth(request)):
|
||
return err
|
||
i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source').first()
|
||
if i is None:
|
||
return JsonResponse({'error': 'not found'}, status=404)
|
||
if refresh.needs_article(i, i.source):
|
||
refresh.load_article(i)
|
||
d = _item_data(i, content=True)
|
||
d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source))
|
||
return JsonResponse({'item': d})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def mark_read(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
ids = [int(x) for x in body.get('ids') or [] if str(x).isdigit()]
|
||
qs = Item.objects.filter(user=request.user, id__in=ids)
|
||
n = qs.update(read_at=timezone.now() if body.get('read', True) else None)
|
||
return JsonResponse({'updated': n})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def item_progress(request, pk):
|
||
"""How far the article was read (from any device); from READ_AT on it's read."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
try:
|
||
progress = min(1.0, max(0.0, float(body.get('progress'))))
|
||
except (TypeError, ValueError):
|
||
return JsonResponse({'error': 'progress 0-1'}, status=400)
|
||
i = Item.objects.filter(user=request.user, pk=pk).first()
|
||
if i is None:
|
||
return JsonResponse({'error': 'not found'}, status=404)
|
||
i.progress = progress
|
||
i.progress_at = timezone.now()
|
||
if progress >= READ_AT and i.read_at is None:
|
||
i.read_at = i.progress_at
|
||
i.save(update_fields=['progress', 'progress_at', 'read_at'])
|
||
return JsonResponse({'progress': i.progress, 'read': i.read_at is not None})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def rename_folder(request):
|
||
"""A folder's new name, on every blog that carries it."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
old, new = _clean_folder(body.get('name')), _clean_folder(body.get('new'))
|
||
if not old or not new:
|
||
return JsonResponse({'error': 'Name fehlt'}, status=400)
|
||
n = 0
|
||
for s in Source.objects.filter(user=request.user, kind=Source.BLOG):
|
||
if _in_folder(s, old):
|
||
kept = [f for f in s.folders if f.casefold() != old.casefold() and f.casefold() != new.casefold()]
|
||
s.folders = kept + [new]
|
||
s.save(update_fields=['folders'])
|
||
n += 1
|
||
return JsonResponse({'blogs': n})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def dissolve_folder(request):
|
||
"""The folder goes from every blog; the blogs stay subscribed."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
name = _clean_folder((_body(request) or {}).get('name'))
|
||
n = 0
|
||
for s in Source.objects.filter(user=request.user, kind=Source.BLOG):
|
||
if name and _in_folder(s, name):
|
||
s.folders = [f for f in s.folders if f.casefold() != name.casefold()]
|
||
s.save(update_fields=['folders'])
|
||
n += 1
|
||
return JsonResponse({'blogs': n})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def star(request, pk):
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
n = Item.objects.filter(user=request.user, pk=pk).update(starred=bool(body.get('starred', True)))
|
||
return JsonResponse({'updated': n})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def notifications(request):
|
||
"""What a device should announce: posts marked when they came in, still
|
||
unread, of blogs that still have the bell on, after the id the device saw
|
||
last. Without `after` (a device's first time) nothing, only where to start."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
qs = Item.objects.filter(user=request.user, notify=True)
|
||
last = qs.order_by('-id').values_list('id', flat=True).first() or 0
|
||
after = request.GET.get('after', '')
|
||
if not after.isdigit():
|
||
return JsonResponse({'items': [], 'last': last})
|
||
items = (qs.filter(id__gt=int(after), read_at=None, gone=False, source__notify=True)
|
||
.select_related('source').order_by('-published')[:50])
|
||
return JsonResponse({'items': [_item_data(i) for i in items], 'last': last})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def save_login(request):
|
||
"""Check a subscriber login for the source's site and keep it; then the
|
||
articles that came only in part are fetched again."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
|
||
username = (body.get('username') or '').strip()
|
||
password = body.get('password') or ''
|
||
if source is None or not username or not password:
|
||
return JsonResponse({'error': 'Benutzername und Passwort nötig'}, status=400)
|
||
if source.synthetic:
|
||
return JsonResponse({'error': 'Diese Quelle hat keine Adresse'}, status=400)
|
||
try:
|
||
adapter = logins.check_and_store(request.user, source.url, username, password)
|
||
except logins.LoginFailed as e:
|
||
return JsonResponse({'error': str(e)}, status=400)
|
||
# Whatever came in part (or not at all) from this site: fetch again, the newest now.
|
||
items = Item.objects.filter(user=request.user, gone=False, content_full=False, url__icontains=adapter.domain)
|
||
items.update(fulltext_failed=False)
|
||
todo = list(items.select_related('source').order_by('-published')[:40])
|
||
refresh._articles(todo, deadline=time.monotonic() + 30)
|
||
done = Item.objects.filter(pk__in=[i.pk for i in todo], content_full=True).count()
|
||
return JsonResponse({'ok': True, 'login': logins.info(request.user.id, source.url), 'refetched': done})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def delete_login(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
|
||
adapter = logins.adapter_for(source.url) if source else None
|
||
if adapter:
|
||
SiteLogin.objects.filter(user=request.user, domain=adapter.domain).delete()
|
||
logins.forget(request.user.id, adapter.domain)
|
||
return JsonResponse({'ok': True})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def settings(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
s = refresh.settings_for(request.user)
|
||
days = body.get('news_keep_days')
|
||
if isinstance(days, int) and 1 <= days <= 30:
|
||
s.news_keep_days = days
|
||
s.save(update_fields=['news_keep_days'])
|
||
return JsonResponse({'news_keep_days': s.news_keep_days})
|