mrwnslz
ae5341d441
Die Folge wird geladen, zu 16 kHz mono dekodiert und an Pausen in Stücke von etwa acht Minuten geschnitten (FLAC für die eingebaute, Opus für fremde Instanzen), parallel an Whisper geschickt (drei Wiederholungen mit wachsender Pause) und per transcripts.merge geglättet. Ein Daemon-Thread arbeitet die Warteschlange ab, ein Auftrag zugleich; hängengebliebene Aufträge starten nach einem Neustart beim ersten Aufruf neu. Abbrechen löscht die Zeile des Auftrags. Gerät der Reservierung: device im Body oder Header X-Diora-Device (docs/transcription.md). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
830 lines
35 KiB
Python
830 lines
35 KiB
Python
"""The feed reader's JSON API (session or Bearer token, like the podcast API).
|
||
|
||
Sources: GET /feeds/state/ sources, unread counts, settings
|
||
POST /feeds/discover/ {url} -> the feeds a page or URL offers
|
||
POST /feeds/sources/add/ {url, kind, category, title?}
|
||
POST /feeds/sources/bulk/ {sources: [{url, title, kind, category}]} (OPML import)
|
||
POST /feeds/sources/<id>/ {title, kind, category (news), folders (blogs), fulltext, fav, hidden, notify}
|
||
POST /feeds/folders/rename/ {name, new} (all blogs in it)
|
||
POST /feeds/folders/dissolve/ {name} (the blogs stay)
|
||
POST /feeds/sources/<id>/delete/
|
||
GET /feeds/export.opml
|
||
POST /feeds/refresh/ refresh the account's sources now
|
||
News: GET /feeds/briefing/ what's new since the briefing was last seen
|
||
POST /feeds/briefing/seen/ seen: everything up to now counts as read
|
||
GET /feeds/news/?before=&category=&ids= stories, newest first (ids: just these, a topic's);
|
||
the first page also has the day's topics
|
||
GET /feeds/stories/<id>/ one story with all its items
|
||
GET /feeds/stories/<id>/overview/ summary, what only one report says, each source's
|
||
words, the story in the other language (feeds/textproc.py)
|
||
Blogs: GET /feeds/blogs/?folder=&source=&unread=1&fav=1&before= (folder= alone: blogs without one;
|
||
without folder/source the hidden blogs stay out)
|
||
Items: GET /feeds/items/<id>/ the article (fetches the page if the feed had a teaser)
|
||
POST /feeds/items/read/ {ids, read}
|
||
POST /feeds/items/<id>/star/ {starred}
|
||
POST /feeds/items/<id>/progress/ {progress} 0-1; from 0.9 on it's read
|
||
GET /feeds/starred/
|
||
Settings: POST /feeds/settings/ {news_keep_days}
|
||
Notices: GET /feeds/notifications/?after=<item id> new posts of blogs with the bell on (devices
|
||
remember the last id; without after: only that id)
|
||
Logins: POST /feeds/logins/ {source, username, password} subscriber login for its site
|
||
POST /feeds/logins/delete/ {source}
|
||
"""
|
||
|
||
import json
|
||
import time
|
||
from datetime import timedelta
|
||
from xml.sax.saxutils import quoteattr
|
||
|
||
import feedparser
|
||
import requests
|
||
from django.core.cache import cache
|
||
from django.db.models import Count
|
||
from django.http import HttpResponse, JsonResponse
|
||
from django.utils import timezone
|
||
from django.utils.dateparse import parse_datetime
|
||
from django.views.decorators.csrf import csrf_exempt
|
||
from django.views.decorators.http import require_http_methods
|
||
|
||
from . import fetch, logins, refresh, textproc
|
||
from .models import Item, SiteLogin, Source, Story
|
||
|
||
PAGE = 40
|
||
BLOG_PAGE = 150 # blog posts per page (all unread ones come at once, up to BLOG_UNREAD_MAX)
|
||
BLOG_UNREAD_MAX = 600
|
||
NEW_HOURS = 24 # a blog post this young and unread is "new"
|
||
RARE_POSTS = 10 # how often a blog posts: the mean gap over its last this many posts
|
||
FOLDER_MAX = 40 # characters of a folder name
|
||
READ_AT = 0.9 # read this far, a post counts as read
|
||
BRIEFING_TOP = 8
|
||
BRIEFING_PER_CATEGORY = 5
|
||
REFRESH_WAIT = 40 # seconds an explicit refresh may take
|
||
CATEGORY_NONE = 'Allgemein'
|
||
|
||
|
||
def _auth(request):
|
||
if not request.user.is_authenticated:
|
||
return JsonResponse({'error': 'authentication required'}, status=401)
|
||
return None
|
||
|
||
|
||
def _body(request):
|
||
try:
|
||
return json.loads(request.body or b'{}')
|
||
except (ValueError, UnicodeDecodeError):
|
||
return None
|
||
|
||
|
||
def _when(value):
|
||
"""A datetime parameter; None when there's none or it's garbled (an unencoded '+' arrives as
|
||
a space)."""
|
||
try:
|
||
return parse_datetime((value or '').strip().replace(' ', '+'))
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def _iso(dt):
|
||
return dt.isoformat() if dt else None
|
||
|
||
|
||
def _source_data(s, unread=0, new=0, interval=None):
|
||
return {
|
||
'id': s.id, 'url': s.url, 'title': s.title, 'site_url': s.site_url, 'icon_url': s.icon_url,
|
||
'kind': s.kind, 'category': s.category, 'folders': s.folders, 'fulltext': s.fulltext,
|
||
'unread': unread, 'new': new,
|
||
'fav': s.fav, 'hidden': s.hidden, 'notify': s.notify, 'synthetic': s.synthetic,
|
||
# Days between posts (blogs, for "Seltene zuerst"); a blog with one post counts as rare.
|
||
'interval_days': interval,
|
||
'last_refreshed_at': _iso(s.last_refreshed_at), 'last_error': s.last_error, 'error_count': s.error_count,
|
||
'login': logins.info(s.user_id, s.url),
|
||
}
|
||
|
||
|
||
def _item_data(i, content=False):
|
||
d = {
|
||
'id': i.id, 'source': i.source_id, 'source_title': i.source.title, 'kind': i.kind,
|
||
'url': i.url, 'title': i.title, 'author': i.author, 'published': _iso(i.published), 'words': i.words,
|
||
'summary': i.summary, 'image_url': i.image_url, 'read': i.read_at is not None, 'progress': i.progress,
|
||
'starred': i.starred, 'story': i.story_id,
|
||
}
|
||
if content:
|
||
d['content'] = i.content
|
||
d['content_full'] = i.content_full
|
||
d['paywalled'] = i.paywalled
|
||
return d
|
||
|
||
|
||
def _story_data(s, items):
|
||
items = sorted(items, key=lambda i: i.published)
|
||
lead = next((i for i in items if i.id == s.lead_id), items[0] if items else None)
|
||
return {
|
||
'id': s.id, 'first_at': _iso(s.first_at), 'last_at': _iso(s.last_at),
|
||
'size': len(items), 'sources': len({i.source_id for i in items}),
|
||
'category': (lead.source.category if lead else '') or CATEGORY_NONE,
|
||
'unread': sum(1 for i in items if i.read_at is None),
|
||
'lead': _item_data(lead) if lead else None,
|
||
'items': [_item_data(i) for i in items],
|
||
}
|
||
|
||
|
||
def _stories(user, qs):
|
||
"""Story dicts for a Story queryset (items fetched in one go)."""
|
||
stories = list(qs)
|
||
items = {}
|
||
for i in Item.objects.filter(story__in=stories, gone=False).select_related('source'):
|
||
items.setdefault(i.story_id, []).append(i)
|
||
return [_story_data(s, items[s.id]) for s in stories if s.id in items]
|
||
|
||
|
||
# ---- sources
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def state(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
unread = dict(Item.objects.filter(user=request.user, gone=False, read_at=None)
|
||
.values_list('source').annotate(n=Count('id')))
|
||
now = timezone.now()
|
||
new = dict(Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False, read_at=None,
|
||
published__gte=now - timedelta(hours=NEW_HOURS))
|
||
.values_list('source').annotate(n=Count('id')))
|
||
sources = list(Source.objects.filter(user=request.user))
|
||
intervals = _intervals(request.user, [x for x in sources if x.kind == Source.BLOG], now)
|
||
s = refresh.settings_for(request.user)
|
||
since = s.news_seen_at or (timezone.now() - timedelta(days=1))
|
||
new_news = Story.objects.filter(user=request.user, last_at__gt=since,
|
||
items__gone=False, items__read_at=None).distinct().count()
|
||
return JsonResponse({
|
||
'sources': [_source_data(x, unread.get(x.id, 0), new.get(x.id, 0), intervals.get(x.id)) for x in sources],
|
||
'categories': sorted({x.category for x in sources if x.kind == Source.NEWS and x.category}),
|
||
'folders': _ordered(_all_folders(sources), s.folder_order),
|
||
'news_new': new_news,
|
||
'blogs_unread': sum(unread.get(x.id, 0) for x in sources if x.kind == Source.BLOG and not x.hidden),
|
||
'blogs_new': sum(new.get(x.id, 0) for x in sources if x.kind == Source.BLOG and not x.hidden),
|
||
'settings': {'news_keep_days': s.news_keep_days, 'news_seen_at': _iso(s.news_seen_at)},
|
||
})
|
||
|
||
|
||
def _clean_folder(name):
|
||
return ' '.join(str(name or '').split())[:FOLDER_MAX]
|
||
|
||
|
||
def _all_folders(sources):
|
||
"""The folders the blogs carry, each once (case doesn't matter), sorted."""
|
||
seen = {}
|
||
for s in sources:
|
||
if s.kind == Source.BLOG:
|
||
for f in s.folders or []:
|
||
seen.setdefault(f.casefold(), f)
|
||
return sorted(seen.values(), key=str.casefold)
|
||
|
||
|
||
def _ordered(names, order):
|
||
"""The folders in the order the user gave them (docs/library.md), new ones after."""
|
||
rank = {n.casefold(): i for i, n in enumerate(order or [])}
|
||
return sorted(names, key=lambda n: (rank.get(n.casefold(), len(rank)), n.casefold()))
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def folder_order(request):
|
||
"""{order: [names]}: the blog folders' chip order."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
order = (_body(request) or {}).get('order')
|
||
if not isinstance(order, list) or not all(isinstance(n, str) for n in order):
|
||
return JsonResponse({'error': 'order: eine Liste von Namen'}, status=400)
|
||
s = refresh.settings_for(request.user)
|
||
s.folder_order = [_clean_folder(n) for n in order if _clean_folder(n)][:100]
|
||
s.save(update_fields=['folder_order'])
|
||
sources = Source.objects.filter(user=request.user, kind=Source.BLOG)
|
||
return JsonResponse({'folders': _ordered(_all_folders(sources), s.folder_order)})
|
||
|
||
|
||
def _folders_for(user, names):
|
||
"""Folder names cleaned, without doubles, spelled as an existing folder is."""
|
||
existing = {f.casefold(): f for f in _all_folders(Source.objects.filter(user=user, kind=Source.BLOG))}
|
||
out = []
|
||
for n in names or []:
|
||
n = _clean_folder(n)
|
||
if n and n.casefold() not in {o.casefold() for o in out}:
|
||
out.append(existing.get(n.casefold(), n))
|
||
return out
|
||
|
||
|
||
def _in_folder(source, name):
|
||
if name == '':
|
||
return not source.folders
|
||
return name.casefold() in {f.casefold() for f in source.folders or []}
|
||
|
||
|
||
def _intervals(user, blogs, now):
|
||
"""Mean days between the last RARE_POSTS posts of each blog, up to now:
|
||
(now - the oldest of them) / their number; one post counts as 30 days."""
|
||
out = {}
|
||
for s in blogs:
|
||
dates = list(s.items.filter(gone=False).order_by('-published').values_list('published', flat=True)[:RARE_POSTS])
|
||
if not dates:
|
||
continue
|
||
days = (now - dates[-1]).total_seconds() / 86400 / len(dates)
|
||
out[s.id] = round(max(days, 30.0) if len(dates) == 1 else days, 2)
|
||
return out
|
||
|
||
|
||
def _probe(url):
|
||
"""(feed url, title, entries) if `url` is a feed, else ([candidate feeds], icon)."""
|
||
r = requests.get(url, timeout=fetch.PAGE_TIMEOUT, headers={'User-Agent': fetch.USER_AGENT})
|
||
r.raise_for_status()
|
||
ctype = r.headers.get('Content-Type', '')
|
||
if 'html' in ctype and b'<rss' not in r.content[:2000] and b'<feed' not in r.content[:2000]:
|
||
return None, fetch.discover(r.text, r.url)
|
||
parsed = feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
|
||
if parsed.get('bozo') and not parsed.entries and not parsed.feed.get('title'):
|
||
return None, ([], '')
|
||
return (r.url, fetch.short_title(fetch.plain(parsed.feed.get('title') or '')) or r.url, parsed), None
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def discover(request):
|
||
"""The feeds behind a URL: itself if it's a feed, else the ones the page links to."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
url = (body.get('url') or '').strip()
|
||
if not url:
|
||
return JsonResponse({'error': 'Adresse fehlt'}, status=400)
|
||
if not url.startswith(('http://', 'https://')):
|
||
url = 'https://' + url
|
||
try:
|
||
feed, page = _probe(url)
|
||
if feed is None:
|
||
candidates, _icon = page
|
||
found = []
|
||
for u, title in candidates[:6]:
|
||
try:
|
||
f, _ = _probe(u)
|
||
except requests.RequestException:
|
||
continue
|
||
if f:
|
||
found.append(f)
|
||
if not found:
|
||
return JsonResponse({'error': 'Dort ist kein Feed zu finden.'}, status=404)
|
||
else:
|
||
found = [feed]
|
||
except requests.RequestException:
|
||
return JsonResponse({'error': 'Die Adresse ist nicht erreichbar.'}, status=502)
|
||
from podcasts.models import PodcastFeed
|
||
out = []
|
||
for u, title, parsed in found:
|
||
entries = parsed.entries[:5]
|
||
teaser = sum(1 for e in entries if fetch.is_teaser(fetch.parse_entry(e, u)['content']))
|
||
# Most entries with a sound or video file: a podcast (Abos & Quellen offers it as one).
|
||
media = sum(1 for e in parsed.entries[:10] if _has_media(e))
|
||
podcast = media and media * 2 >= min(len(parsed.entries), 10)
|
||
out.append({
|
||
'url': u, 'title': title,
|
||
'subscribed': Source.objects.filter(user=request.user, url=u).exists(),
|
||
'podcast_subscribed': PodcastFeed.objects.filter(user=request.user, rss_url=u).exists(),
|
||
'samples': [fetch.plain(e.get('title') or '') for e in entries],
|
||
# Many entries a day suggests news, few long ones a blog.
|
||
'suggested_kind': 'podcast' if podcast
|
||
else Source.NEWS if len(parsed.entries) >= 25 or teaser >= 3 else Source.BLOG,
|
||
})
|
||
return JsonResponse({'feeds': out})
|
||
|
||
|
||
def _has_media(entry):
|
||
for enc in entry.get('enclosures') or []:
|
||
if (enc.get('type') or '').lower().startswith(('audio/', 'video/')):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _place(user, kind, category, folders, **more):
|
||
"""Where a new source goes: news into a section (category), a blog into
|
||
folders (the category of an OPML folder or the add dialog is its folder)."""
|
||
category = (category or '').strip()[:100]
|
||
if kind == Source.NEWS:
|
||
return dict(kind=kind, category=category, **more)
|
||
names = folders if isinstance(folders, list) else ([category] if category else [])
|
||
return dict(kind=kind, category='', folders=_folders_for(user, names), **more)
|
||
|
||
|
||
def _kind(v):
|
||
return v if v in (Source.NEWS, Source.BLOG) else None
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def add_source(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
url = (body.get('url') or '').strip()
|
||
kind = _kind(body.get('kind'))
|
||
if not url.startswith(('http://', 'https://')) or not kind:
|
||
return JsonResponse({'error': 'Adresse und Art (Nachrichten/Blog) nötig'}, status=400)
|
||
source, created = Source.objects.get_or_create(
|
||
user=request.user, url=url[:1000],
|
||
defaults=_place(request.user, kind, body.get('category'), body.get('folders'),
|
||
title=(body.get('title') or url)[:300]))
|
||
if created:
|
||
refresh.refresh([source], deadline=time.monotonic() + 25)
|
||
source.refresh_from_db()
|
||
return JsonResponse({'source': _source_data(source, source.items.filter(read_at=None, gone=False).count()),
|
||
'created': created})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def add_sources(request):
|
||
"""Many at once (an OPML file the page read); refreshed by the next run."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
added = 0
|
||
for x in body.get('sources') or []:
|
||
url = (x.get('url') or '').strip()
|
||
kind = _kind(x.get('kind'))
|
||
if not url.startswith(('http://', 'https://')) or not kind:
|
||
continue
|
||
_, created = Source.objects.get_or_create(
|
||
user=request.user, url=url[:1000],
|
||
defaults=_place(request.user, kind, x.get('category'), x.get('folders'),
|
||
title=(x.get('title') or url)[:300]))
|
||
added += created
|
||
sources = Source.objects.filter(user=request.user, last_refreshed_at=None, synthetic=False)
|
||
refresh.refresh(sources, deadline=time.monotonic() + REFRESH_WAIT)
|
||
return JsonResponse({'added': added})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def edit_source(request, pk):
|
||
if (err := _auth(request)):
|
||
return err
|
||
source = Source.objects.filter(user=request.user, pk=pk).first()
|
||
body = _body(request)
|
||
if source is None or body is None:
|
||
return JsonResponse({'error': 'not found'}, status=404)
|
||
if 'title' in body and (body['title'] or '').strip():
|
||
source.title = body['title'].strip()[:300]
|
||
for flag in ('fav', 'hidden', 'notify'):
|
||
if isinstance(body.get(flag), bool):
|
||
setattr(source, flag, body[flag])
|
||
if isinstance(body.get('folders'), list):
|
||
source.folders = _folders_for(request.user, body['folders'])
|
||
elif 'category' in body and source.kind == Source.BLOG and _kind(body.get('kind')) != Source.NEWS:
|
||
# Older clients: one folder in the category field.
|
||
cat = _clean_folder(body['category'])
|
||
source.folders = _folders_for(request.user, [cat]) if cat else []
|
||
if 'category' in body and (source.kind == Source.NEWS or _kind(body.get('kind')) == Source.NEWS):
|
||
source.category = (body['category'] or '').strip()[:100]
|
||
if body.get('fulltext') in dict(Source.FULLTEXTS):
|
||
source.fulltext = body['fulltext']
|
||
new_kind = _kind(body.get('kind'))
|
||
if source.synthetic and new_kind == Source.NEWS:
|
||
return JsonResponse({'error': 'Diese Quelle bleibt ein Blog'}, status=400)
|
||
if new_kind and new_kind != source.kind:
|
||
# A section becomes a folder and back.
|
||
if new_kind == Source.BLOG and source.category and not source.folders:
|
||
source.folders, source.category = _folders_for(request.user, [source.category]), ''
|
||
elif new_kind == Source.NEWS and source.folders and not source.category:
|
||
source.category = source.folders[0][:100]
|
||
source.kind = new_kind
|
||
# Its items follow; as blog articles they leave their stories.
|
||
source.items.update(kind=new_kind, **({'story': None} if new_kind == Source.BLOG else {}))
|
||
Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
|
||
source.save()
|
||
return JsonResponse({'source': _source_data(source)})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def delete_source(request, pk):
|
||
if (err := _auth(request)):
|
||
return err
|
||
Source.objects.filter(user=request.user, pk=pk).delete()
|
||
Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
|
||
return JsonResponse({'ok': True})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def export_opml(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
groups = {}
|
||
for s in Source.objects.filter(user=request.user, synthetic=False).order_by('kind', 'category', 'title'):
|
||
kind = 'Nachrichten' if s.kind == Source.NEWS else 'Blogs'
|
||
place = s.category if s.kind == Source.NEWS else ((s.folders or [''])[0])
|
||
label = kind + (f' – {place}' if place else '')
|
||
groups.setdefault(label, []).append(s)
|
||
lines = ['<?xml version="1.0" encoding="UTF-8"?>', '<opml version="2.0">',
|
||
'<head><title>diora Feeds</title></head>', '<body>']
|
||
for label, sources in groups.items():
|
||
lines.append(f'<outline text={quoteattr(label)}>')
|
||
for s in sources:
|
||
lines.append(f'<outline type="rss" text={quoteattr(s.title)} xmlUrl={quoteattr(s.url)}'
|
||
+ (f' htmlUrl={quoteattr(s.site_url)}' if s.site_url else '') + '/>')
|
||
lines.append('</outline>')
|
||
lines += ['</body>', '</opml>']
|
||
resp = HttpResponse('\n'.join(lines), content_type='text/x-opml; charset=utf-8')
|
||
resp['Content-Disposition'] = 'attachment; filename="diora-feeds.opml"'
|
||
return resp
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def refresh_now(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
refreshed, failed, new = refresh.refresh(Source.objects.filter(user=request.user, synthetic=False),
|
||
deadline=time.monotonic() + REFRESH_WAIT)
|
||
return JsonResponse({'refreshed': refreshed, 'failed': failed, 'new': new})
|
||
|
||
|
||
# ---- news
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def briefing(request):
|
||
"""Since the briefing was last seen (else the last day): the top stories
|
||
(reported by several sources), then the rest by section, newest first."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
s = refresh.settings_for(request.user)
|
||
since = s.news_seen_at or (timezone.now() - timedelta(days=1))
|
||
qs = Story.objects.filter(user=request.user, last_at__gt=since,
|
||
items__gone=False, items__read_at=None).distinct()
|
||
stories = _stories(request.user, qs)
|
||
stories.sort(key=lambda x: (-x['sources'], -x['size'], x['last_at']), reverse=False)
|
||
top = [x for x in stories if x['sources'] >= 2][:BRIEFING_TOP]
|
||
top_ids = {x['id'] for x in top}
|
||
sections = {}
|
||
for x in sorted(stories, key=lambda x: x['last_at'], reverse=True):
|
||
if x['id'] not in top_ids:
|
||
sections.setdefault(x['category'], []).append(x)
|
||
return JsonResponse({
|
||
'since': _iso(since), 'total': len(stories), 'top': top,
|
||
'sections': [{'category': c, 'count': len(v), 'stories': v[:BRIEFING_PER_CATEGORY]}
|
||
for c, v in sorted(sections.items(), key=lambda kv: (kv[0] == CATEGORY_NONE, kv[0]))],
|
||
})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def briefing_seen(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
now = timezone.now()
|
||
s = refresh.settings_for(request.user)
|
||
Item.objects.filter(user=request.user, kind=Source.NEWS, read_at=None, fetched_at__lte=now).update(read_at=now)
|
||
s.news_seen_at = now
|
||
s.save(update_fields=['news_seen_at'])
|
||
return JsonResponse({'news_seen_at': _iso(now)})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def news(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at')
|
||
ids = [int(x) for x in request.GET.get('ids', '').split(',') if x.strip().isdigit()]
|
||
if ids:
|
||
stories = _stories(request.user, qs.filter(pk__in=ids[:PAGE]))
|
||
return JsonResponse({'stories': stories, 'more': False})
|
||
before = _when(request.GET.get('before'))
|
||
if before:
|
||
qs = qs.filter(last_at__lt=before)
|
||
category = request.GET.get('category')
|
||
if category:
|
||
qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category)
|
||
stories = _stories(request.user, qs[:PAGE])
|
||
out = {'stories': stories, 'more': len(stories) == PAGE}
|
||
if not before:
|
||
topics = _news_text(request.user)['topics']
|
||
if category:
|
||
# The category's share of each topic.
|
||
mine = set(qs.values_list('pk', flat=True)[:500])
|
||
topics = [t for t in ({**t, 'ids': [i for i in t['ids'] if i in mine]} for t in topics)
|
||
if len(t['ids']) >= textproc.TOPIC_MIN]
|
||
out['topics'] = topics
|
||
return JsonResponse(out)
|
||
|
||
|
||
# ---- reading help (feeds/textproc.py)
|
||
|
||
FURNITURE_ITEMS = 40 # a source's recent articles that show its furniture
|
||
TEXT_DAYS = 2 # topics and related stories: of these days
|
||
TEXT_CACHE = 6 * 3600
|
||
|
||
|
||
def _furniture(source):
|
||
key = f'feeds:furniture:{source.id}'
|
||
keys = cache.get(key)
|
||
if keys is None:
|
||
texts = [textproc.plain(c) for c in Item.objects.filter(source=source, gone=False)
|
||
.order_by('-published').values_list('content', flat=True)[:FURNITURE_ITEMS]]
|
||
keys = textproc.furniture(texts)
|
||
cache.set(key, keys, TEXT_CACHE)
|
||
return keys
|
||
|
||
|
||
def _clean_text(i):
|
||
"""An item's text without its source's furniture, plain."""
|
||
return textproc.without_furniture(textproc.plain(i.content) or i.summary, _furniture(i.source))
|
||
|
||
|
||
def _news_text(user):
|
||
"""The news of the last days, looked at as a whole: topics [{label, ids}] and related
|
||
stories in the other language {story id: [ids]}. Cached until a story changes."""
|
||
since = timezone.now() - timedelta(days=TEXT_DAYS)
|
||
qs = Story.objects.filter(user=user, last_at__gte=since, items__gone=False).distinct()
|
||
latest = qs.order_by('-last_at').values_list('last_at', flat=True).first()
|
||
key = f'feeds:newstext:{user.id}:{qs.count()}:{latest.timestamp() if latest else 0}'
|
||
got = cache.get(key)
|
||
if got is not None:
|
||
return got
|
||
items = {}
|
||
for i in Item.objects.filter(story__in=qs, gone=False).select_related('source'):
|
||
items.setdefault(i.story_id, []).append(i)
|
||
tl, sl = [], []
|
||
for sid, its in items.items():
|
||
tl.append((sid, ' '.join(f'{i.title}. {i.summary}' for i in its), {i.source.title for i in its}))
|
||
text = ' '.join(f'{i.title}. {_clean_text(i)[:4000]}' for i in its)
|
||
sl.append((sid, textproc.language(text), textproc.names(text)))
|
||
got = {
|
||
'topics': textproc.topics(tl),
|
||
'related': {sid: [b for b, _ in lst] for sid, lst in textproc.related(sl).items()},
|
||
}
|
||
cache.set(key, got, TEXT_CACHE)
|
||
return got
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def story(request, pk):
|
||
if (err := _auth(request)):
|
||
return err
|
||
found = _stories(request.user, Story.objects.filter(user=request.user, pk=pk))
|
||
if not found:
|
||
return JsonResponse({'error': 'not found'}, status=404)
|
||
return JsonResponse({'story': found[0]})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def story_overview(request, pk):
|
||
"""What a story's reports say together: a summary from all of them, what only one says,
|
||
each source's own words, and the same story in the other language."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
s = Story.objects.filter(user=request.user, pk=pk).first()
|
||
if s is None:
|
||
return JsonResponse({'error': 'not found'}, status=404)
|
||
its = list(Item.objects.filter(story=s, gone=False).select_related('source').order_by('published'))
|
||
key = f'feeds:overview:{s.id}:{len(its)}:{sum(len(i.content) for i in its)}'
|
||
out = cache.get(key)
|
||
if out is None:
|
||
docs = [(i.source.title, _clean_text(i)) for i in its]
|
||
out = {}
|
||
if len({src for src, _ in docs}) >= 2:
|
||
summary = textproc.summarize(docs)
|
||
out['summary'] = [{'source': src, 'text': t} for src, t in summary]
|
||
said = {t for _, t in summary}
|
||
novel = {src: [t for t in ts if t not in said] for src, ts in textproc.novel(docs).items()}
|
||
out['novel'] = {src: ts for src, ts in novel.items() if ts}
|
||
out['words'] = textproc.word_choice(docs)
|
||
cache.set(key, out, TEXT_CACHE)
|
||
rel = _news_text(request.user)['related'].get(s.id, [])
|
||
related = _stories(request.user, Story.objects.filter(user=request.user, pk__in=rel)) if rel else []
|
||
return JsonResponse({**out, 'related': related})
|
||
|
||
|
||
# ---- blogs, items
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def blogs(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source')
|
||
if request.GET.get('source', '').isdigit():
|
||
qs = qs.filter(source_id=int(request.GET['source']))
|
||
elif 'folder' in request.GET:
|
||
# A folder shows all its blogs, the hidden ones too.
|
||
name = request.GET['folder']
|
||
ids = [s.id for s in Source.objects.filter(user=request.user, kind=Source.BLOG) if _in_folder(s, name)]
|
||
qs = qs.filter(source_id__in=ids)
|
||
else:
|
||
qs = qs.filter(source__hidden=False)
|
||
if request.GET.get('fav') == '1':
|
||
qs = qs.filter(source__fav=True)
|
||
unread_only = request.GET.get('unread') == '1'
|
||
if unread_only:
|
||
qs = qs.filter(read_at=None)
|
||
before = _when(request.GET.get('before'))
|
||
if before:
|
||
qs = qs.filter(published__lt=before)
|
||
# Unread: all of them at once (the page sorts them into days, favorites, rare blogs).
|
||
limit = BLOG_UNREAD_MAX if unread_only else BLOG_PAGE
|
||
items = list(qs.order_by('-published')[:limit])
|
||
return JsonResponse({'items': [_item_data(i) for i in items], 'more': len(items) == limit})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def starred(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
items = Item.objects.filter(user=request.user, starred=True).select_related('source').order_by('-published')
|
||
return JsonResponse({'items': [_item_data(i) for i in items[:200]]})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def item(request, pk):
|
||
if (err := _auth(request)):
|
||
return err
|
||
i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source').first()
|
||
if i is None:
|
||
return JsonResponse({'error': 'not found'}, status=404)
|
||
if refresh.needs_article(i, i.source):
|
||
refresh.load_article(i)
|
||
d = _item_data(i, content=True)
|
||
d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source))
|
||
return JsonResponse({'item': d})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def mark_read(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
ids = [int(x) for x in body.get('ids') or [] if str(x).isdigit()]
|
||
qs = Item.objects.filter(user=request.user, id__in=ids)
|
||
n = qs.update(read_at=timezone.now() if body.get('read', True) else None)
|
||
return JsonResponse({'updated': n})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def item_progress(request, pk):
|
||
"""How far the article was read (from any device); from READ_AT on it's read."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
try:
|
||
progress = min(1.0, max(0.0, float(body.get('progress'))))
|
||
except (TypeError, ValueError):
|
||
return JsonResponse({'error': 'progress 0-1'}, status=400)
|
||
i = Item.objects.filter(user=request.user, pk=pk).first()
|
||
if i is None:
|
||
return JsonResponse({'error': 'not found'}, status=404)
|
||
i.progress = progress
|
||
i.progress_at = timezone.now()
|
||
if progress >= READ_AT and i.read_at is None:
|
||
i.read_at = i.progress_at
|
||
i.save(update_fields=['progress', 'progress_at', 'read_at'])
|
||
return JsonResponse({'progress': i.progress, 'read': i.read_at is not None})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def rename_folder(request):
|
||
"""A folder's new name, on every blog that carries it."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
old, new = _clean_folder(body.get('name')), _clean_folder(body.get('new'))
|
||
if not old or not new:
|
||
return JsonResponse({'error': 'Name fehlt'}, status=400)
|
||
n = 0
|
||
for s in Source.objects.filter(user=request.user, kind=Source.BLOG):
|
||
if _in_folder(s, old):
|
||
kept = [f for f in s.folders if f.casefold() != old.casefold() and f.casefold() != new.casefold()]
|
||
s.folders = kept + [new]
|
||
s.save(update_fields=['folders'])
|
||
n += 1
|
||
return JsonResponse({'blogs': n})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def dissolve_folder(request):
|
||
"""The folder goes from every blog; the blogs stay subscribed."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
name = _clean_folder((_body(request) or {}).get('name'))
|
||
n = 0
|
||
for s in Source.objects.filter(user=request.user, kind=Source.BLOG):
|
||
if name and _in_folder(s, name):
|
||
s.folders = [f for f in s.folders if f.casefold() != name.casefold()]
|
||
s.save(update_fields=['folders'])
|
||
n += 1
|
||
return JsonResponse({'blogs': n})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def star(request, pk):
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
n = Item.objects.filter(user=request.user, pk=pk).update(starred=bool(body.get('starred', True)))
|
||
return JsonResponse({'updated': n})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['GET'])
|
||
def notifications(request):
|
||
"""What a device should announce: posts marked when they came in, still
|
||
unread, of blogs that still have the bell on, after the id the device saw
|
||
last. Without `after` (a device's first time) nothing, only where to start.
|
||
The same for finished transcripts (`t_after`, `transcripts`, `t_last`; jobs the
|
||
user's episodes have, docs/transcription.md)."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
qs = Item.objects.filter(user=request.user, notify=True)
|
||
last = qs.order_by('-id').values_list('id', flat=True).first() or 0
|
||
# Jobs run in id order, one at a time, so a job never finishes below an id already seen.
|
||
# (A job that is done with no data only reflects a feed transcript: no notice for it.)
|
||
from podcasts.models import TranscriptionJob
|
||
jobs = TranscriptionJob.objects.filter(
|
||
episode__feed__user=request.user, state=TranscriptionJob.DONE, data__isnull=False)
|
||
t_last = jobs.order_by('-id').values_list('id', flat=True).first() or 0
|
||
out = {'items': [], 'last': last, 'transcripts': [], 't_last': t_last}
|
||
t_after = request.GET.get('t_after', '')
|
||
if t_after.isdigit():
|
||
out['transcripts'] = [{
|
||
'job_id': j.id, 'episode_id': j.episode_id, 'title': j.episode.title,
|
||
'podcast': j.episode.feed.title, 'want_article': j.want_article,
|
||
} for j in jobs.filter(id__gt=int(t_after)).select_related('episode__feed').order_by('id')[:50]]
|
||
after = request.GET.get('after', '')
|
||
if after.isdigit():
|
||
items = (qs.filter(id__gt=int(after), read_at=None, gone=False, source__notify=True)
|
||
.select_related('source').order_by('-published')[:50])
|
||
out['items'] = [_item_data(i) for i in items]
|
||
return JsonResponse(out)
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def save_login(request):
|
||
"""Check a subscriber login for the source's site and keep it; then the
|
||
articles that came only in part are fetched again."""
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
|
||
username = (body.get('username') or '').strip()
|
||
password = body.get('password') or ''
|
||
if source is None or not username or not password:
|
||
return JsonResponse({'error': 'Benutzername und Passwort nötig'}, status=400)
|
||
if source.synthetic:
|
||
return JsonResponse({'error': 'Diese Quelle hat keine Adresse'}, status=400)
|
||
try:
|
||
adapter = logins.check_and_store(request.user, source.url, username, password)
|
||
except logins.LoginFailed as e:
|
||
return JsonResponse({'error': str(e)}, status=400)
|
||
# Whatever came in part (or not at all) from this site: fetch again, the newest now.
|
||
items = Item.objects.filter(user=request.user, gone=False, content_full=False, url__icontains=adapter.domain)
|
||
items.update(fulltext_failed=False)
|
||
todo = list(items.select_related('source').order_by('-published')[:40])
|
||
refresh._articles(todo, deadline=time.monotonic() + 30)
|
||
done = Item.objects.filter(pk__in=[i.pk for i in todo], content_full=True).count()
|
||
return JsonResponse({'ok': True, 'login': logins.info(request.user.id, source.url), 'refetched': done})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def delete_login(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
|
||
adapter = logins.adapter_for(source.url) if source else None
|
||
if adapter:
|
||
SiteLogin.objects.filter(user=request.user, domain=adapter.domain).delete()
|
||
logins.forget(request.user.id, adapter.domain)
|
||
return JsonResponse({'ok': True})
|
||
|
||
|
||
@csrf_exempt
|
||
@require_http_methods(['POST'])
|
||
def settings(request):
|
||
if (err := _auth(request)):
|
||
return err
|
||
body = _body(request) or {}
|
||
s = refresh.settings_for(request.user)
|
||
days = body.get('news_keep_days')
|
||
if isinstance(days, int) and 1 <= days <= 30:
|
||
s.news_keep_days = days
|
||
s.save(update_fields=['news_keep_days'])
|
||
return JsonResponse({'news_keep_days': s.news_keep_days})
|