feeds: unfertige Änderungen aus f2583cb wieder heraus (versehentlich mit committet, liegen weiter im Arbeitsbaum)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f2583cbefd
commit
2327eb06d5
3 changed files with 6 additions and 462 deletions
|
|
@ -1,352 +0,0 @@
|
|||
"""Reading help for news, with classic text methods (no language model):
|
||||
|
||||
- furniture: sentences a source prints under many of its articles (donation boxes,
|
||||
"Diese Nachricht wurde … gesendet") are no part of an article; they are left out
|
||||
when an article is shown and when a story is looked at;
|
||||
- a story's overview: a summary drawn from all its reports (LexRank: the sentences
|
||||
most like the others, without repeating each other — MMR), what only one report
|
||||
says (its sentences unlike any other source's), and each source's own words for
|
||||
the event (log-odds of a word in that report against the story's other reports);
|
||||
- related stories in the other language (German/English): stories of the same days
|
||||
that share names (capitalised words that aren't at a sentence's start), weighted by
|
||||
how rare a name is among the day's stories;
|
||||
- topics: stories grouped by their words when they're about the same matter, not
|
||||
only the same event, labelled with the words they share most.
|
||||
|
||||
Pure Python: these run on the server with the feeds, for one user's stories at a time.
|
||||
"""
|
||||
|
||||
import math
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
from . import fetch
|
||||
|
||||
_SENT_END = re.compile(r'(?<=[.!?…])\s+(?=[„"»A-ZÄÖÜ0-9])')
|
||||
_TOKEN = re.compile(r"[A-Za-zÄÖÜäöüßÀ-ÿ][A-Za-zÄÖÜäöüßÀ-ÿ0-9'’\-]+")
|
||||
|
||||
STOP = set('''
|
||||
aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher
|
||||
da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese
|
||||
diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für
|
||||
gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer
|
||||
in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen
|
||||
nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind
|
||||
so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter
|
||||
weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier
|
||||
heute gestern morgen laut sagt sagte sagten erst mal neu video live ticker meldung bericht hatte hätte
|
||||
wäre würde würden zudem sowohl jedoch allerdings bereits rund etwa demnach zufolge angaben unter
|
||||
the and for with from that this have has are was were will would been into about after over their they
|
||||
his her its our you your not but all can new says said more than when what who how why off out also had
|
||||
one two which there been being some other could should would may might them then just only very
|
||||
'''.split())
|
||||
|
||||
# Capitalised words that are no names (German nouns are capitalised too: those don't matter,
|
||||
# they never match an English text; these would match both).
|
||||
_NOT_NAMES = set('''
|
||||
Der Die Das Den Dem Des Ein Eine Einen Einem Einer Er Sie Es Wir Ihr Ich Und Oder Aber Auch Doch Mit Von
|
||||
Nach Bei Für Auf Aus Im In Am Zum Zur Vor Seit Als Wie Wenn Dass Nicht Noch Nur So Da Dann Hier Jetzt
|
||||
The A An And Or But With From For After Before In On At By To Of As It He She They We I This That These
|
||||
Those Mr Mrs Ms Dr Monday Tuesday Wednesday Thursday Friday Saturday Sunday Montag Dienstag Mittwoch
|
||||
Donnerstag Freitag Samstag Sonntag Sonnabend January February March April May June July August September
|
||||
October November December Januar Februar März Juni Juli Oktober Dezember Reuters AFP AP dpa BBC CNN
|
||||
'''.split())
|
||||
|
||||
|
||||
def plain(html):
|
||||
return fetch.plain(html or '')
|
||||
|
||||
|
||||
def sentences(text):
|
||||
text = re.sub(r'\s+', ' ', text or '').strip()
|
||||
return [s.strip() for s in _SENT_END.split(text) if 40 <= len(s.strip()) <= 400]
|
||||
|
||||
|
||||
def _key(s):
|
||||
"""A sentence, the same however its numbers and spaces differ."""
|
||||
return re.sub(r'\d+', '0', re.sub(r'\s+', ' ', s.lower())).strip()
|
||||
|
||||
|
||||
def words(text):
|
||||
return [w.lower() for w in _TOKEN.findall(text or '') if len(w) > 2 and w.lower() not in STOP]
|
||||
|
||||
|
||||
# ---- furniture ------------------------------------------------------------------------------------
|
||||
|
||||
FURNITURE_MIN = 3 # in this many of a source's articles: furniture
|
||||
|
||||
|
||||
def furniture(texts):
|
||||
"""The sentences (keys) that stand in at least FURNITURE_MIN of these texts (one source's)."""
|
||||
seen = Counter()
|
||||
for t in texts:
|
||||
seen.update({_key(s) for s in _units(t)})
|
||||
return {k for k, n in seen.items() if n >= FURNITURE_MIN}
|
||||
|
||||
|
||||
def _units(text):
|
||||
return [u.strip() for u in re.split(r'\n|(?<=[.!?])\s+', text or '') if len(u.strip()) > 15]
|
||||
|
||||
|
||||
def without_furniture(text, keys):
|
||||
if not keys:
|
||||
return text
|
||||
return ' '.join(u for u in _units(text) if _key(u) not in keys)
|
||||
|
||||
|
||||
_BLOCK = re.compile(r'<(p|li|h[1-6]|blockquote|figcaption|div)\b[^>]*>(.*?)</\1>', re.S | re.I)
|
||||
|
||||
|
||||
def html_without_furniture(html, keys):
|
||||
"""The article's HTML without the blocks whose text is (only) furniture."""
|
||||
if not keys or not html:
|
||||
return html
|
||||
|
||||
def keep(m):
|
||||
inner = plain(m.group(2))
|
||||
units = _units(inner)
|
||||
if units and all(_key(u) in keys for u in units):
|
||||
return ''
|
||||
return m.group(0)
|
||||
return _BLOCK.sub(keep, html)
|
||||
|
||||
|
||||
# ---- tf-idf over sentences -------------------------------------------------------------------------
|
||||
|
||||
def _vectors(texts):
|
||||
toks = [Counter(words(t)) for t in texts]
|
||||
df = Counter()
|
||||
for t in toks:
|
||||
df.update(t.keys())
|
||||
n = len(texts)
|
||||
idf = {w: math.log((n + 1) / (d + 1)) + 1 for w, d in df.items()}
|
||||
vecs = []
|
||||
for t in toks:
|
||||
v = {w: (1 + math.log(c)) * idf[w] for w, c in t.items()}
|
||||
norm = math.sqrt(sum(x * x for x in v.values())) or 1.0
|
||||
vecs.append({w: x / norm for w, x in v.items()})
|
||||
return vecs
|
||||
|
||||
|
||||
def _cos(a, b):
|
||||
if len(a) > len(b):
|
||||
a, b = b, a
|
||||
return sum(x * b.get(w, 0.0) for w, x in a.items())
|
||||
|
||||
|
||||
def summarize(docs, k=3, lam=0.7):
|
||||
"""docs: [(source, text)]. The k sentences that say most of what all say, in their order
|
||||
of appearance: [(source, sentence)]."""
|
||||
ss = [(src, s) for src, t in docs for s in sentences(t)[:30]]
|
||||
if len(ss) <= k:
|
||||
return ss
|
||||
vecs = _vectors([s for _, s in ss])
|
||||
n = len(ss)
|
||||
sim = [[_cos(vecs[a], vecs[b]) if a != b else 0.0 for b in range(n)] for a in range(n)]
|
||||
links = [[1.0 if sim[a][b] > 0.1 else 0.0 for b in range(n)] for a in range(n)]
|
||||
deg = [sum(r) or 1.0 for r in links]
|
||||
rank = [1.0 / n] * n
|
||||
for _ in range(40):
|
||||
rank = [0.15 / n + 0.85 * sum(links[b][a] * rank[b] / deg[b] for b in range(n)) for a in range(n)]
|
||||
top = max(rank) or 1.0
|
||||
chosen = []
|
||||
for _ in range(k):
|
||||
best, score = None, -1e9
|
||||
for a in range(n):
|
||||
if a in chosen:
|
||||
continue
|
||||
s = lam * rank[a] / top - (1 - lam) * max((sim[a][c] for c in chosen), default=0.0)
|
||||
if s > score:
|
||||
best, score = a, s
|
||||
chosen.append(best)
|
||||
return [ss[c] for c in sorted(chosen)]
|
||||
|
||||
|
||||
def novel(docs, per_source=2, limit=0.2):
|
||||
"""What only one report says: per source, up to per_source of its sentences that no other
|
||||
source has anything like (cosine < limit), the most central of its own first."""
|
||||
ss = [(src, s) for src, t in docs for s in sentences(t)[:40]]
|
||||
if len({src for src, _ in ss}) < 2:
|
||||
return {}
|
||||
vecs = _vectors([s for _, s in ss])
|
||||
out = defaultdict(list)
|
||||
for a, (src, s) in enumerate(ss):
|
||||
others = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 != src]
|
||||
if not others or max(others) >= limit:
|
||||
continue
|
||||
own = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 == src and b != a]
|
||||
out[src].append((sum(own) / len(own) if own else 0.0, s))
|
||||
return {src: [s for _, s in sorted(v, key=lambda x: -x[0])[:per_source]] for src, v in out.items()}
|
||||
|
||||
|
||||
def word_choice(docs, per_source=6):
|
||||
"""Each source's own words for the event: log-odds of a word in its report against the
|
||||
other reports (with a prior from all of them), at least twice in its report."""
|
||||
forms = defaultdict(Counter)
|
||||
counts = {}
|
||||
for src, t in docs:
|
||||
c = Counter()
|
||||
for w in _TOKEN.findall(t or ''):
|
||||
lw = w.lower()
|
||||
if len(lw) > 3 and lw not in STOP:
|
||||
c[lw] += 1
|
||||
forms[lw][w] += 1
|
||||
counts[src] = counts.get(src, Counter()) + c
|
||||
if len(counts) < 2:
|
||||
return {}
|
||||
total = sum(counts.values(), Counter())
|
||||
a0 = sum(total.values()) or 1
|
||||
out = {}
|
||||
for src, c in counts.items():
|
||||
n_i = sum(c.values()) or 1
|
||||
rest = total - c
|
||||
n_j = sum(rest.values()) or 1
|
||||
scored = []
|
||||
for w, f in c.items():
|
||||
if f < 2:
|
||||
continue
|
||||
aw = total[w] / a0 * 100
|
||||
d = math.log((f + aw) / (n_i + 100 - f - aw)) - math.log((rest[w] + aw) / (n_j + 100 - rest[w] - aw))
|
||||
scored.append((d / math.sqrt(1 / (f + aw) + 1 / (rest[w] + aw)), w))
|
||||
# Nouns and names (as written: capitalised), not the verbs and particles around them.
|
||||
best = [(z, forms[w].most_common(1)[0][0]) for z, w in sorted(scored, reverse=True) if z > 1.0]
|
||||
out[src] = [w for z, w in best if w[:1].isupper()][:per_source]
|
||||
return {k: v for k, v in out.items() if v}
|
||||
|
||||
|
||||
# ---- names, languages, related stories -------------------------------------------------------------
|
||||
|
||||
_EN = set('the and of to in is that for with was said on as by at from'.split())
|
||||
_DE = set('der die das und ist nicht mit den von zu im sich auf für dem des eine sagte'.split())
|
||||
|
||||
|
||||
def language(text):
|
||||
w = [x.lower() for x in _TOKEN.findall(text[:3000])]
|
||||
en = sum(1 for x in w if x in _EN)
|
||||
de = sum(1 for x in w if x in _DE)
|
||||
return 'en' if en > de else 'de'
|
||||
|
||||
|
||||
def names(text):
|
||||
"""Capitalised words not at a sentence's start (in German: names and nouns; in English:
|
||||
names), as a set of lowercased forms."""
|
||||
out = set()
|
||||
for s in sentences(text) or [text or '']:
|
||||
toks = _TOKEN.findall(s)
|
||||
for t in toks[1:]:
|
||||
if t[0].isupper() and t not in _NOT_NAMES and len(t) > 2:
|
||||
out.add(t.lower().rstrip('s’\''))
|
||||
return out
|
||||
|
||||
|
||||
def related(stories, min_weight=17.0, per_story=3):
|
||||
"""stories: [(id, language, set of names)] of the same days. Pairs of stories in different
|
||||
languages sharing rare names: {id: [(other id, weight)]}, weight = the shared names' idf."""
|
||||
df = Counter()
|
||||
for _, _, ns in stories:
|
||||
df.update(ns)
|
||||
n = len(stories)
|
||||
idf = {w: math.log((n + 1) / (d + 1)) for w, d in df.items()}
|
||||
out = defaultdict(list)
|
||||
for a in range(n):
|
||||
ia, la, na = stories[a]
|
||||
for b in range(a + 1, n):
|
||||
ib, lb, nb = stories[b]
|
||||
if la == lb:
|
||||
continue
|
||||
shared = na & nb
|
||||
if len(shared) < 3:
|
||||
continue
|
||||
w = sum(idf[x] for x in shared)
|
||||
if w >= min_weight:
|
||||
out[ia].append((ib, round(w, 1)))
|
||||
out[ib].append((ia, round(w, 1)))
|
||||
return {k: sorted(v, key=lambda x: -x[1])[:per_story] for k, v in out.items()}
|
||||
|
||||
|
||||
# ---- topics ---------------------------------------------------------------------------------------------
|
||||
|
||||
NOT_TOPICS = set('breaking news free live watch video sekunden gebärdensprache podcast newsletter abo online'.split())
|
||||
|
||||
|
||||
TOPIC_MIN = 3
|
||||
|
||||
|
||||
def topics(stories, threshold=0.12, min_size=TOPIC_MIN, max_topics=8, rounds=3):
|
||||
"""stories: [(id, text, sources)] (titles and summaries, the set of source names). Groups of
|
||||
stories about one matter, the largest first, each with a label (the words most of its stories
|
||||
share): [{'label': str, 'ids': [ids]}]. Leader clustering against the groups' centroids, a
|
||||
few rounds (each story to its nearest centroid, then the centroids anew); a group counts
|
||||
when it has min_size stories from at least two sources."""
|
||||
if len(stories) < min_size:
|
||||
return []
|
||||
vecs = _vectors([s[1] for s in stories])
|
||||
order = sorted(range(len(stories)), key=lambda i: -len(vecs[i]))
|
||||
cents = []
|
||||
assign = [-1] * len(stories)
|
||||
for _ in range(rounds):
|
||||
changed = False
|
||||
for i in order:
|
||||
best, sim = -1, threshold
|
||||
for g, c in enumerate(cents):
|
||||
x = _cos(vecs[i], c)
|
||||
if x > sim:
|
||||
best, sim = g, x
|
||||
if best < 0:
|
||||
cents.append(dict(vecs[i]))
|
||||
best = len(cents) - 1
|
||||
if assign[i] != best:
|
||||
assign[i] = best
|
||||
changed = True
|
||||
# centroids anew from their members
|
||||
acc = [Counter() for _ in cents]
|
||||
for i, g in enumerate(assign):
|
||||
for w, v in vecs[i].items():
|
||||
acc[g][w] += v
|
||||
cents = []
|
||||
remap = {}
|
||||
for g, a in enumerate(acc):
|
||||
if a:
|
||||
norm = math.sqrt(sum(v * v for v in a.values())) or 1.0
|
||||
remap[g] = len(cents)
|
||||
cents.append({w: v / norm for w, v in a.items()})
|
||||
assign = [remap.get(g, -1) for g in assign]
|
||||
if not changed:
|
||||
break
|
||||
groups = defaultdict(list)
|
||||
for i, g in enumerate(assign):
|
||||
groups[g].append(i)
|
||||
out = []
|
||||
source_words = {w.lower() for s in stories for src in s[2] for w in _TOKEN.findall(src)}
|
||||
for g in sorted(groups.values(), key=len, reverse=True):
|
||||
if len(g) < min_size or len({src for i in g for src in stories[i][2]}) < 2:
|
||||
continue
|
||||
# Close enough together: the stories' mean likeness to their group's centre.
|
||||
acc = Counter()
|
||||
for i in g:
|
||||
for w, v in vecs[i].items():
|
||||
acc[w] += v
|
||||
norm = math.sqrt(sum(v * v for v in acc.values())) or 1.0
|
||||
centre = {w: v / norm for w, v in acc.items()}
|
||||
if sum(_cos(vecs[i], centre) for i in g) / len(g) < 0.3:
|
||||
continue
|
||||
df = Counter()
|
||||
for i in g:
|
||||
df.update({w for w in words(stories[i][1]) if w not in NOT_TOPICS and w not in source_words})
|
||||
nice = []
|
||||
for w, c in df.most_common(8):
|
||||
if c < max(2, len(g) // 2):
|
||||
break
|
||||
forms = Counter(t for i in g for t in _TOKEN.findall(stories[i][1]) if t.lower() == w)
|
||||
form = forms.most_common(1)[0][0] if forms else w
|
||||
# Nouns and names make a label, not "rückt" or "näher"; "Linke" once, not "Linken" too.
|
||||
if form[:1].isupper() and not any(x.lower()[:5] == form.lower()[:5] for x in nice):
|
||||
nice.append(form)
|
||||
if len(nice) == 3:
|
||||
break
|
||||
if len(nice) < 2: # one shared word is a coincidence, not a topic
|
||||
continue
|
||||
out.append({'label': ' · '.join(nice), 'ids': [stories[i][0] for i in g]})
|
||||
if len(out) >= max_topics:
|
||||
break
|
||||
return out
|
||||
|
|
@ -15,7 +15,6 @@ urlpatterns = [
|
|||
path('briefing/seen/', views.briefing_seen, name='feeds_briefing_seen'),
|
||||
path('news/', views.news, name='feeds_news'),
|
||||
path('stories/<int:pk>/', views.story, name='feeds_story'),
|
||||
path('stories/<int:pk>/overview/', views.story_overview, name='feeds_story_overview'),
|
||||
path('blogs/', views.blogs, name='feeds_blogs'),
|
||||
path('starred/', views.starred, name='feeds_starred'),
|
||||
path('items/read/', views.mark_read, name='feeds_mark_read'),
|
||||
|
|
|
|||
115
feeds/views.py
115
feeds/views.py
|
|
@ -12,11 +12,8 @@ Sources: GET /feeds/state/ sources, unread counts, settings
|
|||
POST /feeds/refresh/ refresh the account's sources now
|
||||
News: GET /feeds/briefing/ what's new since the briefing was last seen
|
||||
POST /feeds/briefing/seen/ seen: everything up to now counts as read
|
||||
GET /feeds/news/?before=&category=&ids= stories, newest first (ids: just these, a topic's);
|
||||
the first page also has the day's topics
|
||||
GET /feeds/news/?before=&category= stories, newest first
|
||||
GET /feeds/stories/<id>/ one story with all its items
|
||||
GET /feeds/stories/<id>/overview/ summary, what only one report says, each source's
|
||||
words, the story in the other language (feeds/textproc.py)
|
||||
Blogs: GET /feeds/blogs/?folder=&source=&unread=1&fav=1&before= (folder= alone: blogs without one;
|
||||
without folder/source the hidden blogs stay out)
|
||||
Items: GET /feeds/items/<id>/ the article (fetches the page if the feed had a teaser)
|
||||
|
|
@ -38,7 +35,6 @@ from xml.sax.saxutils import quoteattr
|
|||
|
||||
import feedparser
|
||||
import requests
|
||||
from django.core.cache import cache
|
||||
from django.db.models import Count
|
||||
from django.http import HttpResponse, JsonResponse
|
||||
from django.utils import timezone
|
||||
|
|
@ -46,7 +42,7 @@ from django.utils.dateparse import parse_datetime
|
|||
from django.views.decorators.csrf import csrf_exempt
|
||||
from django.views.decorators.http import require_http_methods
|
||||
|
||||
from . import fetch, logins, refresh, textproc
|
||||
from . import fetch, logins, refresh
|
||||
from .models import Item, SiteLogin, Source, Story
|
||||
|
||||
PAGE = 40
|
||||
|
|
@ -75,15 +71,6 @@ def _body(request):
|
|||
return None
|
||||
|
||||
|
||||
def _when(value):
|
||||
"""A datetime parameter; None when there's none or it's garbled (an unencoded '+' arrives as
|
||||
a space)."""
|
||||
try:
|
||||
return parse_datetime((value or '').strip().replace(' ', '+'))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _iso(dt):
|
||||
return dt.isoformat() if dt else None
|
||||
|
||||
|
|
@ -492,76 +479,14 @@ def news(request):
|
|||
if (err := _auth(request)):
|
||||
return err
|
||||
qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at')
|
||||
ids = [int(x) for x in request.GET.get('ids', '').split(',') if x.strip().isdigit()]
|
||||
if ids:
|
||||
stories = _stories(request.user, qs.filter(pk__in=ids[:PAGE]))
|
||||
return JsonResponse({'stories': stories, 'more': False})
|
||||
before = _when(request.GET.get('before'))
|
||||
before = parse_datetime(request.GET.get('before') or '')
|
||||
if before:
|
||||
qs = qs.filter(last_at__lt=before)
|
||||
category = request.GET.get('category')
|
||||
if category:
|
||||
qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category)
|
||||
stories = _stories(request.user, qs[:PAGE])
|
||||
out = {'stories': stories, 'more': len(stories) == PAGE}
|
||||
if not before:
|
||||
topics = _news_text(request.user)['topics']
|
||||
if category:
|
||||
# The category's share of each topic.
|
||||
mine = set(qs.values_list('pk', flat=True)[:500])
|
||||
topics = [t for t in ({**t, 'ids': [i for i in t['ids'] if i in mine]} for t in topics)
|
||||
if len(t['ids']) >= textproc.TOPIC_MIN]
|
||||
out['topics'] = topics
|
||||
return JsonResponse(out)
|
||||
|
||||
|
||||
# ---- reading help (feeds/textproc.py)
|
||||
|
||||
FURNITURE_ITEMS = 40 # a source's recent articles that show its furniture
|
||||
TEXT_DAYS = 2 # topics and related stories: of these days
|
||||
TEXT_CACHE = 6 * 3600
|
||||
|
||||
|
||||
def _furniture(source):
|
||||
key = f'feeds:furniture:{source.id}'
|
||||
keys = cache.get(key)
|
||||
if keys is None:
|
||||
texts = [textproc.plain(c) for c in Item.objects.filter(source=source, gone=False)
|
||||
.order_by('-published').values_list('content', flat=True)[:FURNITURE_ITEMS]]
|
||||
keys = textproc.furniture(texts)
|
||||
cache.set(key, keys, TEXT_CACHE)
|
||||
return keys
|
||||
|
||||
|
||||
def _clean_text(i):
|
||||
"""An item's text without its source's furniture, plain."""
|
||||
return textproc.without_furniture(textproc.plain(i.content) or i.summary, _furniture(i.source))
|
||||
|
||||
|
||||
def _news_text(user):
|
||||
"""The news of the last days, looked at as a whole: topics [{label, ids}] and related
|
||||
stories in the other language {story id: [ids]}. Cached until a story changes."""
|
||||
since = timezone.now() - timedelta(days=TEXT_DAYS)
|
||||
qs = Story.objects.filter(user=user, last_at__gte=since, items__gone=False).distinct()
|
||||
latest = qs.order_by('-last_at').values_list('last_at', flat=True).first()
|
||||
key = f'feeds:newstext:{user.id}:{qs.count()}:{latest.timestamp() if latest else 0}'
|
||||
got = cache.get(key)
|
||||
if got is not None:
|
||||
return got
|
||||
items = {}
|
||||
for i in Item.objects.filter(story__in=qs, gone=False).select_related('source'):
|
||||
items.setdefault(i.story_id, []).append(i)
|
||||
tl, sl = [], []
|
||||
for sid, its in items.items():
|
||||
tl.append((sid, ' '.join(f'{i.title}. {i.summary}' for i in its), {i.source.title for i in its}))
|
||||
text = ' '.join(f'{i.title}. {_clean_text(i)[:4000]}' for i in its)
|
||||
sl.append((sid, textproc.language(text), textproc.names(text)))
|
||||
got = {
|
||||
'topics': textproc.topics(tl),
|
||||
'related': {sid: [b for b, _ in lst] for sid, lst in textproc.related(sl).items()},
|
||||
}
|
||||
cache.set(key, got, TEXT_CACHE)
|
||||
return got
|
||||
return JsonResponse({'stories': stories, 'more': len(stories) == PAGE})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
|
|
@ -575,32 +500,6 @@ def story(request, pk):
|
|||
return JsonResponse({'story': found[0]})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
@require_http_methods(['GET'])
|
||||
def story_overview(request, pk):
|
||||
"""What a story's reports say together: a summary from all of them, what only one says,
|
||||
each source's own words, and the same story in the other language."""
|
||||
if (err := _auth(request)):
|
||||
return err
|
||||
s = Story.objects.filter(user=request.user, pk=pk).first()
|
||||
if s is None:
|
||||
return JsonResponse({'error': 'not found'}, status=404)
|
||||
its = list(Item.objects.filter(story=s, gone=False).select_related('source').order_by('published'))
|
||||
key = f'feeds:overview:{s.id}:{len(its)}:{sum(len(i.content) for i in its)}'
|
||||
out = cache.get(key)
|
||||
if out is None:
|
||||
docs = [(i.source.title, _clean_text(i)) for i in its]
|
||||
out = {}
|
||||
if len({src for src, _ in docs}) >= 2:
|
||||
out['summary'] = [{'source': src, 'text': t} for src, t in textproc.summarize(docs)]
|
||||
out['novel'] = textproc.novel(docs)
|
||||
out['words'] = textproc.word_choice(docs)
|
||||
cache.set(key, out, TEXT_CACHE)
|
||||
rel = _news_text(request.user)['related'].get(s.id, [])
|
||||
related = _stories(request.user, Story.objects.filter(user=request.user, pk__in=rel)) if rel else []
|
||||
return JsonResponse({**out, 'related': related})
|
||||
|
||||
|
||||
# ---- blogs, items
|
||||
|
||||
@csrf_exempt
|
||||
|
|
@ -623,7 +522,7 @@ def blogs(request):
|
|||
unread_only = request.GET.get('unread') == '1'
|
||||
if unread_only:
|
||||
qs = qs.filter(read_at=None)
|
||||
before = _when(request.GET.get('before'))
|
||||
before = parse_datetime(request.GET.get('before') or '')
|
||||
if before:
|
||||
qs = qs.filter(published__lt=before)
|
||||
# Unread: all of them at once (the page sorts them into days, favorites, rare blogs).
|
||||
|
|
@ -651,9 +550,7 @@ def item(request, pk):
|
|||
return JsonResponse({'error': 'not found'}, status=404)
|
||||
if refresh.needs_article(i, i.source):
|
||||
refresh.load_article(i)
|
||||
d = _item_data(i, content=True)
|
||||
d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source))
|
||||
return JsonResponse({'item': d})
|
||||
return JsonResponse({'item': _item_data(i, content=True)})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue