diff --git a/docs/podcasts.md b/docs/podcasts.md index a41b54e..10cb7b3 100644 --- a/docs/podcasts.md +++ b/docs/podcasts.md @@ -59,8 +59,38 @@ first), + (Merkliste), ★. Filters stay: Favoriten, Verlauf, Angefangen, Geh Tabs: **Wiedergabeliste · Merkliste · Strom · Abos**. -Browsing within the subscriptions (themes, people, similar episodes, search) — later, to be -thought through. +## Favourite subscriptions + +★ on a subscription (`PodcastFeed.favorite`): a mark and a filter (Abos: only the favourites; +episode lists: `filter=fave_feeds`), whatever its level — it puts nothing on the Merkliste by +itself. Started with the Stamm subscriptions marked. + +## Stöbern (decided 2026-10-19) + +Within the subscriptions: +- **Search** in titles and shownotes: `GET /podcasts/discover/search/?q=…&stream=1` (all + words, the last as a prefix; titles weigh more). SQLite FTS5 index, contentless + (`podcasts/discover.py`), filled after every refresh; `manage.py index_episodes` fills it. +- **Similar episodes** from the other podcasts: `GET /podcasts/discover/similar//`. +- **People** (who talks, who gives a talk), from episode titles only (`podcasts/people.py`): + `GET /podcasts/discover/people/` → `[{name, episodes, feeds, score}]`, the ones in the most + listened podcasts first; `GET /podcasts/discover/person/?name=` → their episodes. +- **Topics** of the Strom's last days, chips as for the news: `GET /podcasts/discover/topics/?days=7` + → `[{label, ids}]`. +- **Forgotten**: `GET /podcasts/discover/forgotten/` → `{loved: [...], rare: [...]}` — unheard + episodes older than 60 days of the podcasts listened to most, and of the rare ones; the same + all day. + +One controlled step out (Apple's episode search and fyyd; no key): +- **People elsewhere**: `GET /podcasts/discover/elsewhere/?name=` → episodes naming the person + in podcasts the user doesn't have (`{source, podcast, feed_url, title, audio_url, guid, + pub_date, duration_seconds, artwork_url, link}`). `POST /podcasts/discover/take/` with such an + item puts the episode on the Merkliste; its podcast becomes an *Auf Abruf* subscription. +- **Recommendations**: `GET /podcasts/discover/recommend/` → podcasts like the most listened ones + (fyyd), recommended for several first: `{title, feed_url, artwork_url, description, link, + because: [titles], count}`. Subscribing: the usual feeds/add/ (then *Auf Abruf* or Strom). + +Show search stays on Apple (best coverage). ## Moving over diff --git a/podcasts/discover.py b/podcasts/discover.py new file mode 100644 index 0000000..e526f23 --- /dev/null +++ b/podcasts/discover.py @@ -0,0 +1,529 @@ +"""Stöbern (docs/podcasts.md): finding episodes in the user's own podcasts — search in titles +and shownotes, similar episodes, people, topics, forgotten ones — and one controlled step out: +where people from them are guests elsewhere (Apple's episode search, fyyd), and podcasts like +the most listened ones (fyyd's recommendations). An episode from outside goes onto the +Merkliste; its podcast becomes an "Auf Abruf" subscription. + +The full-text index is SQLite FTS5, contentless (only the index, no second copy of 100 MB of +shownotes): podcasts_episode_fts, rowid = the episode's id, filled by index_new() after every +refresh (new episodes have higher ids). Episodes deleted since stay in the index; every query +joins the episodes, so they never show. +""" +import html +import logging +import math +import random +import re +import unicodedata +import urllib.parse +from collections import Counter, defaultdict +from concurrent.futures import ThreadPoolExecutor +from datetime import timedelta + +import requests +from django.core.cache import cache +from django.db import connection +from django.db.models import Q +from django.http import JsonResponse +from django.utils import timezone +from django.utils.dateparse import parse_datetime +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods + +from . import people as people_mod +from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastSettings + +log = logging.getLogger(__name__) + +FTS = 'podcasts_episode_fts' +VOCAB = 'podcasts_episode_fts_vocab' +BODY_CHARS = 6000 # of the shownotes, per episode +INDEX_BATCH = 2000 +SIMILAR_TERMS = 10 +HTTP_TIMEOUT = 6 +USER_AGENT = 'diora podcast discovery' + +CREATE_SQL = [ + f"CREATE VIRTUAL TABLE IF NOT EXISTS {FTS} USING fts5(title, body, content='', contentless_delete=1, " + f"tokenize='unicode61 remove_diacritics 2')", + f"CREATE VIRTUAL TABLE IF NOT EXISTS {VOCAB} USING fts5vocab({FTS}, 'row')", +] +DROP_SQL = [f'DROP TABLE IF EXISTS {VOCAB}', f'DROP TABLE IF EXISTS {FTS}'] + + +def _plain(text): + text = re.sub(r'<[^>]+>', ' ', text or '') + return re.sub(r'\s+', ' ', html.unescape(text)).strip() + + +def _fold(word): + """A word as the index keeps it: lower case, without diacritics.""" + w = unicodedata.normalize('NFKD', word.lower()) + return ''.join(c for c in w if not unicodedata.combining(c)) + + +_WORD = re.compile(r'\w+', re.UNICODE) + + +def _words(text): + return [_fold(w) for w in _WORD.findall(text or '') if not w.isdigit()] + + +def _quoted(term): + return '"' + term.replace('"', '""') + '"' + + +# ---- the index ------------------------------------------------------------------------------------ + +def index_new(limit=None): + """Index the episodes newer than the newest one indexed. Returns how many.""" + with connection.cursor() as cur: + cur.execute(f'SELECT max(rowid) FROM {FTS}') + last = cur.fetchone()[0] or 0 + done = 0 + while True: + rows = list(PodcastEpisode.objects.filter(id__gt=last).order_by('id') + .values_list('id', 'title', 'description')[:INDEX_BATCH]) + if not rows: + return done + with connection.cursor() as cur: + cur.executemany(f'INSERT OR REPLACE INTO {FTS}(rowid, title, body) VALUES (%s, %s, %s)', + [(i, t or '', _plain(d)[:BODY_CHARS]) for i, t, d in rows]) + done += len(rows) + last = rows[-1][0] + if limit and done >= limit: + return done + + +def _user_episode_sql(user, stream=False, other_than_feed=None): + """SQL and params restricting fts rowids to the user's podcast episodes.""" + sql = (f' JOIN podcasts_podcastepisode e ON e.id = {FTS}.rowid' + f' JOIN podcasts_podcastfeed pf ON pf.id = e.feed_id' + f' WHERE pf.user_id = %s AND pf.kind = %s') + params = [user.id, PodcastFeed.PODCAST] + if stream: + sql += ' AND pf.new_episodes != %s' + params.append(PodcastFeed.NEW_IGNORE) + if other_than_feed is not None: + sql += ' AND e.feed_id != %s' + params.append(other_than_feed) + return sql, params + + +def search_ids(user, q, stream=False, limit=50, offset=0): + """Episode ids of the user's podcasts matching all words of q (the last one as a prefix, + while typing), titles weighing three times the shownotes, best first.""" + words = _words(q)[:12] + if not words: + return [] + match = ' '.join(_quoted(w) for w in words[:-1]) + (' ' if len(words) > 1 else '') + _quoted(words[-1]) + '*' + where, params = _user_episode_sql(user, stream) + with connection.cursor() as cur: + cur.execute(f'SELECT {FTS}.rowid FROM {FTS}{where} AND {FTS} MATCH %s ' + f'ORDER BY bm25({FTS}, 3.0, 1.0) LIMIT %s OFFSET %s', params + [match, limit, offset]) + return [r[0] for r in cur.fetchall()] + + +def similar_ids(user, episode, limit=20): + """Episodes of the user's other podcasts most like `episode`: its rarest telling words + (tf·idf over the index) asked for together, best match first.""" + tf = Counter() + for w in _words(episode.title): + tf[w] += 3 + for w in _words(_plain(episode.description)[:BODY_CHARS]): + tf[w] += 1 + terms = [w for w in tf if len(w) > 2] + if not terms: + return [] + with connection.cursor() as cur: + cur.execute(f'SELECT count(*) FROM podcasts_podcastepisode') + n = cur.fetchone()[0] or 1 + df = {} + for i in range(0, len(terms), 500): + chunk = terms[i:i + 500] + cur.execute(f'SELECT term, doc FROM {VOCAB} WHERE term IN ({",".join(["%s"] * len(chunk))})', chunk) + df.update(dict(cur.fetchall())) + scored = [] + for w, c in tf.items(): + d = df.get(w, 0) + if d < 2 or d > max(n * 0.05, 10): # only here, or everywhere: says nothing + continue + scored.append((c * math.log(n / d), w)) + scored.sort(reverse=True) + best = [w for _, w in scored[:SIMILAR_TERMS]] + if not best: + return [] + where, params = _user_episode_sql(user, other_than_feed=episode.feed_id) + with connection.cursor() as cur: + cur.execute(f'SELECT {FTS}.rowid FROM {FTS}{where} AND {FTS} MATCH %s ' + f'ORDER BY bm25({FTS}, 3.0, 1.0) LIMIT %s', + params + [' OR '.join(_quoted(w) for w in best), limit]) + return [r[0] for r in cur.fetchall()] + + +# ---- helpers --------------------------------------------------------------------------------------- + +def _rows(user, ids): + """The episodes as the episode lists give them, in the order of ids.""" + from .views import _episode_list_data + if not ids: + return [] + eps, _ = _episode_list_data(user, ids=ids, limit=len(ids)) + by_id = {e['id']: e for e in eps} + return [by_id[i] for i in ids if i in by_id] + + +def _heard_by_feed(user): + """{feed_id: episodes heard in the last 12 months} (as heard_365 in state/).""" + from .views import HEARD_MIN_SECONDS + since = timezone.now() - timedelta(days=365) + out = Counter() + for fid in (EpisodeProgress.objects.filter(user=user, updated_at__gte=since) + .filter(Q(played=True) | Q(position_seconds__gt=HEARD_MIN_SECONDS)) + .values_list('episode__feed_id', flat=True)): + out[fid] += 1 + return out + + +def _norm_url(url): + u = (url or '').strip().lower() + u = re.sub(r'^https?://', '', u) + return u.rstrip('/') + + +def _get_json(url, **params): + try: + r = requests.get(url, params=params, timeout=HTTP_TIMEOUT, headers={'User-Agent': USER_AGENT}) + r.raise_for_status() + return r.json() + except Exception as e: # noqa: BLE001 - a service away is no error for the user + log.info('discover: %s: %s', url, e) + return None + + +# ---- people ---------------------------------------------------------------------------------------- + +PEOPLE_TTL = 12 * 3600 + + +def people_of(user): + key = f'podcasts:people:{user.id}' + got = cache.get(key) + if got is None: + rows = list(PodcastEpisode.objects.filter(feed__user=user, feed__kind=PodcastFeed.PODCAST) + .values_list('id', 'feed_id', 'title')) + got = people_mod.people(rows, _heard_by_feed(user))[:200] + cache.set(key, got, PEOPLE_TTL) + return got + + +def person_ids(user, name, limit=200): + return list(PodcastEpisode.objects.filter(feed__user=user, feed__kind=PodcastFeed.PODCAST, title__icontains=name) + .order_by('-pub_date', '-id').values_list('id', flat=True)[:limit]) + + +def elsewhere(user, name): + """Episodes with `name` in podcasts the user doesn't have: Apple's episode search and fyyd's, + those that name the person in their title. [{source, podcast, feed_url, title, audio_url, + guid, pub_date, duration_seconds, artwork_url, link}]""" + key = f'podcasts:elsewhere:{user.id}:' + re.sub(r'\W+', '_', _fold(name)) + got = cache.get(key) + if got is not None: + return got + mine = {_norm_url(u) for u in PodcastFeed.objects.filter(user=user).values_list('rss_url', flat=True)} + mine_titles = {t.casefold() for t in PodcastFeed.objects.filter(user=user).values_list('title', flat=True)} + folded = _fold(name) + out, seen = [], set() + + def take(item): + if folded not in _fold(item['title']) or not item['feed_url'] or not item['audio_url']: + return + if _norm_url(item['feed_url']) in mine or item['podcast'].casefold() in mine_titles: + return + k = (_fold(item['title']), _fold(item['podcast'])) + if k in seen: + return + seen.add(k) + out.append(item) + + apple = _get_json('https://itunes.apple.com/search', term=name, media='podcast', entity='podcastEpisode', + limit=50, country='DE') or {} + for r in apple.get('results', []): + take({'source': 'apple', 'podcast': r.get('collectionName', ''), 'feed_url': r.get('feedUrl', ''), + 'title': r.get('trackName', ''), 'audio_url': r.get('episodeUrl', ''), 'guid': r.get('episodeGuid', ''), + 'pub_date': r.get('releaseDate'), 'duration_seconds': int((r.get('trackTimeMillis') or 0) / 1000), + 'artwork_url': r.get('artworkUrl600') or r.get('artworkUrl160') or '', + 'link': r.get('trackViewUrl', '')}) + fy = _get_json('https://api.fyyd.de/0.2/search/episode', term=f'"{name}"', count=50) or {} + eps = [e for e in (fy.get('data') or []) if isinstance(e, dict) and folded in _fold(e.get('title', ''))] + feeds = {} + for pid in list(dict.fromkeys(e.get('podcast_id') for e in eps))[:12]: + p = _get_json('https://api.fyyd.de/0.2/podcast', podcast_id=pid) or {} + if isinstance(p.get('data'), dict): + feeds[pid] = p['data'] + for e in eps: + p = feeds.get(e.get('podcast_id')) + if not p: + continue + take({'source': 'fyyd', 'podcast': p.get('title', ''), 'feed_url': p.get('xmlURL', ''), + 'title': html.unescape(e.get('title', '')), 'audio_url': e.get('enclosure', ''), 'guid': e.get('guid', ''), + 'pub_date': e.get('pubdate'), 'duration_seconds': int(e.get('duration') or 0), + 'artwork_url': e.get('imgURL') or p.get('imgURL') or '', 'link': e.get('url_fyyd', '')}) + out.sort(key=lambda i: i['pub_date'] or '', reverse=True) + cache.set(key, out, 6 * 3600) + return out + + +# ---- topics, forgotten, recommendations ------------------------------------------------------------ + +def topics_of(user, days=7): + """Topics of the Strom's last days (feeds.textproc.topics, as for the news): + [{'label', 'ids'}], the largest first.""" + from feeds import textproc + since = timezone.now() - timedelta(days=days) + eps = list(PodcastEpisode.objects.filter(feed__user=user, feed__kind=PodcastFeed.PODCAST, pub_date__gte=since) + .exclude(feed__new_episodes=PodcastFeed.NEW_IGNORE) + .values_list('id', 'title', 'description', 'feed__title')) + stories = [(i, f'{t}. {_plain(d)[:400]}', {ft}) for i, t, d, ft in eps] + return textproc.topics(stories, max_topics=10) + + +FORGOTTEN_DAYS = 60 + + +def forgotten(user): + """Unheard older episodes of the podcasts the user listens to most, and of the rare ones — + a handful each, the same all day.""" + from .views import _intervals + rnd = random.Random(f'{user.id}:{timezone.localdate()}') + heard = _heard_by_feed(user) + started = set(EpisodeProgress.objects.filter(user=user).filter(Q(played=True) | Q(position_seconds__gt=0)) + .values_list('episode_id', flat=True)) + before = timezone.now() - timedelta(days=FORGOTTEN_DAYS) + + def pick(feed_ids, n): + ids = [i for i in PodcastEpisode.objects.filter(feed_id__in=feed_ids, pub_date__lt=before) + .values_list('id', flat=True) if i not in started] + return rnd.sample(ids, min(n, len(ids))) + + mine = list(PodcastFeed.objects.filter(user=user, kind=PodcastFeed.PODCAST) + .exclude(new_episodes=PodcastFeed.NEW_IGNORE).values_list('id', flat=True)) + loved = [f for f in mine if heard.get(f, 0) >= 3] + intervals = _intervals(user) + rare = [f for f in mine if (intervals.get(f) or 0) >= 30] + return {'loved': _rows(user, pick(loved, 15)), 'rare': _rows(user, pick(rare, 10))} + + +RECOMMEND_FROM = 10 # the most listened podcasts asked about +RECOMMEND_TTL = 24 * 3600 + + +def _fyyd_id(feed): + got = _get_json('https://api.fyyd.de/0.2/search/podcast', title=feed['title'], count=5) or {} + want = _norm_url(feed['rss_url']) + for p in got.get('data') or []: + if isinstance(p, dict) and (_norm_url(p.get('xmlURL')) == want or (p.get('title') or '').casefold() == feed['title'].casefold()): + return p.get('id') + return None + + +def recommend(user): + """Podcasts like the user's most listened (fyyd's recommendations), the ones recommended for + several of them first; none the user has. [{title, feed_url, artwork_url, description, link, + because: [titles], count}]""" + key = f'podcasts:recommend:{user.id}' + got = cache.get(key) + if got is not None: + return got + heard = _heard_by_feed(user) + feeds = list(PodcastFeed.objects.filter(user=user, kind=PodcastFeed.PODCAST).values('id', 'title', 'rss_url')) + top = sorted((f for f in feeds if heard.get(f['id'])), key=lambda f: -heard[f['id']])[:RECOMMEND_FROM] + mine = {_norm_url(f['rss_url']) for f in feeds} + mine_titles = {f['title'].casefold() for f in feeds} + + def ask(feed): + pid = _fyyd_id(feed) + if not pid: + return feed, [] + r = _get_json('https://api.fyyd.de/0.2/podcast/recommend', podcast_id=pid, count=12) or {} + return feed, [p for p in r.get('data') or [] if isinstance(p, dict)] + + found = {} + with ThreadPoolExecutor(max_workers=5) as pool: + for feed, recs in pool.map(ask, top): + for p in recs: + url = p.get('xmlURL') or '' + if not url or _norm_url(url) in mine or (p.get('title') or '').casefold() in mine_titles: + continue + k = _norm_url(url) + item = found.setdefault(k, {'title': p.get('title', ''), 'feed_url': url, 'artwork_url': p.get('imgURL') or '', + 'description': _plain(p.get('description'))[:300], + 'link': p.get('htmlURL') or '', 'because': [], 'count': 0}) + if feed['title'] not in item['because']: + item['because'].append(feed['title']) + item['count'] += 1 + out = sorted(found.values(), key=lambda i: (-i['count'], i['title'].casefold()))[:30] + cache.set(key, out, RECOMMEND_TTL) + return out + + +# ---- an episode from outside ------------------------------------------------------------------------- + +def take(user, item): + """An episode from elsewhere onto the Merkliste: its podcast becomes an "Auf Abruf" + subscription (or is one already), the episode is found in it — or, when the feed no longer + lists it, made from what the search said. Returns (episode, created_feed).""" + from .views import _later_add, _refresh_feed + url = (item.get('feed_url') or '').strip() + feed = PodcastFeed.objects.filter(user=user, rss_url=url).first() + created = False + if feed is None: + feed = PodcastFeed.objects.create( + user=user, rss_url=url, title=(item.get('podcast') or url)[:300], artwork_url=(item.get('artwork_url') or '')[:1000], + **{**PodcastSettings.new_feed_defaults(user), 'new_episodes': PodcastFeed.NEW_IGNORE}) + created = True + try: + _refresh_feed(feed) + except Exception as e: # noqa: BLE001 - the episode is made from the search below + log.info('discover: %s: %s', url, e) + eps = feed.episodes.all() + ep = None + if item.get('guid'): + ep = eps.filter(guid=item['guid']).first() + if ep is None and item.get('audio_url'): + ep = eps.filter(audio_url=item['audio_url']).first() + if ep is None and item.get('title'): + ep = eps.filter(title__iexact=item['title']).first() + if ep is None: + when = parse_datetime(item.get('pub_date') or '') if item.get('pub_date') else None + ep = PodcastEpisode.objects.create( + feed=feed, guid=(item.get('guid') or item.get('audio_url') or item.get('title'))[:1000], + title=(item.get('title') or '')[:500], audio_url=item.get('audio_url', '')[:1000], + duration_seconds=int(item.get('duration_seconds') or 0), pub_date=when, + artwork_url=(item.get('artwork_url') or '')[:1000]) + _later_add(user, [ep.id]) + try: + index_new() + except Exception: # noqa: BLE001 + pass + return ep, created + + +# ---- views --------------------------------------------------------------------------------------------- + +def _auth(request): + if not request.user.is_authenticated: + return JsonResponse({'error': 'authentication required'}, status=401) + return None + + +def _int(request, name, default, low, high): + try: + return min(max(int(request.GET.get(name, default)), low), high) + except ValueError: + return default + + +@require_http_methods(['GET']) +def search_view(request): + """?q=…&stream=1&offset=&limit=: episodes whose title or shownotes have every word.""" + err = _auth(request) + if err: + return err + ids = search_ids(request.user, request.GET.get('q', ''), stream=request.GET.get('stream') == '1', + limit=_int(request, 'limit', 50, 1, 200), offset=_int(request, 'offset', 0, 0, 10 ** 6)) + return JsonResponse({'episodes': _rows(request.user, ids)}) + + +@require_http_methods(['GET']) +def similar_view(request, pk): + err = _auth(request) + if err: + return err + ep = PodcastEpisode.objects.filter(pk=pk, feed__user=request.user).first() + if ep is None: + return JsonResponse({'error': 'not found'}, status=404) + return JsonResponse({'episodes': _rows(request.user, similar_ids(request.user, ep, _int(request, 'limit', 20, 1, 50)))}) + + +@require_http_methods(['GET']) +def people_view(request): + """The people in the user's podcasts: [{name, episodes, feeds, score}], the most relevant first.""" + err = _auth(request) + if err: + return err + return JsonResponse({'people': people_of(request.user)[:_int(request, 'limit', 80, 1, 200)]}) + + +@require_http_methods(['GET']) +def person_view(request): + """?name=: the person's episodes in the user's podcasts, newest first.""" + err = _auth(request) + if err: + return err + name = request.GET.get('name', '').strip() + if len(name) < 3: + return JsonResponse({'error': 'name required'}, status=400) + return JsonResponse({'name': name, 'episodes': _rows(request.user, person_ids(request.user, name))}) + + +@require_http_methods(['GET']) +def elsewhere_view(request): + """?name=: episodes with the person in podcasts the user doesn't have.""" + err = _auth(request) + if err: + return err + name = request.GET.get('name', '').strip() + if len(name) < 3: + return JsonResponse({'error': 'name required'}, status=400) + return JsonResponse({'name': name, 'episodes': elsewhere(request.user, name)}) + + +@require_http_methods(['GET']) +def topics_view(request): + """?days=7: topics of the Strom's last days: [{label, ids}], plus their episodes.""" + err = _auth(request) + if err: + return err + days = _int(request, 'days', 7, 1, 30) + key = f'podcasts:topics:{request.user.id}:{days}' + got = cache.get(key) + if got is None: + got = topics_of(request.user, days) + cache.set(key, got, 3600) + return JsonResponse({'topics': got}) + + +@require_http_methods(['GET']) +def forgotten_view(request): + err = _auth(request) + if err: + return err + return JsonResponse(forgotten(request.user)) + + +@require_http_methods(['GET']) +def recommend_view(request): + err = _auth(request) + if err: + return err + return JsonResponse({'podcasts': recommend(request.user)}) + + +@csrf_exempt +@require_http_methods(['POST']) +def take_view(request): + """{podcast, feed_url, title, audio_url, guid, pub_date, duration_seconds, artwork_url} (an + item of elsewhere/): onto the Merkliste; returns {episode_id, feed_id, subscribed}.""" + import json + err = _auth(request) + if err: + return err + try: + item = json.loads(request.body or b'{}') + except (json.JSONDecodeError, ValueError): + return JsonResponse({'error': 'invalid JSON'}, status=400) + if not isinstance(item, dict) or not re.match(r'^https?://', item.get('feed_url') or '') or not item.get('audio_url'): + return JsonResponse({'error': 'feed_url and audio_url required'}, status=400) + ep, created = take(request.user, item) + return JsonResponse({'ok': True, 'episode_id': ep.id, 'feed_id': ep.feed_id, 'subscribed': created}) diff --git a/podcasts/management/commands/index_episodes.py b/podcasts/management/commands/index_episodes.py new file mode 100644 index 0000000..228dc7a --- /dev/null +++ b/podcasts/management/commands/index_episodes.py @@ -0,0 +1,10 @@ +from django.core.management.base import BaseCommand + +from podcasts.discover import index_new + + +class Command(BaseCommand): + help = 'Put the episodes not yet in the full-text index into it (podcasts/discover.py)' + + def handle(self, *args, **options): + self.stdout.write(f'{index_new()} episodes indexed') diff --git a/podcasts/management/commands/refresh_feeds.py b/podcasts/management/commands/refresh_feeds.py index 8049cad..4ec9dfd 100644 --- a/podcasts/management/commands/refresh_feeds.py +++ b/podcasts/management/commands/refresh_feeds.py @@ -2,6 +2,7 @@ from django.core.management.base import BaseCommand from django.utils import timezone from podcasts.models import PodcastFeed +from podcasts.discover import index_new from podcasts.views import _in_backoff_q, _refresh_feeds, refresh_due @@ -40,4 +41,6 @@ class Command(BaseCommand): # Parallel downloads, one per distinct feed URL (see _refresh_feeds). refreshed, failed, new = _refresh_feeds(qs, full=options['full']) - self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes') + # New episodes into the full-text index (Stöbern, podcasts/discover.py). + indexed = index_new() + self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes, {indexed} indexed') diff --git a/podcasts/migrations/0023_episode_fts.py b/podcasts/migrations/0023_episode_fts.py new file mode 100644 index 0000000..41ad84d --- /dev/null +++ b/podcasts/migrations/0023_episode_fts.py @@ -0,0 +1,27 @@ +# The full-text index for Stöbern (podcasts/discover.py): SQLite FTS5, contentless. +from django.db import migrations + +from podcasts.discover import CREATE_SQL, DROP_SQL + + +def create(apps, schema_editor): + if schema_editor.connection.vendor != 'sqlite': + return + for sql in CREATE_SQL: + schema_editor.execute(sql) + + +def drop(apps, schema_editor): + if schema_editor.connection.vendor != 'sqlite': + return + for sql in DROP_SQL: + schema_editor.execute(sql) + + +class Migration(migrations.Migration): + + dependencies = [ + ('podcasts', '0022_lists_and_levels'), + ] + + operations = [migrations.RunPython(create, drop)] diff --git a/podcasts/migrations/0024_feed_favorite.py b/podcasts/migrations/0024_feed_favorite.py new file mode 100644 index 0000000..f882e87 --- /dev/null +++ b/podcasts/migrations/0024_feed_favorite.py @@ -0,0 +1,18 @@ +# Generated by Django 6.1 on 2026-10-19 16:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('podcasts', '0023_episode_fts'), + ] + + operations = [ + migrations.AddField( + model_name='podcastfeed', + name='favorite', + field=models.BooleanField(default=False), + ), + ] diff --git a/podcasts/models.py b/podcasts/models.py index d71ed96..ddb7700 100644 --- a/podcasts/models.py +++ b/podcasts/models.py @@ -67,6 +67,9 @@ class PodcastFeed(models.Model): # New episodes: what happens to them, and which ones count at all. new_episodes = models.CharField(max_length=10, choices=NEW_EPISODE_ACTIONS, default=NEW_INBOX) refresh_interval = models.CharField(max_length=10, choices=REFRESH_INTERVALS, default='', blank=True) + # A favourite subscription (★ in the Abos): only a mark and a filter, whatever its level — + # it puts nothing on the Merkliste by itself (docs/podcasts.md). + favorite = models.BooleanField(default=False) # Comma-separated words, case-insensitive, matched against the title. include_words = models.CharField(max_length=500, blank=True) exclude_words = models.CharField(max_length=500, blank=True) diff --git a/podcasts/people.py b/podcasts/people.py new file mode 100644 index 0000000..a7f0096 --- /dev/null +++ b/podcasts/people.py @@ -0,0 +1,122 @@ +"""People in the user's podcasts (docs/podcasts.md, Stöbern): who is talked with, who gives a +talk — found in episode titles and descriptions, no service asked. + +A name is two or three capitalised words ("Peter Decker", "Ulrike Eifler", "Jean-Luc Mélenchon"), +possibly with a particle ("Wolfgang von Goethe"). Two kinds of evidence: +- strong: right after a cue that announces a person — "mit", "w/", "with", "feat.", "Gast:", + "im Gespräch mit", "interview with", "von" in a talk's title, " - Name - " between dashes; +- weak: anywhere else in a title or description. +A name counts when it has strong evidence in at least STRONG_MIN episodes, or turns up in at least +FEEDS_MIN podcasts (a person known across the user's world). Words that are no names (places, +institutions, weekdays, sentence starts) are left out by a list and by the rule that a name's +first word must not be one of them. +""" +import html +import re +from collections import Counter, defaultdict + +_W = r"[A-ZÄÖÜÀ-Ý][a-zäöüßà-ÿ]+(?:-[A-ZÄÖÜÀ-Ý][a-zäöüßà-ÿ]+)?" +_PART = r"(?:von|van|der|de|da|di|du|la|le|zu|ten|del|bin|al)" +_NAME = rf"{_W}(?:\s+{_PART})?(?:\s+{_W}){{1,2}}" +NAME_RE = re.compile(rf"(?]+>', ' ', text or '') + return re.sub(r'\s+', ' ', html.unescape(text)).strip() + + +def _clean(name): + words = name.split() + # "Mit Peter Decker" etc.: drop leading words that are no name. + while words and words[0] in NOT_NAME_WORDS: + words = words[1:] + while words and words[-1] in NOT_NAME_WORDS: + words = words[:-1] + caps = [w for w in words if w[0].isupper()] + if len(caps) < 2 or any(w in NOT_NAME_WORDS for w in caps): + return None + if any(len(w) > 1 and w.isupper() for w in caps): # "WAS TUN", "ARTE Reportage": no person + return None + return ' '.join(words) + + +def mentions(title): + """(strong, weak): the names in one episode's title.""" + strong, weak = set(), set() + for m in CUE_RE.finditer(title or ''): + n = _clean(m.group(1) or m.group(2)) + if n: + strong.add(n) + for m in NAME_RE.finditer(title or ''): + n = _clean(m.group(1)) + if n: + weak.add(n) + return strong, weak - strong + + +def people(episodes, weights=None): + """episodes: [(episode_id, feed_id, title)]; weights: {feed_id: how much the user listens} + (heard_365). The people found, the most relevant first: [{'name', 'episodes', 'feeds'}]. + + Only titles: descriptions carry credits and boilerplate. First names are learnt from the + strong mentions ("mit Peter Decker": Peter); a name found elsewhere in a title counts only + with such a first name.""" + found = [(ep_id, feed_id, *mentions(title)) for ep_id, feed_id, title in episodes] + surnames = defaultdict(set) + for _, _, strong, _ in found: + for n in strong: + w = n.split() + surnames[w[0]].add(w[-1]) + # A first name: one that comes with at least two different surnames in strong mentions. + first_names = {f for f, last in surnames.items() if len(last) >= 2 and f not in NOT_NAME_WORDS} + eps = defaultdict(set) + feeds = defaultdict(set) + strong_n = Counter() + for ep_id, feed_id, strong, weak in found: + for n in strong: + strong_n[n] += 1 + for n in {n for n in strong | weak if n.split()[0] in first_names}: + eps[n].add(ep_id) + feeds[n].add(feed_id) + weights = weights or {} + out = [] + for n, ids in eps.items(): + if strong_n[n] >= STRONG_MIN or len(feeds[n]) >= FEEDS_MIN: + score = len(ids) + 5 * sum(min(weights.get(f, 0), 50) for f in feeds[n]) ** 0.5 + out.append({'name': n, 'episodes': len(ids), 'feeds': len(feeds[n]), 'score': round(score, 1)}) + out.sort(key=lambda p: (-p['score'], p['name'])) + return out diff --git a/podcasts/test_discover.py b/podcasts/test_discover.py new file mode 100644 index 0000000..0419eaa --- /dev/null +++ b/podcasts/test_discover.py @@ -0,0 +1,149 @@ +"""Stöbern (podcasts/discover.py): the full-text index, similar episodes, people, an episode +from outside onto the Merkliste.""" +import json +from datetime import datetime, timedelta, timezone +from unittest.mock import patch + +from django.contrib.auth.models import User +from django.core.cache import cache +from django.test import TestCase + +from . import discover, people +from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastLater + + +class PeopleTests(TestCase): + def test_names_from_titles(self): + eps = [ + (1, 1, 'Episode 651: Die Brandmauer mit Peter Decker'), + (2, 1, 'Episode 568: Trumps Zollkrieg mit Peter Decker'), + (3, 2, 'Imperialismus heute - Peter Decker - 2006'), + (4, 3, 'Gespräch mit Peter Schadt über Arbeit'), + (5, 3, 'Ulrike Eifler im Gespräch mit Peter Thiel'), + (6, 4, 'Was tun? Mit Ulrike Herrmann'), + (9, 4, 'Streik! Diskussion mit Ulrike Eifler'), + (7, 5, 'Gespräch mit Ulrike Herrmann über das Ende des Kapitalismus'), + (8, 6, 'Live aus Berlin: Die Linke und Friedrich Merz'), + ] + found = {p['name']: p for p in people.people(eps)} + self.assertEqual(found['Peter Decker']['episodes'], 3) + self.assertEqual(found['Peter Decker']['feeds'], 2) + self.assertIn('Ulrike Herrmann', found) + self.assertNotIn('Die Brandmauer', found) + self.assertNotIn('Die Linke', found) + + +class IndexTests(TestCase): + def setUp(self): + cache.clear() + self.user = User.objects.create_user(username='finder', password='pw12345678') + self.client.force_login(self.user) + self.other = User.objects.create_user(username='other', password='pw12345678') + self.f1 = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/1', title='Eins') + self.f2 = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/2', title='Zwei') + self.f3 = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/3', title='Drei', + new_episodes=PodcastFeed.NEW_IGNORE) + theirs = PodcastFeed.objects.create(user=self.other, rss_url='https://example.org/1', title='Eins') + + def ep(feed, guid, title, desc=''): + return PodcastEpisode.objects.create(feed=feed, guid=guid, title=title, description=desc, + audio_url=f'https://example.org/{guid}.mp3', + pub_date=datetime(2026, 1, 1, tzinfo=timezone.utc)) + self.a = ep(self.f1, 'a', 'Über die Ökonomie der Plattformen', '

Mit Gästen über Plattformkapitalismus und Arbeit.

') + self.b = ep(self.f2, 'b', 'Plattformkapitalismus erklärt', 'Wie Plattformen Arbeit organisieren.') + self.c = ep(self.f2, 'c', 'Gartenarbeit im Herbst', 'Tomaten, Kürbisse und Laub.') + self.d = ep(self.f3, 'd', 'Noch mehr Plattformkapitalismus', 'Auf Abruf.') + for i in range(30): # words everywhere say nothing + ep(self.f1, f'x{i}', f'Folge {i}', 'Arbeit und Gesellschaft') + ep(theirs, 't', 'Plattformkapitalismus bei jemand anderem', '') + self.assertGreaterEqual(discover.index_new(), 35) + self.assertEqual(discover.index_new(), 0, 'nothing twice') + + def test_search_titles_shownotes_prefix_folding(self): + r = self.client.get('/podcasts/discover/search/?q=plattformkap').json() + self.assertEqual({e['id'] for e in r['episodes']}, {self.a.id, self.b.id, self.d.id}) + self.assertEqual(r['episodes'][0]['id'] in (self.b.id, self.d.id), True, 'a title match first') + r = self.client.get('/podcasts/discover/search/?q=okonomie').json() + self.assertEqual([e['id'] for e in r['episodes']], [self.a.id]) + r = self.client.get('/podcasts/discover/search/?q=plattformkapitalismus&stream=1').json() + self.assertNotIn(self.d.id, [e['id'] for e in r['episodes']], 'Auf Abruf is not in the Strom') + + def test_similar_from_other_podcasts(self): + r = self.client.get(f'/podcasts/discover/similar/{self.b.id}/').json() + ids = [e['id'] for e in r['episodes']] + self.assertIn(self.a.id, ids) + self.assertNotIn(self.c.id, ids, 'same podcast, and about gardens') + self.assertEqual(self.client.get('/podcasts/discover/similar/999999/').status_code, 404) + + def test_person_episodes(self): + PodcastEpisode.objects.create(feed=self.f1, guid='p', title='Arbeit mit Peter Decker', audio_url='https://example.org/p.mp3') + r = self.client.get('/podcasts/discover/person/?name=Peter Decker').json() + self.assertEqual([e['title'] for e in r['episodes']], ['Arbeit mit Peter Decker']) + + def test_forgotten(self): + old = PodcastEpisode.objects.create(feed=self.f2, guid='old', title='Alt', audio_url='https://example.org/old.mp3', + pub_date=datetime.now(timezone.utc) - timedelta(days=200)) + for e in (self.b, self.c, self.a): + EpisodeProgress.objects.create(user=self.user, episode=e, played=True) + PodcastEpisode.objects.create(feed=self.f2, guid='h1', title='H1', audio_url='https://example.org/h1.mp3') + heard = PodcastEpisode.objects.create(feed=self.f2, guid='h2', title='H2', audio_url='https://example.org/h2.mp3') + EpisodeProgress.objects.create(user=self.user, episode=heard, played=True) + r = self.client.get('/podcasts/discover/forgotten/').json() + self.assertEqual([e['id'] for e in r['loved']], [old.id]) + + +class OutsideTests(TestCase): + def setUp(self): + cache.clear() + self.user = User.objects.create_user(username='out', password='pw12345678') + self.client.force_login(self.user) + PodcastFeed.objects.create(user=self.user, rss_url='https://mine.example/feed', title='Mein Podcast') + + def test_elsewhere_keeps_only_the_person_and_unknown_podcasts(self): + apple = {'results': [ + {'collectionName': 'Fremd', 'feedUrl': 'https://other.example/feed', 'trackName': 'Gespräch mit Peter Decker', + 'episodeUrl': 'https://other.example/1.mp3', 'episodeGuid': 'g1', 'releaseDate': '2026-01-02T00:00:00Z', + 'trackTimeMillis': 60000}, + {'collectionName': 'Mein Podcast', 'feedUrl': 'https://mine.example/feed', 'trackName': 'Mit Peter Decker', + 'episodeUrl': 'https://mine.example/1.mp3'}, + {'collectionName': 'Fremd', 'feedUrl': 'https://other.example/feed', 'trackName': 'Peter Pan', + 'episodeUrl': 'https://other.example/2.mp3'}, + ]} + + def fake(url, **params): + return apple if 'itunes' in url else {'data': []} + with patch.object(discover, '_get_json', fake): + r = self.client.get('/podcasts/discover/elsewhere/?name=Peter Decker').json() + self.assertEqual([e['title'] for e in r['episodes']], ['Gespräch mit Peter Decker']) + self.assertEqual(r['episodes'][0]['duration_seconds'], 60) + + def test_take_subscribes_auf_abruf_and_merklists(self): + item = {'podcast': 'Fremd', 'feed_url': 'https://other.example/feed', 'title': 'Gespräch mit Peter Decker', + 'audio_url': 'https://other.example/1.mp3', 'guid': 'g1', 'pub_date': '2026-01-02T00:00:00Z', + 'duration_seconds': 60} + with patch('podcasts.views._refresh_feed', side_effect=OSError('offline')): + r = self.client.post('/podcasts/discover/take/', json.dumps(item), content_type='application/json').json() + self.assertTrue(r['subscribed']) + feed = PodcastFeed.objects.get(pk=r['feed_id']) + self.assertEqual(feed.new_episodes, PodcastFeed.NEW_IGNORE) + self.assertTrue(PodcastLater.objects.filter(user=self.user, episode_id=r['episode_id']).exists()) + with patch('podcasts.views._refresh_feed', side_effect=OSError('offline')): + r2 = self.client.post('/podcasts/discover/take/', json.dumps(item), content_type='application/json').json() + self.assertEqual((r2['episode_id'], r2['subscribed']), (r['episode_id'], False)) + + def test_recommend_counts_and_leaves_out_mine(self): + EpisodeProgress.objects.create( + user=self.user, played=True, + episode=PodcastEpisode.objects.create(feed=PodcastFeed.objects.get(title='Mein Podcast'), guid='m', + title='M', audio_url='https://mine.example/m.mp3')) + + def fake(url, **params): + if url.endswith('search/podcast'): + return {'data': [{'id': 7, 'title': 'Mein Podcast', 'xmlURL': 'https://mine.example/feed'}]} + if url.endswith('recommend'): + return {'data': [{'id': 8, 'title': 'Neu', 'xmlURL': 'https://new.example/feed'}, + {'id': 7, 'title': 'Mein Podcast', 'xmlURL': 'https://mine.example/feed'}]} + return None + with patch.object(discover, '_get_json', fake): + r = self.client.get('/podcasts/discover/recommend/').json() + self.assertEqual([(p['title'], p['because']) for p in r['podcasts']], [('Neu', ['Mein Podcast'])]) diff --git a/podcasts/test_lists.py b/podcasts/test_lists.py index 947f744..6fe4c19 100644 --- a/podcasts/test_lists.py +++ b/podcasts/test_lists.py @@ -151,6 +151,16 @@ class IntervalTests(Base): due = refresh_due(list(PodcastFeed.objects.filter(pk__in=[daily.pk, stale.pk, new.pk])), now) self.assertEqual(set(due), {stale.pk, new.pk}) + def test_favourite_subscriptions(self): + r = self.post(f'/podcasts/feeds/{self.feed.pk}/settings/', {'favorite': True}).json() + self.assertTrue(r['favorite']) + other = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/o', title='O') + PodcastEpisode.objects.create(feed=other, guid='o', title='O1', audio_url='https://example.org/o.mp3') + got = self.client.get('/podcasts/episodes/?filter=fave_feeds&limit=50').json() + self.assertEqual({e['feed__id'] for e in got['episodes']}, {self.feed.pk}) + self.assertTrue([f for f in self.state()['feeds'] if f['id'] == self.feed.pk][0]['favorite']) + self.assertEqual(PodcastLater.objects.filter(user=self.user).count(), 0, 'a favourite queues nothing') + def test_setting_it(self): r = self.post(f'/podcasts/feeds/{self.feed.pk}/settings/', {'refresh_interval': 'weekly'}).json() self.assertEqual(r['refresh_interval'], 'weekly') diff --git a/podcasts/tests.py b/podcasts/tests.py index 702b16d..4265b61 100644 --- a/podcasts/tests.py +++ b/podcasts/tests.py @@ -1106,7 +1106,7 @@ FEED_DEFAULTS = { 'playback_speed': None, 'skip_intro': 0, 'skip_outro': 0, 'volume': 1.0, 'new_episodes': 'inbox', 'include_words': '', 'exclude_words': '', 'min_duration': 0, 'auto_queue': False, 'auto_download': False, 'notify': False, 'episode_sort': '', 'episode_filter': [], 'skip_silence': None, 'kind': 'podcast', - 'folder': '', 'folders': [], 'refresh_interval': '', + 'folder': '', 'folders': [], 'refresh_interval': '', 'favorite': False, } @@ -1134,7 +1134,7 @@ class FeedSettingsTests(TestCase): 'new_episodes': 'inbox', 'include_words': 'News, Talk', 'exclude_words': 'Teaser', 'min_duration': 10, 'auto_queue': False, 'auto_download': False, 'notify': False, 'episode_sort': '', 'episode_filter': [], 'skip_silence': None, 'kind': 'podcast', 'folder': '', 'folders': [], - 'refresh_interval': '', + 'refresh_interval': '', 'favorite': False, }) self.feed.refresh_from_db() self.assertEqual((self.feed.playback_speed, self.feed.skip_intro, self.feed.volume, self.feed.min_duration), diff --git a/podcasts/urls.py b/podcasts/urls.py index f548b79..2433919 100644 --- a/podcasts/urls.py +++ b/podcasts/urls.py @@ -1,5 +1,5 @@ from django.urls import path -from . import audiobooks, folders, views +from . import audiobooks, discover, folders, views urlpatterns = [ path('search/', views.podcast_search, name='podcast_search'), @@ -39,6 +39,16 @@ urlpatterns = [ path('later/remove/', views.later_remove, name='podcast_later_remove'), path('lists/', views.lists_switch, name='podcast_lists_switch'), path('stream/seen/', views.stream_seen, name='podcast_stream_seen'), + # Stöbern (docs/podcasts.md, podcasts/discover.py) + path('discover/search/', discover.search_view, name='podcast_discover_search'), + path('discover/similar//', discover.similar_view, name='podcast_discover_similar'), + path('discover/people/', discover.people_view, name='podcast_discover_people'), + path('discover/person/', discover.person_view, name='podcast_discover_person'), + path('discover/elsewhere/', discover.elsewhere_view, name='podcast_discover_elsewhere'), + path('discover/topics/', discover.topics_view, name='podcast_discover_topics'), + path('discover/forgotten/', discover.forgotten_view, name='podcast_discover_forgotten'), + path('discover/recommend/', discover.recommend_view, name='podcast_discover_recommend'), + path('discover/take/', discover.take_view, name='podcast_discover_take'), path('settings/', views.podcast_settings, name='podcast_settings'), path('progress/save/', views.save_progress, name='podcast_save_progress'), path('progress/mark-played/', views.mark_played, name='podcast_mark_played'), diff --git a/podcasts/views.py b/podcasts/views.py index 6c712fb..3d2a9e7 100644 --- a/podcasts/views.py +++ b/podcasts/views.py @@ -892,7 +892,7 @@ FEED_SETTING_FIELDS = ( 'playback_speed', 'skip_intro', 'skip_outro', 'volume', 'new_episodes', 'include_words', 'exclude_words', 'min_duration', 'auto_queue', 'auto_download', 'notify', 'episode_sort', 'episode_filter', 'skip_silence', 'kind', 'folder', 'folders', - 'refresh_interval', + 'refresh_interval', 'favorite', ) @@ -1186,6 +1186,7 @@ FEED_SETTING_PARSERS = { 'folders': folders_mod.clean_list, 'skip_silence': _optional_boolean, 'refresh_interval': lambda v: _refresh_interval(v), + 'favorite': _boolean, } @@ -1599,6 +1600,8 @@ EPISODE_FILTERS = { 'later': Q(in_later=True), # The Strom (docs/podcasts.md): every subscription but the "Auf Abruf" ones. 'stream': ~Q(feed__new_episodes=PodcastFeed.NEW_IGNORE), + # Episodes of the favourite subscriptions. + 'fave_feeds': Q(feed__favorite=True), } # Episodes of unknown length (0) come last when sorting by length. _UNKNOWN_LENGTH_LAST = Case(When(duration_seconds__gt=0, then=Value(0)), default=Value(1), output_field=IntegerField()) diff --git a/static/js/sw.js b/static/js/sw.js index 461a9dd..42ee1f0 100644 --- a/static/js/sw.js +++ b/static/js/sw.js @@ -2,7 +2,7 @@ * diora service worker — caches the app shell for offline use. */ -const CACHE = 'diora-v137'; +const CACHE = 'diora-v138'; const SHELL = [ '/static/css/app.css', '/static/js/app/01-core.js',