From f4b8bcc8910c09df759fe61566c9b18ebbce919c Mon Sep 17 00:00:00 2001 From: mrwnslz Date: Fri, 2 Oct 2026 07:56:44 +0200 Subject: [PATCH] =?UTF-8?q?Podcasts:=20Kapitel,=20Shownotes=20und=20Transk?= =?UTF-8?q?ripte=20f=C3=BCr=20die=20Apps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Feeds liefern jetzt Podlove-Kapitel (psc:chapters), Podcasting-2.0-Kapitel (podcast:chapters, JSON-URL) und Transkripte (podcast:transcript, auch mehrere); als Beschreibung wird content:encoded bevorzugt. - Bekannte Folgen werden beim Aktualisieren nachgezogen, aber nur die Zeilen, die sich wirklich geändert haben. - Neuer Endpunkt GET /podcasts/episodes// mit bereinigten Shownotes, Zeitmarken, Kapiteln, Transkripten und Fortschritt. JSON-Kapitel werden einmal serverseitig geholt (nur öffentliche Adressen, max. 1 MB) und an der Folge gespeichert. Co-Authored-By: Claude Opus 5.5 --- podcasts/chapters.py | 166 ++++++++++ podcasts/feedext.py | 43 +++ .../0005_episode_chapters_transcripts.py | 33 ++ podcasts/models.py | 9 + podcasts/shownotes.py | 209 ++++++++++++ podcasts/tests.py | 305 ++++++++++++++++++ podcasts/urls.py | 1 + podcasts/views.py | 174 +++++++++- 8 files changed, 938 insertions(+), 2 deletions(-) create mode 100644 podcasts/chapters.py create mode 100644 podcasts/feedext.py create mode 100644 podcasts/migrations/0005_episode_chapters_transcripts.py create mode 100644 podcasts/shownotes.py diff --git a/podcasts/chapters.py b/podcasts/chapters.py new file mode 100644 index 0000000..3888489 --- /dev/null +++ b/podcasts/chapters.py @@ -0,0 +1,166 @@ +"""Podcast chapters: Podlove Simple Chapters from the feed, and Podcasting 2.0 +JSON chapters fetched from the URL a feed points to. + +Both end up in one shape, a list sorted by start time: + [{"start": 62.5, "title": "…", "url": "https://…", "img": "https://…"}] +url and img only when present. + +The JSON file lives on a server the feed's author chose, so fetching it is an +SSRF surface like the WebDAV import, and it reuses that module's guards: +public addresses only (checked before connecting and again on the connected +socket), redirects followed by hand with each hop re-checked, and a size cap. +""" + +import json +import re +from urllib.parse import urljoin + +import requests + +from accounts.webdav import WebDAVError, WebDAVInputError, _assert_peer_is_safe, _read_capped, assert_safe_url + +MAX_CHAPTERS = 500 +MAX_TITLE = 500 +MAX_URL = 1000 +CHAPTERS_TIMEOUT = 10 # seconds +CHAPTERS_MAX_BYTES = 1024 * 1024 +CHAPTERS_MAX_REDIRECTS = 3 + +_SCHEMES = ('http://', 'https://') + + +def _clean_url(value): + value = (value or '').strip() if isinstance(value, str) else '' + if value.lower().startswith(_SCHEMES) and len(value) <= MAX_URL: + return value + return '' + + +def _chapter(start, title, url='', img=''): + """One normalised chapter, or None if the start isn't a usable time.""" + try: + start = float(start) + except (TypeError, ValueError): + return None + if start != start or start < 0 or start == float('inf'): # NaN, negative, inf + return None + ch = {'start': round(start, 3), 'title': ' '.join(str(title or '').split())[:MAX_TITLE]} + url, img = _clean_url(url), _clean_url(img) + if url: + ch['url'] = url + if img: + ch['img'] = img + return ch + + +def _finish(chapters): + chapters = [c for c in chapters if c is not None] + chapters.sort(key=lambda c: c['start']) + return chapters[:MAX_CHAPTERS] + + +# --------------------------------------------------------------------------- +# Podlove Simple Chapters (inline in the feed) +# --------------------------------------------------------------------------- + +# "1:02:03.500", "01:02:03", "02:03", "123.5" (normal play time, as PSC uses) +_NPT = re.compile(r'^\s*(?:(?:(\d+):)?(\d{1,2}):)?(\d+(?:\.\d+)?)\s*$') + + +def parse_npt(value): + """Seconds (float) from a PSC start time, or None.""" + m = _NPT.match(str(value or '')) + if not m: + return None + hours, minutes, seconds = m.groups() + return int(hours or 0) * 3600 + int(minutes or 0) * 60 + float(seconds) + + +def from_psc(entry): + """Chapters from feedparser's entry["psc_chapters"], [] if there are none.""" + raw = (entry.get('psc_chapters') or {}).get('chapters') or [] + return _finish( + _chapter(parse_npt(c.get('start')), c.get('title'), c.get('href'), c.get('image')) + for c in raw + ) + + +# --------------------------------------------------------------------------- +# Podcasting 2.0 JSON chapters (fetched) +# --------------------------------------------------------------------------- + +def from_json(data): + """Normalise a JSON chapters document (https://github.com/Podcastindex-org/ + podcast-namespace/blob/main/chapters/jsonChapters.md).""" + raw = data.get('chapters') if isinstance(data, dict) else None + if not isinstance(raw, list): + raise ValueError('no chapters list') + chapters = [] + for c in raw: + if not isinstance(c, dict): + continue + # toc: false = "don't show this one to the user" (e.g. art changes). + if c.get('toc') is False: + continue + chapters.append(_chapter(c.get('startTime'), c.get('title'), c.get('url'), c.get('img'))) + return _finish(chapters) + + +class ChaptersUnavailable(Exception): + """The chapters can't be loaded right now; worth trying again later.""" + + +class ChaptersInvalid(Exception): + """The chapters URL is unusable for good (blocked, missing, not chapters).""" + + +def fetch_json(url): + """Download and parse the JSON at `url`. + + Raises ChaptersInvalid for permanent problems and ChaptersUnavailable for + ones that may go away (network trouble, server errors). + """ + for _ in range(CHAPTERS_MAX_REDIRECTS + 1): + try: + assert_safe_url(url) + except WebDAVInputError as e: + raise ChaptersInvalid(str(e)) + try: + r = requests.get( + url, timeout=CHAPTERS_TIMEOUT, allow_redirects=False, stream=True, + headers={'User-Agent': 'diora podcast chapters', 'Accept': 'application/json'}, + ) + except requests.RequestException as e: + raise ChaptersUnavailable(e.__class__.__name__) + try: + _assert_peer_is_safe(r) + if r.status_code in (301, 302, 303, 307, 308): + location = r.headers.get('Location') + if not location: + raise ChaptersInvalid('redirect without a location') + url = urljoin(url, location) + continue + if r.status_code >= 500 or r.status_code == 429: + raise ChaptersUnavailable(f'HTTP {r.status_code}') + if r.status_code >= 400: + raise ChaptersInvalid(f'HTTP {r.status_code}') + body = _read_capped(r, CHAPTERS_MAX_BYTES, lambda: ChaptersInvalid('too large')) + except WebDAVInputError as e: + raise ChaptersInvalid(str(e)) + except WebDAVError as e: # transfer broke off in _read_capped + raise ChaptersUnavailable(str(e)) + finally: + r.close() + try: + return json.loads(body) + except ValueError: + raise ChaptersInvalid('not JSON') + raise ChaptersInvalid('too many redirects') + + +def fetch(url): + """Normalised chapters from a Podcasting 2.0 chapters URL.""" + try: + return from_json(fetch_json(url)) + except ValueError: + raise ChaptersInvalid('not a chapters document') diff --git a/podcasts/feedext.py b/podcasts/feedext.py new file mode 100644 index 0000000..33061a1 --- /dev/null +++ b/podcasts/feedext.py @@ -0,0 +1,43 @@ +"""Teach feedparser the Podcasting 2.0 tags we read. + +feedparser 6 doesn't know the podcast namespace. Unknown tags still come +through, but only under whatever prefix the feed happened to declare, and an +unknown tag with attributes is stored as a single dict, so a second + overwrites the first. Registering the namespace fixes the +prefix ("podcast_…" for every feed); the handler below keeps every +transcript in a list, entry["podcast_transcripts"]. + +podcast:chapters is left to feedparser's default handling, which gives +entry["podcast_chapters"] = {"url": …, "type": …} (one per item, as the spec +says). +""" + +from feedparser.mixin import _FeedParserMixin as FeedParserMixin +from feedparser.util import FeedParserDict + +PODCAST_NAMESPACES = ( + 'https://podcastindex.org/namespace/1.0', + # The URL the namespace was announced under; some early feeds still use it. + 'https://github.com/podcastindex-org/podcast-namespace/blob/main/docs/1.0.md', +) + + +def _start_podcast_transcript(self, attrs_d): + context = self._get_context() + context.setdefault('podcast_transcripts', []).append(FeedParserDict(attrs_d)) + + +def _end_podcast_transcript(self): + pass + + +def install(): + """Idempotent; called when podcasts.views is imported.""" + for uri in PODCAST_NAMESPACES: + FeedParserMixin.namespaces.setdefault(uri, 'podcast') + # Filled lazily from `namespaces` by the first parser; once it has + # been, later additions have to go in here directly. + if FeedParserMixin._matchnamespaces: + FeedParserMixin._matchnamespaces.setdefault(uri.lower(), 'podcast') + FeedParserMixin._start_podcast_transcript = _start_podcast_transcript + FeedParserMixin._end_podcast_transcript = _end_podcast_transcript diff --git a/podcasts/migrations/0005_episode_chapters_transcripts.py b/podcasts/migrations/0005_episode_chapters_transcripts.py new file mode 100644 index 0000000..426d83d --- /dev/null +++ b/podcasts/migrations/0005_episode_chapters_transcripts.py @@ -0,0 +1,33 @@ +# Generated by Django 6.1 on 2026-10-02 05:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('podcasts', '0004_podcastsettings'), + ] + + operations = [ + migrations.AddField( + model_name='podcastepisode', + name='chapters', + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name='podcastepisode', + name='chapters_fetched_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='podcastepisode', + name='chapters_url', + field=models.URLField(blank=True, max_length=1000), + ), + migrations.AddField( + model_name='podcastepisode', + name='transcripts', + field=models.JSONField(blank=True, default=list), + ), + ] diff --git a/podcasts/models.py b/podcasts/models.py index a80f060..d63e427 100644 --- a/podcasts/models.py +++ b/podcasts/models.py @@ -34,6 +34,15 @@ class PodcastEpisode(models.Model): season_number = models.IntegerField(null=True, blank=True) artwork_url = models.URLField(max_length=1000, blank=True) discovered_at = models.DateTimeField(auto_now_add=True) + # Podcasting 2.0 : a JSON file, loaded on demand. + chapters_url = models.URLField(max_length=1000, blank=True) + # [{start: seconds, title, url?, img?}], from Podlove Simple Chapters in + # the feed, or the JSON at chapters_url once it has been fetched. + chapters = models.JSONField(default=list, blank=True) + # When chapters_url was fetched into `chapters`; null = not yet. + chapters_fetched_at = models.DateTimeField(null=True, blank=True) + # [{url, type, language}] from . + transcripts = models.JSONField(default=list, blank=True) class Meta: unique_together = ('feed', 'guid') diff --git a/podcasts/shownotes.py b/podcasts/shownotes.py new file mode 100644 index 0000000..3740e3d --- /dev/null +++ b/podcasts/shownotes.py @@ -0,0 +1,209 @@ +"""Shownotes HTML: an allowlist sanitizer and timestamp extraction. + +Feed HTML is written by whoever runs the podcast, so it is untrusted. feedparser +already cleans it once while parsing, but the apps render what this server hands +out in web views, so we clean it again on the way out with a deliberately small +allowlist: unknown tags are dropped (their text kept), script-like tags are +dropped with their content, and only a handful of attributes survive, with +links limited to http(s)/mailto. +""" + +import html +import re +from html.parser import HTMLParser + +ALLOWED_TAGS = { + 'a', 'abbr', 'b', 'blockquote', 'br', 'cite', 'code', 'dd', 'del', 'div', 'dl', 'dt', + 'em', 'figcaption', 'figure', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', + 'ins', 'li', 'mark', 'ol', 'p', 'pre', 'q', 's', 'small', 'span', 'strong', 'sub', + 'sup', 'table', 'tbody', 'td', 'tfoot', 'th', 'thead', 'time', 'tr', 'u', 'ul', +} +VOID_TAGS = {'br', 'hr', 'img'} +# Dropped together with everything inside them. +DROP_CONTENT_TAGS = { + 'script', 'style', 'iframe', 'object', 'embed', 'noscript', 'template', 'svg', + 'math', 'head', 'title', 'frameset', 'frame', 'applet', 'audio', 'video', 'canvas', + 'select', 'textarea', 'button', +} +ALLOWED_ATTRS = { + 'a': {'href', 'title'}, + 'img': {'src', 'alt', 'title', 'width', 'height'}, + 'td': {'colspan', 'rowspan'}, + 'th': {'colspan', 'rowspan'}, + 'abbr': {'title'}, + 'time': {'datetime'}, + 'ol': {'start'}, +} +URL_ATTRS = {'href', 'src'} +LINK_SCHEMES = {'http', 'https', 'mailto'} +IMAGE_SCHEMES = {'http', 'https'} + +_CONTROL_AND_SPACE = re.compile(r'[\x00-\x20\x7f]+') +_SCHEME = re.compile(r'^([a-z][a-z0-9+.\-]*):') + + +def _safe_url(value, schemes): + """The URL if it is absolute with an allowed scheme, else None. + + Browsers ignore whitespace and control characters inside a scheme + ("java\\tscript:"), so those are removed before looking at it. + """ + compact = _CONTROL_AND_SPACE.sub('', value or '') + m = _SCHEME.match(compact.lower()) + if not m or m.group(1) not in schemes: + return None + return value.strip() + + +class _Sanitizer(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.out = [] + self.open = [] # allowed tags currently open, outermost first + self.dropping = 0 # depth inside DROP_CONTENT_TAGS + + def handle_starttag(self, tag, attrs): + self._start(tag, attrs, closed=False) + + def handle_startendtag(self, tag, attrs): + self._start(tag, attrs, closed=True) + + def _start(self, tag, attrs, closed): + if tag in DROP_CONTENT_TAGS: + if not closed: + self.dropping += 1 + return + if self.dropping or tag not in ALLOWED_TAGS: + return + kept = [] + allowed = ALLOWED_ATTRS.get(tag, set()) + for name, value in attrs: + name = name.lower() + if name not in allowed or value is None: + continue + if name in URL_ATTRS: + value = _safe_url(value, IMAGE_SCHEMES if tag == 'img' else LINK_SCHEMES) + if value is None: + continue + kept.append((name, value)) + if tag == 'img' and not any(n == 'src' for n, _ in kept): + return + if tag == 'a': + kept.append(('rel', 'nofollow noopener noreferrer')) + attr_text = ''.join(f' {n}="{html.escape(v, quote=True)}"' for n, v in kept) + self.out.append(f'<{tag}{attr_text}>') + if tag not in VOID_TAGS and not closed: + self.open.append(tag) + + def handle_endtag(self, tag): + if tag in DROP_CONTENT_TAGS: + if self.dropping: + self.dropping -= 1 + return + if self.dropping or tag not in self.open: + return + # Close everything opened inside it too, so the output stays nested. + while self.open: + t = self.open.pop() + self.out.append(f'') + if t == tag: + break + + def handle_data(self, data): + if not self.dropping: + self.out.append(html.escape(data, quote=False)) + + def result(self): + self.close() + while self.open: + self.out.append(f'') + return ''.join(self.out) + + +def sanitize_html(text): + """Return `text` reduced to the allowlist above.""" + if not text: + return '' + s = _Sanitizer() + s.feed(text) + return s.result() + + +# --------------------------------------------------------------------------- +# Plain text and timestamps +# --------------------------------------------------------------------------- + +_BLOCK_TAGS = { + 'br', 'p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', + 'pre', 'dd', 'dt', 'hr', 'table', 'ul', 'ol', 'figure', 'figcaption', +} + + +class _TextExtractor(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.out = [] + self.dropping = 0 + + def handle_starttag(self, tag, attrs): + if tag in DROP_CONTENT_TAGS: + self.dropping += 1 + elif tag in _BLOCK_TAGS: + self.out.append('\n') + + def handle_startendtag(self, tag, attrs): + if tag in _BLOCK_TAGS: + self.out.append('\n') + + def handle_endtag(self, tag): + if tag in DROP_CONTENT_TAGS: + self.dropping = max(self.dropping - 1, 0) + elif tag in _BLOCK_TAGS: + self.out.append('\n') + + def handle_data(self, data): + if not self.dropping: + self.out.append(data) + + +def html_to_text(text): + """Shownotes as plain text, one line per block/line break.""" + if not text: + return '' + p = _TextExtractor() + p.feed(text) + p.close() + return ''.join(p.out) + + +# A time at the start of a line, optionally in brackets, then the label: +# "12:34 Intro", "(1:02:03) - Fragen", "[00:05] Begrüßung". +_TIMESTAMP_LINE = re.compile( + r'^[\s\-*•·]*[(\[]?(?:(\d{1,2}):)?(\d{1,2}):(\d{2})(?:\.\d+)?[)\]]?(?=\s|$|[-–—:|])' + r'\s*[-–—:|]?\s*(.*)$' +) +MAX_TIMESTAMPS = 300 + + +def extract_timestamps(text, duration_seconds=0): + """[{seconds, label}] for lines of the shownotes that start with a time. + + Only line starts are looked at: a time in the middle of a sentence is just + as likely to be a time of day. + """ + result = [] + for line in html_to_text(text).splitlines(): + m = _TIMESTAMP_LINE.match(line) + if not m: + continue + hours, minutes, seconds, label = m.groups() + minutes, seconds = int(minutes), int(seconds) + if seconds >= 60 or (hours is not None and minutes >= 60): + continue + total = int(hours or 0) * 3600 + minutes * 60 + seconds + if duration_seconds and total > duration_seconds: + continue + result.append({'seconds': total, 'label': ' '.join(label.split())[:300]}) + if len(result) >= MAX_TIMESTAMPS: + break + return result diff --git a/podcasts/tests.py b/podcasts/tests.py index 4a0f420..65e03ce 100644 --- a/podcasts/tests.py +++ b/podcasts/tests.py @@ -228,3 +228,308 @@ class RefreshAllTests(TestCase): self.assertEqual([e['id'] for e in data['inbox']], [ep.id]) self.assertTrue(data['inbox'][0]['in_queue']) self.assertEqual(data['settings'], {'enqueue_position': 'back', 'auto_queue_position': 'front'}) + + +# --------------------------------------------------------------------------- +# Shownotes, chapters, transcripts +# --------------------------------------------------------------------------- + +NS = ( + 'xmlns:podcast="https://podcastindex.org/namespace/1.0" ' + 'xmlns:psc="http://podlove.org/simple-chapters" ' + 'xmlns:content="http://purl.org/rss/1.0/modules/content/" ' + 'xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"' +) + + +def _rich_rss(items, ns=NS): + """Like _rss, but each item is (guid, extra XML inside the ).""" + entries = ''.join( + f'{guid}{guid}Mon, 05 Jan 2026 10:00:00 GMT' + f'{extra}' + for guid, extra in items + ) + return f'Feed{entries}' + + +PSC = ( + '' + '' + '' + '' + '' + '' +) +TRANSCRIPTS = ( + '' + '' + '' +) +CHAPTERS_JSON = { + 'version': '1.2.0', + 'chapters': [ + {'startTime': 95.5, 'title': 'Zweites', 'url': 'https://x.org/b', 'img': 'https://x.org/b.jpg'}, + {'startTime': 0, 'title': 'Erstes'}, + {'startTime': 50, 'title': 'Nur Bild', 'toc': False}, + {'startTime': 'kaputt', 'title': 'Ungültig'}, + {'startTime': 120, 'title': 'Link', 'url': 'javascript:alert(1)'}, + ], +} + + +class FeedExtrasParsingTests(TestCase): + """What feedparser (6.0.x) hands us for these tags, and what we store.""" + + def setUp(self): + self.user = User.objects.create_user(username='alice', password='pw12345678') + self.feed = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/feed', title='Feed') + + def refresh(self, rss): + with patch('podcasts.views._fetch_feed', lambda url: _parse(rss)): + _refresh_feed(self.feed) + + def test_feedparser_keys(self): + import podcasts.views # noqa: F401 -- installs the feedparser extension + rss = _rich_rss([('a', PSC + TRANSCRIPTS + '')]) + entry = _parse(rss).entries[0] + self.assertEqual(entry['podcast_chapters']['url'], 'https://x.org/c.json') + self.assertEqual([t['url'] for t in entry['podcast_transcripts']], + ['https://example.org/a.vtt', 'https://example.org/a.srt', 'javascript:alert(1)']) + self.assertEqual(entry['psc_chapters']['chapters'][1]['start'], '00:01:02.500') + + def test_other_prefix_for_the_podcast_namespace(self): + ns = NS.replace('xmlns:podcast=', 'xmlns:pc=') + rss = _rich_rss([('a', '' + '')], ns) + self.refresh(rss) + ep = PodcastEpisode.objects.get() + self.assertEqual(ep.chapters_url, 'https://x.org/c.json') + self.assertEqual(ep.transcripts, [{'url': 'https://example.org/a.vtt', 'type': 'text/vtt', 'language': ''}]) + + def test_psc_chapters(self): + self.refresh(_rich_rss([('a', PSC)])) + ep = PodcastEpisode.objects.get() + self.assertEqual(ep.chapters, [ + {'start': 0.0, 'title': 'Intro'}, + {'start': 62.5, 'title': 'Teil & zwei', 'url': 'https://x.org/2', 'img': 'https://x.org/2.jpg'}, + {'start': 725.0, 'title': 'Kurz'}, + {'start': 3723.0, 'title': 'Drei'}, + ]) + self.assertEqual(ep.chapters_url, '') + + def test_transcripts_and_chapters_url(self): + self.refresh(_rich_rss([('a', TRANSCRIPTS + '')])) + ep = PodcastEpisode.objects.get() + self.assertEqual(ep.transcripts, [ + {'url': 'https://example.org/a.vtt', 'type': 'text/vtt', 'language': 'de'}, + {'url': 'https://example.org/a.srt', 'type': 'application/srt', 'language': ''}, + ]) + self.assertEqual(ep.chapters_url, 'https://x.org/c.json') + self.assertEqual(ep.chapters, []) + self.assertIsNone(ep.chapters_fetched_at) + + def test_content_encoded_is_preferred(self): + self.refresh(_rich_rss([ + ('a', 'Kurz' + 'Lang Link

]]>'), + ('b', 'Nur kurz'), + ])) + self.assertIn('Lang', PodcastEpisode.objects.get(guid='a').description) + self.assertEqual(PodcastEpisode.objects.get(guid='b').description, 'Nur kurz') + + def test_existing_episodes_are_updated(self): + self.refresh(_rich_rss([('a', 'Alt'), ('b', 'Gleich')])) + a, b = PodcastEpisode.objects.get(guid='a'), PodcastEpisode.objects.get(guid='b') + rss = _rich_rss([ + ('a', 'Neu' + PSC + TRANSCRIPTS), + ('b', 'Gleich'), + ]) + with patch('podcasts.views.PodcastEpisode.objects.bulk_update', + wraps=PodcastEpisode.objects.bulk_update) as bulk: + self.refresh(rss) + # Only the episode that changed is written. + self.assertEqual([e.guid for e in bulk.call_args.args[0]], ['a']) + a.refresh_from_db() + self.assertEqual(a.description, 'Neu') + self.assertEqual(len(a.chapters), 4) + self.assertEqual(len(a.transcripts), 2) + self.assertEqual(PodcastEpisode.objects.count(), 2) + # Nothing changed: nothing written. + with patch('podcasts.views.PodcastEpisode.objects.bulk_update') as bulk: + self.refresh(rss) + bulk.assert_not_called() + + def test_fetched_chapters_survive_a_refresh_until_the_url_changes(self): + tag = '' + self.refresh(_rich_rss([('a', tag.format('c1'))])) + fetched = datetime(2026, 1, 2, tzinfo=timezone.utc) + PodcastEpisode.objects.update(chapters=[{'start': 0.0, 'title': 'X'}], chapters_fetched_at=fetched) + self.refresh(_rich_rss([('a', tag.format('c1'))])) + ep = PodcastEpisode.objects.get() + self.assertEqual(ep.chapters, [{'start': 0.0, 'title': 'X'}]) + self.assertEqual(ep.chapters_fetched_at, fetched) + self.refresh(_rich_rss([('a', tag.format('c2'))])) + ep.refresh_from_db() + self.assertEqual((ep.chapters_url, ep.chapters, ep.chapters_fetched_at), ('https://x.org/c2.json', [], None)) + + +class ShownotesTests(TestCase): + def test_sanitizer(self): + from .shownotes import sanitize_html + dirty = ( + '

Hallo Link' + '' + 'böseauch' + '' + 'Text bleibtoffen' + ) + self.assertEqual( + sanitize_html(dirty), + '

Hallo Link' + 'böseauch' + 'Text bleibtoffen

', + ) + self.assertEqual(sanitize_html('1 < 2 && 3'), '1 < 2 && 3') + self.assertEqual(sanitize_html('Mail'), + 'Mail') + + def test_timestamps(self): + from .shownotes import extract_timestamps + notes = ( + '

Kapitel:
00:00 Intro
(12:34) - Thema eins
1:02:03 Zum Schluss

' + '
  • [05:07] Liste
  • Um 20:15 geht es los
  • 99:99 kaputt
' + ) + self.assertEqual(extract_timestamps(notes), [ + {'seconds': 0, 'label': 'Intro'}, + {'seconds': 754, 'label': 'Thema eins'}, + {'seconds': 3723, 'label': 'Zum Schluss'}, + {'seconds': 307, 'label': 'Liste'}, + ]) + self.assertEqual(len(extract_timestamps(notes, duration_seconds=1000)), 3) + + +class _FakeResponse: + def __init__(self, body=b'', status=200, headers=None): + self.body, self.status_code, self.headers = body, status, headers or {} + self.raw = None + + def iter_content(self, chunk_size=1): + yield self.body + + def close(self): + pass + + +class EpisodeDetailTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username='alice', password='pw12345678') + self.client.force_login(self.user) + self.feed = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/feed', title='Feed', + artwork_url='https://example.org/art.jpg') + self.ep = PodcastEpisode.objects.create( + feed=self.feed, guid='a', title='A', audio_url='https://example.org/a.mp3', duration_seconds=3600, + pub_date=datetime(2026, 1, 5, 10, tzinfo=timezone.utc), + description='

Hallo
01:02 Thema x

', + transcripts=[{'url': 'https://example.org/a.vtt', 'type': 'text/vtt', 'language': 'de'}], + chapters_url='https://x.org/c.json', + ) + self.requests = [] + + def fake_get(self, url, **kwargs): + self.requests.append(url) + return _FakeResponse(json.dumps(CHAPTERS_JSON).encode()) + + def get(self, ep=None): + with patch('podcasts.chapters.assert_safe_url'), patch('podcasts.chapters.requests.get', self.fake_get): + return self.client.get(f'/podcasts/episodes/{(ep or self.ep).id}/') + + def test_detail(self): + EpisodeProgress.objects.create(user=self.user, episode=self.ep, position_seconds=42) + r = self.get() + self.assertEqual(r.status_code, 200) + data = r.json() + self.assertEqual(data['description'], '

Hallo
01:02 Thema x

') + self.assertEqual(data['timestamps'], [{'seconds': 62, 'label': 'Thema x'}]) + self.assertEqual(data['transcripts'], self.ep.transcripts) + self.assertEqual(data['feed'], {'id': self.feed.id, 'title': 'Feed', + 'artwork_url': 'https://example.org/art.jpg'}) + self.assertEqual((data['position_seconds'], data['played'], data['in_queue']), (42, False, False)) + self.assertEqual(data['pub_date'], '2026-01-05T10:00:00+00:00') + self.assertEqual(data['chapters'], [ + {'start': 0.0, 'title': 'Erstes'}, + {'start': 95.5, 'title': 'Zweites', 'url': 'https://x.org/b', 'img': 'https://x.org/b.jpg'}, + {'start': 120.0, 'title': 'Link'}, + ]) + + def test_chapters_are_fetched_once(self): + self.get() + self.get() + self.assertEqual(self.requests, ['https://x.org/c.json']) + self.ep.refresh_from_db() + self.assertEqual(len(self.ep.chapters), 3) + self.assertIsNotNone(self.ep.chapters_fetched_at) + + def test_inline_chapters_need_no_fetch(self): + self.ep.chapters = [{'start': 0.0, 'title': 'Inline'}] + self.ep.save() + self.assertEqual(self.get().json()['chapters'], [{'start': 0.0, 'title': 'Inline'}]) + self.assertEqual(self.requests, []) + + def test_redirect_is_followed_and_rechecked(self): + def fake_get(url, **kwargs): + self.requests.append(url) + if url.endswith('c.json'): + return _FakeResponse(status=302, headers={'Location': '/real.json'}) + return _FakeResponse(json.dumps(CHAPTERS_JSON).encode()) + with patch('podcasts.chapters.assert_safe_url') as check, patch('podcasts.chapters.requests.get', fake_get): + r = self.client.get(f'/podcasts/episodes/{self.ep.id}/') + self.assertEqual(len(r.json()['chapters']), 3) + self.assertEqual([c.args[0] for c in check.call_args_list], ['https://x.org/c.json', 'https://x.org/real.json']) + + def test_private_chapters_url_is_refused_and_remembered(self): + self.ep.chapters_url = 'http://127.0.0.1/c.json' + self.ep.save() + with patch('podcasts.chapters.requests.get', self.fake_get): + r = self.client.get(f'/podcasts/episodes/{self.ep.id}/') + self.assertEqual(r.json()['chapters'], []) + self.assertEqual(self.requests, []) + self.ep.refresh_from_db() + self.assertIsNotNone(self.ep.chapters_fetched_at) + + def test_oversized_or_broken_chapters(self): + from . import chapters + with patch('podcasts.chapters.assert_safe_url'), patch( + 'podcasts.chapters.requests.get', + lambda url, **kw: _FakeResponse(b'x' * (chapters.CHAPTERS_MAX_BYTES + 1)), + ): + with self.assertRaises(chapters.ChaptersInvalid): + chapters.fetch('https://x.org/big.json') + with patch('podcasts.chapters.assert_safe_url'), patch( + 'podcasts.chapters.requests.get', lambda url, **kw: _FakeResponse(b'{"chapters": 3}'), + ): + with self.assertRaises(chapters.ChaptersInvalid): + chapters.fetch('https://x.org/odd.json') + + def test_server_error_is_retried_later(self): + with patch('podcasts.chapters.assert_safe_url'), patch( + 'podcasts.chapters.requests.get', lambda url, **kw: _FakeResponse(status=503), + ): + r = self.client.get(f'/podcasts/episodes/{self.ep.id}/') + self.assertEqual(r.json()['chapters'], []) + self.ep.refresh_from_db() + self.assertIsNone(self.ep.chapters_fetched_at) + + def test_other_users_episode_is_not_found(self): + bob = User.objects.create_user(username='bob', password='pw12345678') + feed = PodcastFeed.objects.create(user=bob, rss_url='https://example.org/b', title='B') + ep = PodcastEpisode.objects.create(feed=feed, guid='x', title='X', audio_url='https://example.org/x.mp3', + chapters_url='https://x.org/c.json') + self.assertEqual(self.get(ep).status_code, 404) + self.assertEqual(self.requests, []) + + def test_login_required(self): + self.client.logout() + self.assertEqual(self.get().status_code, 401) diff --git a/podcasts/urls.py b/podcasts/urls.py index 4f65147..1f67788 100644 --- a/podcasts/urls.py +++ b/podcasts/urls.py @@ -12,6 +12,7 @@ urlpatterns = [ path('feeds//remove/', views.remove_feed, name='podcast_remove_feed'), path('feeds//set-auto-queue/', views.set_auto_queue, name='podcast_set_auto_queue'), path('feeds//episodes/', views.feed_episodes, name='podcast_feed_episodes'), + path('episodes//', views.episode_detail, name='podcast_episode_detail'), path('queue/', views.queue_get, name='podcast_queue_get'), path('queue/add/', views.queue_add, name='podcast_queue_add'), path('queue/remove/', views.queue_remove, name='podcast_queue_remove'), diff --git a/podcasts/views.py b/podcasts/views.py index a651052..566c624 100644 --- a/podcasts/views.py +++ b/podcasts/views.py @@ -14,13 +14,20 @@ from django.utils import timezone from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods +from . import chapters as chapters_mod +from . import feedext from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings +from .shownotes import extract_timestamps, sanitize_html + +feedext.install() log = logging.getLogger(__name__) FEED_TIMEOUT = 20 # seconds per feed download NEW_ON_SUBSCRIBE = 1 # episodes of a new subscription that count as new FEED_WORKERS = 8 # feeds downloaded at the same time +MAX_DESCRIPTION = 50_000 # characters of shownotes HTML kept per episode +MAX_TRANSCRIPTS = 10 # --------------------------------------------------------------------------- @@ -116,6 +123,78 @@ def _parse_duration(raw): return 0 +# --------------------------------------------------------------------------- +# Shownotes, chapters and transcripts of a feed entry +# --------------------------------------------------------------------------- + +def _entry_description(entry): + """Shownotes HTML: content:encoded when the feed has it (that's where the + full notes usually are), else the description/summary.""" + for c in entry.get('content') or []: + value = (c.get('value') or '').strip() + if value: + return value[:MAX_DESCRIPTION] + return (entry.get('summary') or entry.get('description') or '')[:MAX_DESCRIPTION] + + +def _entry_chapters_url(entry): + url = ((entry.get('podcast_chapters') or {}).get('url') or '').strip() + return url[:1000] if url.lower().startswith(('http://', 'https://')) else '' + + +def _entry_transcripts(entry): + raw = entry.get('podcast_transcripts') + if raw is None and entry.get('podcast_transcript'): + raw = [entry['podcast_transcript']] # feedparser without feedext + transcripts = [] + for t in raw or []: + url = (t.get('url') or '').strip() + if not url.lower().startswith(('http://', 'https://')) or len(url) > 1000: + continue + transcripts.append({ + 'url': url, + 'type': (t.get('type') or '')[:100], + 'language': (t.get('language') or '')[:35], + }) + if len(transcripts) >= MAX_TRANSCRIPTS: + break + return transcripts + + +def _apply_extras(episode, description, chapters_url, psc_chapters, transcripts): + """Copy what the feed says now onto `episode`; True if anything changed. + + Inline (Podlove) chapters win. Without them, chapters fetched from + chapters_url stay cached for as long as the URL stays the same. + """ + changed = [] + if episode.description != description: + episode.description = description + changed.append('description') + if episode.transcripts != transcripts: + episode.transcripts = transcripts + changed.append('transcripts') + if psc_chapters: + chapters, fetched_at = psc_chapters, episode.chapters_fetched_at + elif chapters_url and chapters_url == episode.chapters_url and episode.chapters_fetched_at: + chapters, fetched_at = episode.chapters, episode.chapters_fetched_at + else: + chapters, fetched_at = [], None + if episode.chapters_url != chapters_url: + episode.chapters_url = chapters_url + changed.append('chapters_url') + if episode.chapters != chapters: + episode.chapters = chapters + changed.append('chapters') + if episode.chapters_fetched_at != fetched_at: + episode.chapters_fetched_at = fetched_at + changed.append('chapters_fetched_at') + return bool(changed) + + +EXTRA_FIELDS = ['description', 'transcripts', 'chapters_url', 'chapters', 'chapters_fetched_at'] + + # --------------------------------------------------------------------------- # Shared feed refresh helper # --------------------------------------------------------------------------- @@ -190,8 +269,18 @@ def _refresh_feed(feed_obj, parsed=None): max_ep = getattr(settings, 'PODCAST_MAX_EPISODES_PER_FEED', 200) new_count = 0 new_episodes_list = [] + entries = parsed.entries[:max_ep] - for entry in parsed.entries[:max_ep]: + # Episodes we already have, in one query, so shownotes/chapters/transcripts + # that changed in the feed can be brought up to date (only those rows). + guids = [str(e.get('id') or e.get('guid') or '')[:1000] for e in entries] + existing = { + ep.guid: ep + for ep in feed_obj.episodes.filter(guid__in=[g for g in guids if g]).only('id', 'guid', *EXTRA_FIELDS) + } + changed_episodes = {} + + for entry in entries: # Find audio enclosure audio_url = '' for enc in entry.get('enclosures', []): @@ -205,7 +294,16 @@ def _refresh_feed(feed_obj, parsed=None): guid = entry.get('id') or entry.get('guid') or audio_url guid = str(guid)[:1000] title = entry.get('title', 'Untitled')[:500] - description = entry.get('summary', entry.get('description', '')) + description = _entry_description(entry) + chapters_url = _entry_chapters_url(entry) + psc_chapters = chapters_mod.from_psc(entry) + transcripts = _entry_transcripts(entry) + + known = existing.get(guid) + if known is not None: + if _apply_extras(known, description, chapters_url, psc_chapters, transcripts): + changed_episodes[known.id] = known + continue duration_raw = entry.get('itunes_duration', 0) duration_seconds = _parse_duration(duration_raw) @@ -242,11 +340,20 @@ def _refresh_feed(feed_obj, parsed=None): 'episode_number': ep_number, 'season_number': season_number, 'artwork_url': artwork_url, + 'chapters_url': chapters_url, + 'chapters': psc_chapters, + 'transcripts': transcripts, }, ) + existing[guid] = episode_obj # the same guid twice in one feed if created: new_count += 1 new_episodes_list.append(episode_obj) + elif _apply_extras(episode_obj, description, chapters_url, psc_chapters, transcripts): + changed_episodes[episode_obj.id] = episode_obj + + if changed_episodes: + PodcastEpisode.objects.bulk_update(list(changed_episodes.values()), EXTRA_FIELDS, batch_size=100) if first_fetch and len(new_episodes_list) > NEW_ON_SUBSCRIBE: # Only the newest episodes count as new; the rest stay in the @@ -475,6 +582,69 @@ def feed_episodes(request, pk): }) +# --------------------------------------------------------------------------- +# One episode in detail: shownotes, chapters, transcripts +# --------------------------------------------------------------------------- + +def _load_chapters(episode): + """Fetch the JSON chapters of `episode` once and cache them on it. + + Problems that may go away (timeouts, server errors) aren't cached, so the + next request tries again; anything else is remembered as "no chapters". + """ + url = episode.chapters_url + try: + chapters = chapters_mod.fetch(url) + except chapters_mod.ChaptersUnavailable as e: + log.info('chapters of episode %s not available: %s', episode.id, e) + return [] + except chapters_mod.ChaptersInvalid as e: + log.info('chapters of episode %s unusable: %s', episode.id, e) + chapters = [] + now = timezone.now() + # Only if a refresh hasn't changed the URL in the meantime. + PodcastEpisode.objects.filter(pk=episode.pk, chapters_url=url).update( + chapters=chapters, chapters_fetched_at=now, + ) + episode.chapters, episode.chapters_fetched_at = chapters, now + return chapters + + +@csrf_exempt +@require_http_methods(['GET']) +def episode_detail(request, pk): + if not request.user.is_authenticated: + return JsonResponse({'error': 'authentication required'}, status=401) + try: + ep = PodcastEpisode.objects.select_related('feed').get(pk=pk, feed__user=request.user) + except PodcastEpisode.DoesNotExist: + return JsonResponse({'error': 'not found'}, status=404) + + chapters = ep.chapters + if not chapters and ep.chapters_url and ep.chapters_fetched_at is None: + chapters = _load_chapters(ep) + + progress = EpisodeProgress.objects.filter(user=request.user, episode=ep).first() + return JsonResponse({ + 'id': ep.id, + 'title': ep.title, + 'description': sanitize_html(ep.description), + 'timestamps': extract_timestamps(ep.description, ep.duration_seconds), + 'chapters': chapters, + 'transcripts': ep.transcripts, + 'feed': {'id': ep.feed.id, 'title': ep.feed.title, 'artwork_url': ep.feed.artwork_url}, + 'audio_url': ep.audio_url, + 'artwork_url': ep.artwork_url, + 'duration_seconds': ep.duration_seconds, + 'pub_date': ep.pub_date.isoformat() if ep.pub_date else None, + 'episode_number': ep.episode_number, + 'season_number': ep.season_number, + 'position_seconds': progress.position_seconds if progress else 0, + 'played': progress.played if progress else False, + 'in_queue': PodcastQueue.objects.filter(user=request.user, episode=ep).exists(), + }) + + # --------------------------------------------------------------------------- # Import OPML # ---------------------------------------------------------------------------