From bd09d813662107daca16c98ca92df08cf0fc7345 Mon Sep 17 00:00:00 2001 From: mrwnslz Date: Mon, 9 Nov 2026 09:36:34 +0100 Subject: [PATCH] Smarte Transkription D: Artikel-Endpunkt und Blog-Anbindung MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST/DELETE /podcasts/episodes//article/ (podcasts/article.py): legt die Quelle Podcast-Artikel an und erzeugt oder ersetzt das Item der Folge; Sanitizer behält href=#t=; _item_data und Item-Ansicht liefern episode_id; feeds.js: Folge anhören und Zeitmarken-Links. Co-Authored-By: Claude Sonnet 5.5 --- docs/transcription.md | 7 ++ feeds/views.py | 14 +++- podcasts/article.py | 119 +++++++++++++++++++++++++++++ podcasts/shownotes.py | 7 +- podcasts/test_article.py | 146 ++++++++++++++++++++++++++++++++++++ podcasts/urls.py | 3 +- static/js/app/65-article.js | 3 +- static/js/feeds.js | 67 ++++++++++++++++- 8 files changed, 358 insertions(+), 8 deletions(-) create mode 100644 podcasts/article.py create mode 100644 podcasts/test_article.py diff --git a/docs/transcription.md b/docs/transcription.md index 4ac2821..dc5c873 100644 --- a/docs/transcription.md +++ b/docs/transcription.md @@ -91,6 +91,13 @@ author as `author`, the episode's artwork as `image_url`, now as `published` (so the blog stream under "heute"), `words` counted. Saving clears the wish. `DELETE` on the same address removes the article and its item. `GET /feeds/items//` and `_item_data` carry `episode_id` for such items. +`body_html` may contain `href="#t="` links; `clean_html` keeps exactly this fragment form +(`podcasts/shownotes.py` `TIME_LINK`) and no other `#` links. Validation: `title` 1 to 300 +characters, `body_html` non-empty after cleaning and at most 1,000,000 characters, `summary` at most +2000, `model` at most 100, `kind` at most 20 (400 otherwise). The item has `content_full` true (no +page is ever fetched) and, as the reader's furniture filter would strip repeated "▶ ab" blocks, +`GET /feeds/items//` skips that filter for synthetic sources. The endpoint is +`podcasts/article.py`. Synthetic sources are skipped by the feed refresh (`refresh_newsfeeds`, `refresh_now`) and by the pruning of old blog items (`feeds/refresh.py`). They can be hidden or have their bell turned off diff --git a/feeds/views.py b/feeds/views.py index b51e476..9c6ce57 100644 --- a/feeds/views.py +++ b/feeds/views.py @@ -108,6 +108,11 @@ def _item_data(i, content=False): 'summary': i.summary, 'image_url': i.image_url, 'read': i.read_at is not None, 'progress': i.progress, 'starred': i.starred, 'story': i.story_id, } + if i.source.synthetic: + # The article of a podcast episode (docs/transcription.md); the lists fetch it with the item. + art = getattr(i, 'episode_article', None) + if art is not None: + d['episode_id'] = art.episode_id if content: d['content'] = i.content d['content_full'] = i.content_full @@ -613,7 +618,7 @@ def story_overview(request, pk): def blogs(request): if (err := _auth(request)): return err - qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source') + qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source', 'episode_article') if request.GET.get('source', '').isdigit(): qs = qs.filter(source_id=int(request.GET['source'])) elif 'folder' in request.GET: @@ -642,7 +647,7 @@ def blogs(request): def starred(request): if (err := _auth(request)): return err - items = Item.objects.filter(user=request.user, starred=True).select_related('source').order_by('-published') + items = Item.objects.filter(user=request.user, starred=True).select_related('source', 'episode_article').order_by('-published') return JsonResponse({'items': [_item_data(i) for i in items[:200]]}) @@ -651,13 +656,14 @@ def starred(request): def item(request, pk): if (err := _auth(request)): return err - i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source').first() + i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source', 'episode_article').first() if i is None: return JsonResponse({'error': 'not found'}, status=404) if refresh.needs_article(i, i.source): refresh.load_article(i) d = _item_data(i, content=True) - d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source)) + if not i.source.synthetic: + d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source)) return JsonResponse({'item': d}) diff --git a/podcasts/article.py b/podcasts/article.py new file mode 100644 index 0000000..7491fc4 --- /dev/null +++ b/podcasts/article.py @@ -0,0 +1,119 @@ +"""The article written from an episode's transcript (docs/transcription.md): its text +is a feeds.Item in the user's favourite blog "Podcast-Artikel".""" + +import hashlib +import json + +from django.db import transaction +from django.http import JsonResponse +from django.utils import timezone +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods + +from feeds import fetch +from feeds.models import Item, Source + +from .models import EpisodeArticle, PodcastEpisode, TranscriptionJob + +SOURCE_URL = 'diora://podcast-artikel' +SOURCE_TITLE = 'Podcast-Artikel' +TITLE_MAX = 300 +SUMMARY_MAX = 2000 +BODY_MAX = 1_000_000 # characters of body_html +MODEL_MAX = 100 +KIND_MAX = 20 + + +def _auth(request): + if not request.user.is_authenticated: + return JsonResponse({'error': 'authentication required'}, status=401) + return None + + +def _body(request): + try: + body = json.loads(request.body or b'{}') + except (ValueError, UnicodeDecodeError): + return None + return body if isinstance(body, dict) else None + + +def article_source(user): + """The user's blog "Podcast-Artikel", made on first use.""" + source, _ = Source.objects.get_or_create(user=user, url=SOURCE_URL, defaults={ + 'title': SOURCE_TITLE, 'kind': Source.BLOG, 'fav': True, 'synthetic': True}) + return source + + +def _text(value, limit): + return value.strip() if isinstance(value, str) and len(value.strip()) <= limit else None + + +@csrf_exempt +@require_http_methods(['POST', 'DELETE']) +def episode_article(request, pk): + if (err := _auth(request)): + return err + ep = PodcastEpisode.objects.filter(pk=pk, feed__user=request.user).select_related('feed').first() + if ep is None: + return JsonResponse({'error': 'not found'}, status=404) + if request.method == 'DELETE': + art = EpisodeArticle.objects.filter(episode=ep).select_related('item').first() + if art is not None: + art.item.delete() # the article goes with it (cascade) + return JsonResponse({'ok': True}) + + body = _body(request) + if body is None: + return JsonResponse({'error': 'invalid JSON'}, status=400) + title = _text(body.get('title'), TITLE_MAX) + if not title: + return JsonResponse({'error': 'title needed (at most %d characters)' % TITLE_MAX}, status=400) + html_in = body.get('body_html') + if not isinstance(html_in, str) or not html_in.strip(): + return JsonResponse({'error': 'body_html needed'}, status=400) + if len(html_in) > BODY_MAX: + return JsonResponse({'error': 'body_html too large'}, status=400) + summary = body.get('summary') or '' + if not isinstance(summary, str) or len(summary) > SUMMARY_MAX: + return JsonResponse({'error': 'summary too long'}, status=400) + model = body.get('model') or '' + kind = body.get('kind') or '' + if not isinstance(model, str) or len(model) > MODEL_MAX or not isinstance(kind, str) or len(kind) > KIND_MAX: + return JsonResponse({'error': 'model or kind too long'}, status=400) + + url = getattr(ep, 'link', '') or ep.audio_url + content = fetch.clean_html(html_in, url) + if not fetch.plain(content): + return JsonResponse({'error': 'body_html empty'}, status=400) + fields = { + 'url': url[:2000], + 'title': title, + 'author': (ep.feed.author or '')[:300], + 'published': timezone.now(), + 'summary': fetch.plain(summary), + 'content': content, + 'content_full': True, # nothing to fetch from a page + 'words': fetch.word_count(content), + 'image_url': (ep.artwork_url or ep.feed.artwork_url or '')[:2000], + 'read_at': None, 'progress': 0, 'progress_at': None, + } + with transaction.atomic(): + source = article_source(request.user) + art = EpisodeArticle.objects.filter(episode=ep).select_related('item').first() + if art is not None: + item = art.item + for k, v in fields.items(): + setattr(item, k, v) + item.source = source + item.save() + art.model, art.kind = model, kind + art.save(update_fields=['model', 'kind']) + else: + guid = hashlib.sha256(f'diora-article:{ep.pk}'.encode()).hexdigest() + item = Item.objects.create(source=source, user=request.user, kind=Source.BLOG, + guid_hash=guid, **fields) + EpisodeArticle.objects.create(episode=ep, item=item, model=model, kind=kind) + # Saving fulfils the wish and frees the reservation. + TranscriptionJob.objects.filter(episode=ep).update(want_article=False, claimed_by='', claimed_until=None) + return JsonResponse({'item_id': item.id}) diff --git a/podcasts/shownotes.py b/podcasts/shownotes.py index 3740e3d..c39c777 100644 --- a/podcasts/shownotes.py +++ b/podcasts/shownotes.py @@ -37,6 +37,9 @@ ALLOWED_ATTRS = { URL_ATTRS = {'href', 'src'} LINK_SCHEMES = {'http', 'https', 'mailto'} IMAGE_SCHEMES = {'http', 'https'} +# A link to a second in the episode ("#t=754"), as in the article of a transcript +# (docs/transcription.md): the only fragment link that is kept. +TIME_LINK = re.compile(r'^#t=\d{1,6}$') _CONTROL_AND_SPACE = re.compile(r'[\x00-\x20\x7f]+') _SCHEME = re.compile(r'^([a-z][a-z0-9+.\-]*):') @@ -81,7 +84,9 @@ class _Sanitizer(HTMLParser): name = name.lower() if name not in allowed or value is None: continue - if name in URL_ATTRS: + if tag == 'a' and name == 'href' and TIME_LINK.match(value.strip()): + value = value.strip() + elif name in URL_ATTRS: value = _safe_url(value, IMAGE_SCHEMES if tag == 'img' else LINK_SCHEMES) if value is None: continue diff --git a/podcasts/test_article.py b/podcasts/test_article.py new file mode 100644 index 0000000..68c0055 --- /dev/null +++ b/podcasts/test_article.py @@ -0,0 +1,146 @@ +import json + +from django.contrib.auth.models import User +from django.test import TestCase +from django.utils import timezone + +from feeds.models import Item, Source + +from .models import EpisodeArticle, PodcastEpisode, PodcastFeed, TranscriptionJob + +BODY = ('

Zusammenfassung

Teil eins

' + '

▶ ab 12:34 und Quelle

') + + +class ArticleTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username='alice', password='pw12345678') + self.client.force_login(self.user) + self.feed = PodcastFeed.objects.create(user=self.user, rss_url='https://p.example/rss', title='Salon', + author='Anna Autorin', artwork_url='https://p.example/feed.jpg') + self.ep = PodcastEpisode.objects.create(feed=self.feed, guid='e1', title='Folge 1', + audio_url='https://p.example/e1.mp3', artwork_url='https://p.example/e1.jpg') + + def url(self, ep=None): + return f'/podcasts/episodes/{(ep or self.ep).id}/article/' + + def post(self, data=None, ep=None): + data = {'title': 'Ein Artikel', 'summary': 'Kurz.', 'body_html': BODY, 'model': 'claude', 'kind': 'Interview'} \ + if data is None else data + return self.client.post(self.url(ep), json.dumps(data), content_type='application/json') + + def test_requires_login_and_ownership(self): + self.client.logout() + self.assertEqual(self.post().status_code, 401) + self.assertEqual(self.client.delete(self.url()).status_code, 401) + bob = User.objects.create_user(username='bob', password='pw12345678') + self.client.force_login(bob) + self.assertEqual(self.post().status_code, 404) + self.assertEqual(self.client.delete(self.url()).status_code, 404) + self.assertFalse(Item.objects.exists()) + + def test_create(self): + r = self.post() + self.assertEqual(r.status_code, 200) + item = Item.objects.get(pk=r.json()['item_id']) + self.assertEqual(item.user, self.user) + self.assertEqual(item.kind, Source.BLOG) + self.assertEqual(item.title, 'Ein Artikel') + self.assertEqual(item.summary, 'Kurz.') + self.assertEqual(item.author, 'Anna Autorin') + self.assertEqual(item.image_url, 'https://p.example/e1.jpg') + self.assertEqual(item.url, 'https://p.example/e1.mp3') + self.assertTrue(item.content_full) + self.assertGreater(item.words, 3) + self.assertLess((timezone.now() - item.published).total_seconds(), 60) + art = EpisodeArticle.objects.get(episode=self.ep) + self.assertEqual((art.item, art.model, art.kind), (item, 'claude', 'Interview')) + self.assertEqual(item.episode_article, art) + + def test_image_falls_back_to_feed_artwork(self): + self.ep.artwork_url = '' + self.ep.save() + item = Item.objects.get(pk=self.post().json()['item_id']) + self.assertEqual(item.image_url, 'https://p.example/feed.jpg') + + def test_sanitizer_keeps_time_links_only(self): + item = Item.objects.get(pk=self.post().json()['item_id']) + self.assertNotIn('ab'}) + content = Item.objects.get(pk=evil.json()['item_id']).content + self.assertNotIn('onfocus', content) + self.assertNotIn('#top', content) + + def test_source_made_once(self): + other = PodcastEpisode.objects.create(feed=self.feed, guid='e2', title='Folge 2', audio_url='https://p.example/e2.mp3') + self.post() + self.post(ep=other) + src = Source.objects.get(user=self.user, url='diora://podcast-artikel') + self.assertEqual((src.title, src.kind, src.fav, src.synthetic), ('Podcast-Artikel', Source.BLOG, True, True)) + self.assertEqual(Source.objects.count(), 1) + self.assertEqual(src.items.count(), 2) + + def test_replace_keeps_one_item(self): + first = self.post().json()['item_id'] + Item.objects.filter(pk=first).update(read_at=timezone.now(), progress=1) + second = self.post({'title': 'Neu', 'body_html': '

Ganz anderer Text

', 'model': 'm2'}).json()['item_id'] + self.assertEqual(first, second) + self.assertEqual(Item.objects.count(), 1) + item = Item.objects.get() + self.assertEqual((item.title, item.read_at, item.progress), ('Neu', None, 0)) + self.assertIn('Ganz anderer Text', item.content) + self.assertEqual(EpisodeArticle.objects.get().model, 'm2') + + def test_delete(self): + self.post() + self.assertEqual(self.client.delete(self.url()).status_code, 200) + self.assertFalse(Item.objects.exists()) + self.assertFalse(EpisodeArticle.objects.exists()) + self.assertEqual(Source.objects.count(), 1) + self.assertEqual(self.client.delete(self.url()).status_code, 200) # again: nothing to do + + def test_validation(self): + for data in ({}, {'title': '', 'body_html': BODY}, {'title': 'x' * 301, 'body_html': BODY}, + {'title': 'x'}, {'title': 'x', 'body_html': ''}, + {'title': 'x', 'body_html': 'a' * 1_000_001}, + {'title': 'x', 'body_html': BODY, 'model': 'm' * 101}): + self.assertEqual(self.post(data).status_code, 400, data and list(data)) + self.assertEqual(self.client.post(self.url(), 'nope', content_type='application/json').status_code, 400) + self.assertFalse(Item.objects.exists()) + self.assertEqual(self.client.get(self.url()).status_code, 405) + + def test_saving_clears_the_wish(self): + TranscriptionJob.objects.create(episode=self.ep, state='done', want_article=True, claimed_by='phone', + claimed_until=timezone.now() + timezone.timedelta(minutes=10)) + self.post() + job = TranscriptionJob.objects.get() + self.assertEqual((job.want_article, job.claimed_by, job.claimed_until), (False, '', None)) + + def test_in_favourite_blogs_with_episode_id(self): + item_id = self.post().json()['item_id'] + items = self.client.get('/feeds/blogs/?fav=1').json()['items'] + self.assertEqual([i['id'] for i in items], [item_id]) + self.assertEqual(items[0]['episode_id'], self.ep.id) + detail = self.client.get(f'/feeds/items/{item_id}/').json()['item'] + self.assertEqual(detail['episode_id'], self.ep.id) + self.assertIn('href="#t=754"', detail['content']) + + def test_ordinary_items_have_no_episode_id(self): + src = Source.objects.create(user=self.user, url='https://b.example/feed', title='B', kind=Source.BLOG, fav=True) + Item.objects.create(source=src, user=self.user, kind=Source.BLOG, guid_hash='g', title='T', + published=timezone.now(), content='

x

', content_full=True) + self.post() + items = self.client.get('/feeds/blogs/?fav=1').json()['items'] + self.assertEqual(len(items), 2) + self.assertEqual(sorted('episode_id' in i for i in items), [False, True]) + + def test_blog_list_needs_no_query_per_article(self): + for n in range(5): + self.post(ep=PodcastEpisode.objects.create(feed=self.feed, guid=f'x{n}', title=str(n), + audio_url='https://p.example/x.mp3')) + with self.assertNumQueries(3): + self.client.get('/feeds/blogs/?fav=1') diff --git a/podcasts/urls.py b/podcasts/urls.py index 6ffc0d7..2c32eed 100644 --- a/podcasts/urls.py +++ b/podcasts/urls.py @@ -1,5 +1,5 @@ from django.urls import path -from . import audiobooks, discover, folders, radio, transcription, views +from . import article, audiobooks, discover, folders, radio, transcription, views urlpatterns = [ path('search/', views.podcast_search, name='podcast_search'), @@ -28,6 +28,7 @@ urlpatterns = [ path('episodes/discard/', views.discard_episodes, name='podcast_discard_episodes'), path('episodes//', views.episode_detail, name='podcast_episode_detail'), path('episodes//transcript/', views.episode_transcript, name='podcast_episode_transcript'), + path('episodes//article/', article.episode_article, name='podcast_episode_article'), path('episodes//silences/', views.episode_silences, name='podcast_episode_silences'), path('episodes//sponsor/', views.episode_sponsor, name='podcast_episode_sponsor'), path('episodes//favorite/', views.set_favorite, name='podcast_set_favorite'), diff --git a/static/js/app/65-article.js b/static/js/app/65-article.js index 1458159..9c4d1ea 100644 --- a/static/js/app/65-article.js +++ b/static/js/app/65-article.js @@ -26,7 +26,8 @@ const href = a.getAttribute('href'); a.removeAttribute('href'); if (href.length < 2) return; - if (!_isNoteBacklink(a) && _looksLikeFootnoteLink(a, href)) { + // "#t=754": a second in a podcast's article, never a note. + if (!/^#t=\d+$/.test(href) && !_isNoteBacklink(a) && _looksLikeFootnoteLink(a, href)) { a.dataset.footnoteRef = href.slice(1); a.dataset.footnoteHref = href; a.style.cursor = 'pointer'; diff --git a/static/js/feeds.js b/static/js/feeds.js index b2c88e2..5c81bd1 100644 --- a/static/js/feeds.js +++ b/static/js/feeds.js @@ -720,6 +720,10 @@ if (story) h += '
' + (story.overview ? overviewHtml(story) : '') + '
'; const content = i.content || ''; if (i.image_url && content.indexOf(i.image_url) < 0) h += ''; + if (i.episode_id) { + // The article of a podcast episode (docs/transcription.md): to the episode itself. + h += '

'; + } h += '
' + (content || '

' + esc(i.summary) + '

') + '
'; if (i.paywalled) { h += '

Nur der freie Teil des Artikels. Mit deiner Abo-Anmeldung (⚙ → ' + esc(i.source_title) + @@ -800,12 +804,72 @@ }); } + // ---- the episode of a podcast article (docs/transcription.md) + + function podcastHost() { + return window.DioraPodcastHost || window.DioraNative || null; + } + + function parseJson(text, fallback) { + try { return JSON.parse(text); } catch (e) { return fallback; } + } + + // Play the episode from `seconds` (null: open its page, where ▶ starts it). The browser and the + // diora desktop have the host of podcasts-web.js; the apps' bridge has the same calls + // (podcastPlayEpisodeAt is the one that starts at a place; without it the episode starts at its own place). + async function playEpisode(episodeId, seconds) { + const host = podcastHost(); + if (!host || !host.playerState) { alert('Der Podcast-Bereich ist hier nicht verfügbar.'); return; } + const now = parseJson(host.playerState(), {}); + const ms = seconds == null ? null : Math.max(0, Math.round(seconds * 1000)); + let opened = false; + if (now.kind === 'podcast' && now.episodeId === episodeId) { + if (ms != null && host.podcastSeek) host.podcastSeek(ms); + if (ms != null && !now.playing && host.podcastToggle) host.podcastToggle(); + opened = ms != null; + } else { + let d; + try { + d = await api('/podcasts/episodes/' + episodeId + '/'); + } catch (e) { + alert(e.message === 'Failed to fetch' ? 'Keine Verbindung.' : 'Die Folge ist nicht mehr da.'); + return; + } + const row = {id: d.id, title: d.title, audio_url: d.audio_url, duration_seconds: d.duration_seconds, + artwork_url: d.artwork_url, pub_date: d.pub_date, position_seconds: d.position_seconds, + feed__id: d.feed.id, feed__title: d.feed.title, feed__artwork_url: d.feed.artwork_url}; + closeReader(); + if (ms != null && host.podcastPlayEpisodeAt) { + host.podcastPlayEpisodeAt(JSON.stringify(row), now.episodeId || 0, ms); + opened = true; + } else if (ms == null && window.__dioraPodcastOpenEpisode && window.__dioraPodcastOpenEpisode(row)) { + return; + } else if (host.podcastPlayEpisode) { + host.podcastPlayEpisode(JSON.stringify(row), now.episodeId || 0); + opened = true; + } + } + closeReader(); + if (opened && window.__dioraOpenPlayer) window.__dioraOpenPlayer('podcast'); + } + // Clicks in the article (65-article.js hands them over): its other reports, the next article. function onReaderClick(e) { + // A time mark in a podcast article ("▶ ab 12:34", href="#t=754"; the reader keeps it as data-internal-href). + const t = e.target.closest('a[data-internal-href^="#t="], a[href^="#t="]'); + if (t && readerItem && readerItem.episode_id) { + const sec = parseInt((t.dataset.internalHref || t.getAttribute('href')).slice(3), 10); + if (sec >= 0) { + e.preventDefault(); + playEpisode(readerItem.episode_id, sec); + } + return; + } const b = e.target.closest('[data-r]'); if (!b) return; const r = b.dataset.r; - if (r === 'item') openItem(parseInt(b.dataset.id, 10), readerStory); + if (r === 'episode') playEpisode(parseInt(b.dataset.id, 10), null); + else if (r === 'item') openItem(parseInt(b.dataset.id, 10), readerStory); else if (r === 'story') { const other = readerStory && readerStory.overview && (readerStory.overview.related || []).find(x => x.id === parseInt(b.dataset.id, 10)); if (other) openItem((other.lead || other.items[0]).id, other); @@ -1464,6 +1528,7 @@ und Ordner. Schlage nur Änderungen vor, die wirklich helfen; eine Quelle ohne s '.fd-artview .fd-art-title { font-size: 1.45em; line-height: 1.25; margin: 10px 0 6px; font-weight: 700; }', '.fd-artview .fd-art-meta { font-family: var(--font); font-size: 12px; color: var(--nui-muted, #9a9a9a); margin: 0 0 14px; }', '.fd-artview .fd-art-img, .fd-artview .fd-art-body img { display: block; max-width: 100%; height: auto; margin: 12px auto; border-radius: 4px; }', + '.fd-artview .fd-art-listen { margin: 0 0 12px; }', '.fd-artview .fd-art-body a { color: var(--accent, #e63946); }', '.fd-artview .fd-art-body figure { margin: 12px 0; }', '.fd-artview .fd-art-body figcaption { font-size: 0.8em; color: var(--nui-muted, #9a9a9a); }',