Smarte Transkription D: Artikel-Endpunkt und Blog-Anbindung

POST/DELETE /podcasts/episodes/<id>/article/ (podcasts/article.py): legt die Quelle Podcast-Artikel an und erzeugt oder ersetzt das Item der Folge; Sanitizer behält href=#t=<Sekunden>; _item_data und Item-Ansicht liefern episode_id; feeds.js: Folge anhören und Zeitmarken-Links.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
mrwnslz 2026-11-09 09:36:34 +01:00
parent 69442f4a4d
commit bd09d81366
8 changed files with 358 additions and 8 deletions

View file

@ -91,6 +91,13 @@ author as `author`, the episode's artwork as `image_url`, now as `published` (so
the blog stream under "heute"), `words` counted. Saving clears the wish. `DELETE` on the same
address removes the article and its item. `GET /feeds/items/<id>/` and `_item_data` carry
`episode_id` for such items.
`body_html` may contain `href="#t=<seconds>"` links; `clean_html` keeps exactly this fragment form
(`podcasts/shownotes.py` `TIME_LINK`) and no other `#` links. Validation: `title` 1 to 300
characters, `body_html` non-empty after cleaning and at most 1,000,000 characters, `summary` at most
2000, `model` at most 100, `kind` at most 20 (400 otherwise). The item has `content_full` true (no
page is ever fetched) and, as the reader's furniture filter would strip repeated "▶ ab" blocks,
`GET /feeds/items/<id>/` skips that filter for synthetic sources. The endpoint is
`podcasts/article.py`.
Synthetic sources are skipped by the feed refresh (`refresh_newsfeeds`, `refresh_now`) and by the
pruning of old blog items (`feeds/refresh.py`). They can be hidden or have their bell turned off

View file

@ -108,6 +108,11 @@ def _item_data(i, content=False):
'summary': i.summary, 'image_url': i.image_url, 'read': i.read_at is not None, 'progress': i.progress,
'starred': i.starred, 'story': i.story_id,
}
if i.source.synthetic:
# The article of a podcast episode (docs/transcription.md); the lists fetch it with the item.
art = getattr(i, 'episode_article', None)
if art is not None:
d['episode_id'] = art.episode_id
if content:
d['content'] = i.content
d['content_full'] = i.content_full
@ -613,7 +618,7 @@ def story_overview(request, pk):
def blogs(request):
if (err := _auth(request)):
return err
qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source')
qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source', 'episode_article')
if request.GET.get('source', '').isdigit():
qs = qs.filter(source_id=int(request.GET['source']))
elif 'folder' in request.GET:
@ -642,7 +647,7 @@ def blogs(request):
def starred(request):
if (err := _auth(request)):
return err
items = Item.objects.filter(user=request.user, starred=True).select_related('source').order_by('-published')
items = Item.objects.filter(user=request.user, starred=True).select_related('source', 'episode_article').order_by('-published')
return JsonResponse({'items': [_item_data(i) for i in items[:200]]})
@ -651,13 +656,14 @@ def starred(request):
def item(request, pk):
if (err := _auth(request)):
return err
i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source').first()
i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source', 'episode_article').first()
if i is None:
return JsonResponse({'error': 'not found'}, status=404)
if refresh.needs_article(i, i.source):
refresh.load_article(i)
d = _item_data(i, content=True)
d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source))
if not i.source.synthetic:
d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source))
return JsonResponse({'item': d})

119
podcasts/article.py Normal file
View file

@ -0,0 +1,119 @@
"""The article written from an episode's transcript (docs/transcription.md): its text
is a feeds.Item in the user's favourite blog "Podcast-Artikel"."""
import hashlib
import json
from django.db import transaction
from django.http import JsonResponse
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from feeds import fetch
from feeds.models import Item, Source
from .models import EpisodeArticle, PodcastEpisode, TranscriptionJob
SOURCE_URL = 'diora://podcast-artikel'
SOURCE_TITLE = 'Podcast-Artikel'
TITLE_MAX = 300
SUMMARY_MAX = 2000
BODY_MAX = 1_000_000 # characters of body_html
MODEL_MAX = 100
KIND_MAX = 20
def _auth(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
return None
def _body(request):
try:
body = json.loads(request.body or b'{}')
except (ValueError, UnicodeDecodeError):
return None
return body if isinstance(body, dict) else None
def article_source(user):
"""The user's blog "Podcast-Artikel", made on first use."""
source, _ = Source.objects.get_or_create(user=user, url=SOURCE_URL, defaults={
'title': SOURCE_TITLE, 'kind': Source.BLOG, 'fav': True, 'synthetic': True})
return source
def _text(value, limit):
return value.strip() if isinstance(value, str) and len(value.strip()) <= limit else None
@csrf_exempt
@require_http_methods(['POST', 'DELETE'])
def episode_article(request, pk):
if (err := _auth(request)):
return err
ep = PodcastEpisode.objects.filter(pk=pk, feed__user=request.user).select_related('feed').first()
if ep is None:
return JsonResponse({'error': 'not found'}, status=404)
if request.method == 'DELETE':
art = EpisodeArticle.objects.filter(episode=ep).select_related('item').first()
if art is not None:
art.item.delete() # the article goes with it (cascade)
return JsonResponse({'ok': True})
body = _body(request)
if body is None:
return JsonResponse({'error': 'invalid JSON'}, status=400)
title = _text(body.get('title'), TITLE_MAX)
if not title:
return JsonResponse({'error': 'title needed (at most %d characters)' % TITLE_MAX}, status=400)
html_in = body.get('body_html')
if not isinstance(html_in, str) or not html_in.strip():
return JsonResponse({'error': 'body_html needed'}, status=400)
if len(html_in) > BODY_MAX:
return JsonResponse({'error': 'body_html too large'}, status=400)
summary = body.get('summary') or ''
if not isinstance(summary, str) or len(summary) > SUMMARY_MAX:
return JsonResponse({'error': 'summary too long'}, status=400)
model = body.get('model') or ''
kind = body.get('kind') or ''
if not isinstance(model, str) or len(model) > MODEL_MAX or not isinstance(kind, str) or len(kind) > KIND_MAX:
return JsonResponse({'error': 'model or kind too long'}, status=400)
url = getattr(ep, 'link', '') or ep.audio_url
content = fetch.clean_html(html_in, url)
if not fetch.plain(content):
return JsonResponse({'error': 'body_html empty'}, status=400)
fields = {
'url': url[:2000],
'title': title,
'author': (ep.feed.author or '')[:300],
'published': timezone.now(),
'summary': fetch.plain(summary),
'content': content,
'content_full': True, # nothing to fetch from a page
'words': fetch.word_count(content),
'image_url': (ep.artwork_url or ep.feed.artwork_url or '')[:2000],
'read_at': None, 'progress': 0, 'progress_at': None,
}
with transaction.atomic():
source = article_source(request.user)
art = EpisodeArticle.objects.filter(episode=ep).select_related('item').first()
if art is not None:
item = art.item
for k, v in fields.items():
setattr(item, k, v)
item.source = source
item.save()
art.model, art.kind = model, kind
art.save(update_fields=['model', 'kind'])
else:
guid = hashlib.sha256(f'diora-article:{ep.pk}'.encode()).hexdigest()
item = Item.objects.create(source=source, user=request.user, kind=Source.BLOG,
guid_hash=guid, **fields)
EpisodeArticle.objects.create(episode=ep, item=item, model=model, kind=kind)
# Saving fulfils the wish and frees the reservation.
TranscriptionJob.objects.filter(episode=ep).update(want_article=False, claimed_by='', claimed_until=None)
return JsonResponse({'item_id': item.id})

View file

@ -37,6 +37,9 @@ ALLOWED_ATTRS = {
URL_ATTRS = {'href', 'src'}
LINK_SCHEMES = {'http', 'https', 'mailto'}
IMAGE_SCHEMES = {'http', 'https'}
# A link to a second in the episode ("#t=754"), as in the article of a transcript
# (docs/transcription.md): the only fragment link that is kept.
TIME_LINK = re.compile(r'^#t=\d{1,6}$')
_CONTROL_AND_SPACE = re.compile(r'[\x00-\x20\x7f]+')
_SCHEME = re.compile(r'^([a-z][a-z0-9+.\-]*):')
@ -81,7 +84,9 @@ class _Sanitizer(HTMLParser):
name = name.lower()
if name not in allowed or value is None:
continue
if name in URL_ATTRS:
if tag == 'a' and name == 'href' and TIME_LINK.match(value.strip()):
value = value.strip()
elif name in URL_ATTRS:
value = _safe_url(value, IMAGE_SCHEMES if tag == 'img' else LINK_SCHEMES)
if value is None:
continue

146
podcasts/test_article.py Normal file
View file

@ -0,0 +1,146 @@
import json
from django.contrib.auth.models import User
from django.test import TestCase
from django.utils import timezone
from feeds.models import Item, Source
from .models import EpisodeArticle, PodcastEpisode, PodcastFeed, TranscriptionJob
BODY = ('<p>Zusammenfassung <script>alert(1)</script></p><h2>Teil eins</h2>'
'<p><a href="#t=754" onclick="evil()">▶ ab 12:34</a> und <a href="https://example.org/x">Quelle</a></p>')
class ArticleTests(TestCase):
def setUp(self):
self.user = User.objects.create_user(username='alice', password='pw12345678')
self.client.force_login(self.user)
self.feed = PodcastFeed.objects.create(user=self.user, rss_url='https://p.example/rss', title='Salon',
author='Anna Autorin', artwork_url='https://p.example/feed.jpg')
self.ep = PodcastEpisode.objects.create(feed=self.feed, guid='e1', title='Folge 1',
audio_url='https://p.example/e1.mp3', artwork_url='https://p.example/e1.jpg')
def url(self, ep=None):
return f'/podcasts/episodes/{(ep or self.ep).id}/article/'
def post(self, data=None, ep=None):
data = {'title': 'Ein Artikel', 'summary': 'Kurz.', 'body_html': BODY, 'model': 'claude', 'kind': 'Interview'} \
if data is None else data
return self.client.post(self.url(ep), json.dumps(data), content_type='application/json')
def test_requires_login_and_ownership(self):
self.client.logout()
self.assertEqual(self.post().status_code, 401)
self.assertEqual(self.client.delete(self.url()).status_code, 401)
bob = User.objects.create_user(username='bob', password='pw12345678')
self.client.force_login(bob)
self.assertEqual(self.post().status_code, 404)
self.assertEqual(self.client.delete(self.url()).status_code, 404)
self.assertFalse(Item.objects.exists())
def test_create(self):
r = self.post()
self.assertEqual(r.status_code, 200)
item = Item.objects.get(pk=r.json()['item_id'])
self.assertEqual(item.user, self.user)
self.assertEqual(item.kind, Source.BLOG)
self.assertEqual(item.title, 'Ein Artikel')
self.assertEqual(item.summary, 'Kurz.')
self.assertEqual(item.author, 'Anna Autorin')
self.assertEqual(item.image_url, 'https://p.example/e1.jpg')
self.assertEqual(item.url, 'https://p.example/e1.mp3')
self.assertTrue(item.content_full)
self.assertGreater(item.words, 3)
self.assertLess((timezone.now() - item.published).total_seconds(), 60)
art = EpisodeArticle.objects.get(episode=self.ep)
self.assertEqual((art.item, art.model, art.kind), (item, 'claude', 'Interview'))
self.assertEqual(item.episode_article, art)
def test_image_falls_back_to_feed_artwork(self):
self.ep.artwork_url = ''
self.ep.save()
item = Item.objects.get(pk=self.post().json()['item_id'])
self.assertEqual(item.image_url, 'https://p.example/feed.jpg')
def test_sanitizer_keeps_time_links_only(self):
item = Item.objects.get(pk=self.post().json()['item_id'])
self.assertNotIn('<script', item.content)
self.assertNotIn('alert(1)', item.content)
self.assertNotIn('onclick', item.content)
self.assertIn('href="#t=754"', item.content)
self.assertIn('href="https://example.org/x"', item.content)
evil = self.post({'title': 'x', 'body_html': '<a href="#t=1&quot; onfocus=&quot;x">a</a><a href="#top">b</a>'})
content = Item.objects.get(pk=evil.json()['item_id']).content
self.assertNotIn('onfocus', content)
self.assertNotIn('#top', content)
def test_source_made_once(self):
other = PodcastEpisode.objects.create(feed=self.feed, guid='e2', title='Folge 2', audio_url='https://p.example/e2.mp3')
self.post()
self.post(ep=other)
src = Source.objects.get(user=self.user, url='diora://podcast-artikel')
self.assertEqual((src.title, src.kind, src.fav, src.synthetic), ('Podcast-Artikel', Source.BLOG, True, True))
self.assertEqual(Source.objects.count(), 1)
self.assertEqual(src.items.count(), 2)
def test_replace_keeps_one_item(self):
first = self.post().json()['item_id']
Item.objects.filter(pk=first).update(read_at=timezone.now(), progress=1)
second = self.post({'title': 'Neu', 'body_html': '<p>Ganz anderer Text</p>', 'model': 'm2'}).json()['item_id']
self.assertEqual(first, second)
self.assertEqual(Item.objects.count(), 1)
item = Item.objects.get()
self.assertEqual((item.title, item.read_at, item.progress), ('Neu', None, 0))
self.assertIn('Ganz anderer Text', item.content)
self.assertEqual(EpisodeArticle.objects.get().model, 'm2')
def test_delete(self):
self.post()
self.assertEqual(self.client.delete(self.url()).status_code, 200)
self.assertFalse(Item.objects.exists())
self.assertFalse(EpisodeArticle.objects.exists())
self.assertEqual(Source.objects.count(), 1)
self.assertEqual(self.client.delete(self.url()).status_code, 200) # again: nothing to do
def test_validation(self):
for data in ({}, {'title': '', 'body_html': BODY}, {'title': 'x' * 301, 'body_html': BODY},
{'title': 'x'}, {'title': 'x', 'body_html': '<script>1</script>'},
{'title': 'x', 'body_html': 'a' * 1_000_001},
{'title': 'x', 'body_html': BODY, 'model': 'm' * 101}):
self.assertEqual(self.post(data).status_code, 400, data and list(data))
self.assertEqual(self.client.post(self.url(), 'nope', content_type='application/json').status_code, 400)
self.assertFalse(Item.objects.exists())
self.assertEqual(self.client.get(self.url()).status_code, 405)
def test_saving_clears_the_wish(self):
TranscriptionJob.objects.create(episode=self.ep, state='done', want_article=True, claimed_by='phone',
claimed_until=timezone.now() + timezone.timedelta(minutes=10))
self.post()
job = TranscriptionJob.objects.get()
self.assertEqual((job.want_article, job.claimed_by, job.claimed_until), (False, '', None))
def test_in_favourite_blogs_with_episode_id(self):
item_id = self.post().json()['item_id']
items = self.client.get('/feeds/blogs/?fav=1').json()['items']
self.assertEqual([i['id'] for i in items], [item_id])
self.assertEqual(items[0]['episode_id'], self.ep.id)
detail = self.client.get(f'/feeds/items/{item_id}/').json()['item']
self.assertEqual(detail['episode_id'], self.ep.id)
self.assertIn('href="#t=754"', detail['content'])
def test_ordinary_items_have_no_episode_id(self):
src = Source.objects.create(user=self.user, url='https://b.example/feed', title='B', kind=Source.BLOG, fav=True)
Item.objects.create(source=src, user=self.user, kind=Source.BLOG, guid_hash='g', title='T',
published=timezone.now(), content='<p>x</p>', content_full=True)
self.post()
items = self.client.get('/feeds/blogs/?fav=1').json()['items']
self.assertEqual(len(items), 2)
self.assertEqual(sorted('episode_id' in i for i in items), [False, True])
def test_blog_list_needs_no_query_per_article(self):
for n in range(5):
self.post(ep=PodcastEpisode.objects.create(feed=self.feed, guid=f'x{n}', title=str(n),
audio_url='https://p.example/x.mp3'))
with self.assertNumQueries(3):
self.client.get('/feeds/blogs/?fav=1')

View file

@ -1,5 +1,5 @@
from django.urls import path
from . import audiobooks, discover, folders, radio, transcription, views
from . import article, audiobooks, discover, folders, radio, transcription, views
urlpatterns = [
path('search/', views.podcast_search, name='podcast_search'),
@ -28,6 +28,7 @@ urlpatterns = [
path('episodes/discard/', views.discard_episodes, name='podcast_discard_episodes'),
path('episodes/<int:pk>/', views.episode_detail, name='podcast_episode_detail'),
path('episodes/<int:pk>/transcript/', views.episode_transcript, name='podcast_episode_transcript'),
path('episodes/<int:pk>/article/', article.episode_article, name='podcast_episode_article'),
path('episodes/<int:pk>/silences/', views.episode_silences, name='podcast_episode_silences'),
path('episodes/<int:pk>/sponsor/', views.episode_sponsor, name='podcast_episode_sponsor'),
path('episodes/<int:pk>/favorite/', views.set_favorite, name='podcast_set_favorite'),

View file

@ -26,7 +26,8 @@
const href = a.getAttribute('href');
a.removeAttribute('href');
if (href.length < 2) return;
if (!_isNoteBacklink(a) && _looksLikeFootnoteLink(a, href)) {
// "#t=754": a second in a podcast's article, never a note.
if (!/^#t=\d+$/.test(href) && !_isNoteBacklink(a) && _looksLikeFootnoteLink(a, href)) {
a.dataset.footnoteRef = href.slice(1);
a.dataset.footnoteHref = href;
a.style.cursor = 'pointer';

View file

@ -720,6 +720,10 @@
if (story) h += '<div class="fd-ov">' + (story.overview ? overviewHtml(story) : '') + '</div>';
const content = i.content || '';
if (i.image_url && content.indexOf(i.image_url) < 0) h += '<img class="fd-art-img" alt="" src="' + esc(i.image_url) + '">';
if (i.episode_id) {
// The article of a podcast episode (docs/transcription.md): to the episode itself.
h += '<p class="fd-art-listen"><button type="button" class="btn" data-r="episode" data-id="' + i.episode_id + '">▶ Folge anhören</button></p>';
}
h += '<div class="fd-art-body">' + (content || '<p>' + esc(i.summary) + '</p>') + '</div>';
if (i.paywalled) {
h += '<p class="fd-note">Nur der freie Teil des Artikels. Mit deiner Abo-Anmeldung (⚙ → ' + esc(i.source_title) +
@ -800,12 +804,72 @@
});
}
// ---- the episode of a podcast article (docs/transcription.md)
function podcastHost() {
return window.DioraPodcastHost || window.DioraNative || null;
}
function parseJson(text, fallback) {
try { return JSON.parse(text); } catch (e) { return fallback; }
}
// Play the episode from `seconds` (null: open its page, where ▶ starts it). The browser and the
// diora desktop have the host of podcasts-web.js; the apps' bridge has the same calls
// (podcastPlayEpisodeAt is the one that starts at a place; without it the episode starts at its own place).
async function playEpisode(episodeId, seconds) {
const host = podcastHost();
if (!host || !host.playerState) { alert('Der Podcast-Bereich ist hier nicht verfügbar.'); return; }
const now = parseJson(host.playerState(), {});
const ms = seconds == null ? null : Math.max(0, Math.round(seconds * 1000));
let opened = false;
if (now.kind === 'podcast' && now.episodeId === episodeId) {
if (ms != null && host.podcastSeek) host.podcastSeek(ms);
if (ms != null && !now.playing && host.podcastToggle) host.podcastToggle();
opened = ms != null;
} else {
let d;
try {
d = await api('/podcasts/episodes/' + episodeId + '/');
} catch (e) {
alert(e.message === 'Failed to fetch' ? 'Keine Verbindung.' : 'Die Folge ist nicht mehr da.');
return;
}
const row = {id: d.id, title: d.title, audio_url: d.audio_url, duration_seconds: d.duration_seconds,
artwork_url: d.artwork_url, pub_date: d.pub_date, position_seconds: d.position_seconds,
feed__id: d.feed.id, feed__title: d.feed.title, feed__artwork_url: d.feed.artwork_url};
closeReader();
if (ms != null && host.podcastPlayEpisodeAt) {
host.podcastPlayEpisodeAt(JSON.stringify(row), now.episodeId || 0, ms);
opened = true;
} else if (ms == null && window.__dioraPodcastOpenEpisode && window.__dioraPodcastOpenEpisode(row)) {
return;
} else if (host.podcastPlayEpisode) {
host.podcastPlayEpisode(JSON.stringify(row), now.episodeId || 0);
opened = true;
}
}
closeReader();
if (opened && window.__dioraOpenPlayer) window.__dioraOpenPlayer('podcast');
}
// Clicks in the article (65-article.js hands them over): its other reports, the next article.
function onReaderClick(e) {
// A time mark in a podcast article ("▶ ab 12:34", href="#t=754"; the reader keeps it as data-internal-href).
const t = e.target.closest('a[data-internal-href^="#t="], a[href^="#t="]');
if (t && readerItem && readerItem.episode_id) {
const sec = parseInt((t.dataset.internalHref || t.getAttribute('href')).slice(3), 10);
if (sec >= 0) {
e.preventDefault();
playEpisode(readerItem.episode_id, sec);
}
return;
}
const b = e.target.closest('[data-r]');
if (!b) return;
const r = b.dataset.r;
if (r === 'item') openItem(parseInt(b.dataset.id, 10), readerStory);
if (r === 'episode') playEpisode(parseInt(b.dataset.id, 10), null);
else if (r === 'item') openItem(parseInt(b.dataset.id, 10), readerStory);
else if (r === 'story') {
const other = readerStory && readerStory.overview && (readerStory.overview.related || []).find(x => x.id === parseInt(b.dataset.id, 10));
if (other) openItem((other.lead || other.items[0]).id, other);
@ -1464,6 +1528,7 @@ und Ordner. Schlage nur Änderungen vor, die wirklich helfen; eine Quelle ohne s
'.fd-artview .fd-art-title { font-size: 1.45em; line-height: 1.25; margin: 10px 0 6px; font-weight: 700; }',
'.fd-artview .fd-art-meta { font-family: var(--font); font-size: 12px; color: var(--nui-muted, #9a9a9a); margin: 0 0 14px; }',
'.fd-artview .fd-art-img, .fd-artview .fd-art-body img { display: block; max-width: 100%; height: auto; margin: 12px auto; border-radius: 4px; }',
'.fd-artview .fd-art-listen { margin: 0 0 12px; }',
'.fd-artview .fd-art-body a { color: var(--accent, #e63946); }',
'.fd-artview .fd-art-body figure { margin: 12px 0; }',
'.fd-artview .fd-art-body figcaption { font-size: 0.8em; color: var(--nui-muted, #9a9a9a); }',