mrwnslz/diora-web
1
0
Fork 0
Code Issues 4 Pull requests Projects Releases Packages Wiki Activity Actions

Merge origin/master in master
Some checks failed
Build and push Docker image / build (push) Successful in 17s
Details
Test / test (push) Successful in 10m58s
Details
Test / e2e (push) Failing after 1m33s
Details

Browse source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
marwin 2026-10-26 11:48:23 +01:00
commit 14a45f4cb4
208 changed files with 49009 additions and 7721 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
.forgejo/workflows/test.yml
View file

@ -29,7 +29,40 @@ jobs:
/tmp/venv/bin/pip install --no-cache-dir -r requirements.txt
- name: Django system check
run: /tmp/venv/bin/python manage.py check
# The SQLite file lives in data/ (a volume in production, git-ignored).
# Since the podcast models have JSON fields, the check opens the
# database to ask whether it supports them — so the folder must exist.
run: |
mkdir -p data
/tmp/venv/bin/python manage.py check
- name: Run tests
run: /tmp/venv/bin/python manage.py test
e2e:
# The browser tests (e2e/, docs/refactoring.md): a real runserver on its own
# throwaway database, a book uploaded, read, highlighted, the theme switched.
runs-on: ubuntu-latest
container:
image: catthehacker/ubuntu:act-24.04
steps:
- uses: actions/checkout@v4
- name: Install Python dependencies
run: |
python -m venv /tmp/venv
/tmp/venv/bin/pip install --no-cache-dir -r requirements.txt
- name: Install Playwright and Chromium
run: |
npm ci
npx playwright install --with-deps chromium
- name: Browser tests
env:
CI: '1'
E2E_PYTHON: /tmp/venv/bin/python
run: |
mkdir -p data
npx playwright test

5
accounts/middleware.py
View file

@ -22,4 +22,9 @@ class ApiTokenAuthMiddleware:
api_token = ApiToken.objects.select_related('user').filter(token=token).first()
if api_token is not None:
request.user = api_token.user
# Browsers never attach this header on their own, so a
# token request can't be a cross-site forgery; native
# clients have no CSRF cookie to send. (CsrfViewMiddleware
# checks this flag in process_view, which runs after us.)
request._dont_enforce_csrf_checks = True
return self.get_response(request)

33
accounts/migrations/0006_webdav_inbox.py Normal file
View file

@ -0,0 +1,33 @@
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('accounts', '0005_webdavsource'),
]
operations = [
migrations.AddField(
model_name='webdavsource',
name='inbox',
field=models.BooleanField(default=False),
),
migrations.CreateModel(
name='WebDAVSeen',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('path', models.CharField(max_length=1000)),
('size', models.BigIntegerField(default=0)),
('modified', models.CharField(blank=True, max_length=100)),
('claimed_at', models.DateTimeField(blank=True, null=True)),
('done_at', models.DateTimeField(blank=True, null=True)),
('result', models.CharField(blank=True, max_length=20)),
('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='seen', to='accounts.webdavsource')),
],
options={
'unique_together': {('source', 'path')},
},
),
]

28
accounts/migrations/0007_uservault.py Normal file
View file

@ -0,0 +1,28 @@
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('accounts', '0006_webdav_inbox'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='UserVault',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=40)),
('ct', models.TextField()),
('iv', models.CharField(max_length=64)),
('updated_at', models.DateTimeField(auto_now=True)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='vault', to=settings.AUTH_USER_MODEL)),
],
options={
'unique_together': {('user', 'name')},
},
),
]

33
accounts/models.py
View file

@ -56,6 +56,9 @@ class WebDAVSource(models.Model):
username = models.CharField(max_length=200, blank=True)
password = models.CharField(max_length=500, blank=True)
root_path = models.CharField(max_length=500, blank=True, default='')
# "Eingang" (docs/library.md): new books in it come into the library by themselves —
# a device with the key fetches, encrypts and uploads them (WebDAVSeen keeps which).
inbox = models.BooleanField(default=False)
created_at = models.DateTimeField(auto_now_add=True)
last_used_at = models.DateTimeField(null=True, blank=True)
@ -89,6 +92,36 @@ class WebDAVSource(models.Model):
return url
class WebDAVSeen(models.Model):
"""A file of an "Eingang" source a device took (or is taking) into the library: taken
files aren't offered again; a claim keeps two devices from taking the same file at once
and runs out when a device didn't finish."""
source = models.ForeignKey(WebDAVSource, on_delete=models.CASCADE, related_name='seen')
path = models.CharField(max_length=1000)
size = models.BigIntegerField(default=0)
modified = models.CharField(max_length=100, blank=True)
claimed_at = models.DateTimeField(null=True, blank=True)
done_at = models.DateTimeField(null=True, blank=True)
result = models.CharField(max_length=20, blank=True) # imported | known | too_big | failed
class Meta:
unique_together = ('source', 'path')
class UserVault(models.Model):
"""Small private settings of the devices, end-to-end encrypted with the book key
(docs/ai.md: the Anthropic API key of the "KI" features). The server keeps the blob,
one per name, and never sees what is in it."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='vault')
name = models.CharField(max_length=40)
ct = models.TextField()
iv = models.CharField(max_length=64)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
unique_together = ('user', 'name')
def _generate_token():
return secrets.token_hex(32)

33
accounts/sync.py
View file

@ -3,7 +3,8 @@ from django.http import JsonResponse
from django.utils import timezone
from django.views.decorators.http import require_http_methods
from books.models import EBook, EBookProgress, EBookHighlights, EBookBookmarks
from books.models import EBook, EBookProgress, EBookHighlights, EBookBookmarks, book_rows
from books.views import _now_ms, progress_state
from podcasts.models import PodcastFeed, EpisodeProgress, PodcastQueue
from radio.models import SavedStation
@ -19,21 +20,22 @@ def sync_snapshot(request):
endpoints (e.g. GET /books/<id>/data/), which this same token also
unlocks. Writes (progress, highlights, bookmarks, ...) go through those
existing endpoints too, to reuse their merge semantics rather than
duplicating it here."""
duplicating it here.
?only=books: just the books' part. The apps' book sync needs nothing else,
and the podcast part is by far the biggest (every episode ever played:
megabytes for an account that moved over from AntennaPod)."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
user = request.user
books = list(EBook.objects.filter(user=user).values('id', 'meta_ct', 'meta_iv', 'uploaded_at', 'is_read'))
for b in books:
b['uploaded_at'] = b['uploaded_at'].isoformat()
books = [{k: b[k] for k in ('id', 'meta_ct', 'meta_iv', 'uploaded_at', 'is_read')} for b in book_rows(user)]
book_progress = list(
EBookProgress.objects.filter(user=user).values('book_id', 'scroll_fraction', 'position_anchor', 'updated_at')
)
for p in book_progress:
p['updated_at'] = p['updated_at'].isoformat()
book_progress = [
{'book_id': p.book_id, 'updated_at': p.updated_at.isoformat(), **progress_state(p)}
for p in EBookProgress.objects.filter(user=user)
]
book_highlights = list(EBookHighlights.objects.filter(user=user).values('book_id', 'ct', 'iv', 'updated_at'))
for h in book_highlights:
@ -43,6 +45,16 @@ def sync_snapshot(request):
for bm in book_bookmarks:
bm['updated_at'] = bm['updated_at'].isoformat()
if request.GET.get('only') == 'books':
return JsonResponse({
'server_time': timezone.now().isoformat(),
'server_now': _now_ms(),
'books': books,
'book_progress': book_progress,
'book_highlights': book_highlights,
'book_bookmarks': book_bookmarks,
})
podcast_feeds = list(
PodcastFeed.objects.filter(user=user)
.annotate(latest_episode_at=Max('episodes__pub_date'))
@ -71,6 +83,7 @@ def sync_snapshot(request):
return JsonResponse({
'server_time': timezone.now().isoformat(),
'server_now': _now_ms(),
'books': books,
'book_progress': book_progress,
'book_highlights': book_highlights,

153
accounts/tests.py
View file

@ -42,6 +42,20 @@ class ApiTokenAuthMiddlewareTests(TestCase):
resp = self.client.get('/books/', HTTP_AUTHORIZATION=f'Bearer {self.token.token}')
self.assertEqual(resp.status_code, 200)
def test_bearer_token_needs_no_csrf_token(self):
# import_m3u is not csrf_exempt: without a token it must still be protected.
from django.test import Client
client = Client(enforce_csrf_checks=True)
m3u = '#EXTM3U\n#EXTINF:-1,Test FM\nhttps://example.com/stream\n'
resp = client.post('/radio/import/', data=m3u, content_type='text/plain')
self.assertIn(resp.status_code, (302, 401, 403))
resp = client.post('/radio/import/', data=m3u, content_type='text/plain',
HTTP_AUTHORIZATION=f'Bearer {self.token.token}')
self.assertNotEqual(resp.status_code, 403)
resp = client.post('/radio/import/', data=m3u, content_type='text/plain',
HTTP_AUTHORIZATION='Bearer not-a-real-token')
self.assertIn(resp.status_code, (302, 401, 403))
def test_regenerate_view_requires_login_and_rotates_token(self):
resp = self.client.post('/accounts/api-token/regenerate/')
self.assertEqual(resp.status_code, 302)
@ -71,6 +85,12 @@ class SyncSnapshotTests(TestCase):
resp = self.client.get('/api/sync/')
self.assertEqual(resp.status_code, 401)
def test_only_books(self):
resp = self.client.get('/api/sync/?only=books', HTTP_AUTHORIZATION=f'Bearer {self.token.token}')
data = resp.json()
self.assertEqual(set(data), {'server_time', 'server_now', 'books', 'book_progress', 'book_highlights', 'book_bookmarks'})
self.assertEqual(data['book_highlights'][0]['ct'], 'hl-ct')
def test_snapshot_shape_and_data(self):
resp = self.client.get('/api/sync/', HTTP_AUTHORIZATION=f'Bearer {self.token.token}')
self.assertEqual(resp.status_code, 200)
@ -526,3 +546,136 @@ class CloudFetchCachingTests(TestCase):
with patch('books.webdav.fetch_file', return_value=b'EPUB-BYTES'):
resp = self.client.get(f'/books/cloud/{self.source.pk}/fetch/?path=Dune.epub')
self.assertEqual(resp['Cache-Control'], 'no-store')
class WebDAVEditTests(TestCase):
def setUp(self):
self.user = User.objects.create_user(username='alice', password='pw12345678')
self.other = User.objects.create_user(username='bob', password='pw12345678')
self.source = WebDAVSource.objects.create(
user=self.user, label='cloud', base_url='https://cloud.example.com',
username='alice', password='app-pw',
)
def _edit(self, pk, **data):
form = {'label': 'cloud', 'base_url': 'https://cloud.example.com', 'username': 'alice', 'root_path': ''}
form.update(data)
with patch('accounts.views.list_directory', return_value=[]):
return self.client.post(f'/accounts/webdav/{pk}/edit/', form)
def test_sets_start_folder_and_keeps_password(self):
self.client.force_login(self.user)
resp = self._edit(self.source.pk, root_path='Books', password='')
self.assertEqual(resp.status_code, 302)
self.source.refresh_from_db()
self.assertEqual(self.source.root_path, 'Books')
self.assertEqual(self.source.password, 'app-pw')
def test_new_password_replaces_the_old_one(self):
self.client.force_login(self.user)
self._edit(self.source.pk, password='new-pw')
self.source.refresh_from_db()
self.assertEqual(self.source.password, 'new-pw')
def test_name_and_url_are_required(self):
self.client.force_login(self.user)
self._edit(self.source.pk, label='', root_path='Books')
self.source.refresh_from_db()
self.assertEqual(self.source.root_path, '')
def test_foreign_source_is_not_editable(self):
self.client.force_login(self.other)
self._edit(self.source.pk, root_path='x')
self.source.refresh_from_db()
self.assertEqual(self.source.root_path, '')
def test_settings_page_does_not_show_the_password(self):
self.client.force_login(self.user)
html = self.client.get('/accounts/settings/').content.decode()
self.assertIn(f'/accounts/webdav/{self.source.pk}/edit/', html)
self.assertNotIn('app-pw', html)
class CloudInboxTests(TestCase):
"""An "Eingang" source (docs/library.md): its new books are handed out once, claimed for
one device for ten minutes, and not again once a device took them."""
def setUp(self):
self.user = User.objects.create_user('inbox', password='pw12345678')
self.client.force_login(self.user)
self.src = WebDAVSource.objects.create(user=self.user, label='Bücher', base_url='https://b.example.org/dav/',
username='d', password='x', inbox=True)
WebDAVSource.objects.create(user=self.user, label='andere', base_url='https://c.example.org/dav/')
def listing(self, path=''):
if path == '':
return [{'name': 'Sub', 'path': 'Sub', 'is_dir': True, 'size': 0, 'modified': '', 'is_book': False},
{'name': 'A - Eins (2020).epub', 'path': 'A - Eins (2020).epub', 'is_dir': False,
'size': 100, 'modified': 'Mon', 'is_book': True},
{'name': 'notiz.txt', 'path': 'notiz.txt', 'is_dir': False, 'size': 1, 'modified': '', 'is_book': False}]
return [{'name': 'B.pdf', 'path': 'Sub/B.pdf', 'is_dir': False, 'size': 200, 'modified': '', 'is_book': True}]
def inbox(self):
with patch('books.webdav.list_directory', side_effect=lambda source, path='': self.listing(path)):
return self.client.get('/books/cloud/inbox/').json()
def done(self, path, result):
return self.client.post(f'/books/cloud/{self.src.pk}/done/', json.dumps({'path': path, 'result': result}),
content_type='application/json')
def test_new_books_once_then_claimed_then_done(self):
files = self.inbox()['files']
self.assertEqual(sorted(f['path'] for f in files), ['A - Eins (2020).epub', 'Sub/B.pdf'])
self.assertEqual(self.inbox()['files'], []) # claimed for this device: no one else gets them now
self.assertEqual(self.done('A - Eins (2020).epub', 'imported').status_code, 200)
self.assertEqual(self.done('Sub/B.pdf', 'failed').status_code, 200)
from accounts.models import WebDAVSeen
WebDAVSeen.objects.filter(path='Sub/B.pdf').update(claimed_at=None) # the claim ran out
self.assertEqual([f['path'] for f in self.inbox()['files']], ['Sub/B.pdf'])
def test_a_changed_file_counts_as_new(self):
self.inbox()
self.done('A - Eins (2020).epub', 'imported')
self.done('Sub/B.pdf', 'known')
self.assertEqual(self.inbox()['files'], [])
old = self.listing
self.listing = lambda path='': [dict(e, size=999) if e['path'].endswith('.epub') else e for e in old(path)]
self.assertEqual([f['path'] for f in self.inbox()['files']], ['A - Eins (2020).epub'])
def test_only_own_sources_and_valid_results(self):
other = User.objects.create_user('x', password='pw12345678')
foreign = WebDAVSource.objects.create(user=other, label='f', base_url='https://d.example.org/dav/', inbox=True)
self.assertEqual(self.client.post(f'/books/cloud/{foreign.pk}/done/', '{}', content_type='application/json').status_code, 404)
self.inbox()
self.assertEqual(self.done('A - Eins (2020).epub', 'whatever').status_code, 400)
self.client.logout()
self.assertEqual(self.client.get('/books/cloud/inbox/').status_code, 401)
def test_settings_switch_the_inbox(self):
r = self.client.post(f'/accounts/webdav/{self.src.pk}/edit/', {'label': 'Bücher', 'base_url': self.src.base_url})
self.assertEqual(r.status_code, 302)
self.src.refresh_from_db()
self.assertFalse(self.src.inbox)
class VaultTests(TestCase):
"""The vault keeps what the devices encrypted, per name; only for its owner."""
def test_keep_replace_remove(self):
u = User.objects.create_user('v', password='pw12345678')
self.client.force_login(u)
post = lambda body: self.client.post('/accounts/vault/ai/', json.dumps(body), content_type='application/json')
self.assertEqual(self.client.get('/accounts/vault/ai/').json(), {'ct': '', 'iv': ''})
self.assertEqual(post({'ct': 'abc', 'iv': 'i1'}).status_code, 200)
self.assertEqual(post({'ct': 'def', 'iv': 'i2'}).status_code, 200)
self.assertEqual(self.client.get('/accounts/vault/ai/').json(), {'ct': 'def', 'iv': 'i2'})
self.assertEqual(post({'iv': 'x'}).status_code, 400)
other = User.objects.create_user('w', password='pw12345678')
self.client.force_login(other)
self.assertEqual(self.client.get('/accounts/vault/ai/').json()['ct'], '')
self.client.force_login(u)
self.assertEqual(post({'ct': ''}).status_code, 200)
self.assertEqual(self.client.get('/accounts/vault/ai/').json()['ct'], '')
self.client.logout()
self.assertEqual(self.client.get('/accounts/vault/ai/').status_code, 401)

2
accounts/urls.py
View file

@ -17,6 +17,8 @@ urlpatterns = [
path('change-password/', views.change_password, name='change_password'),
path('api-token/regenerate/', views.regenerate_api_token, name='regenerate_api_token'),
path('webdav/add/', views.webdav_add, name='webdav_add'),
path('webdav/<int:pk>/edit/', views.webdav_edit, name='webdav_edit'),
path('webdav/<int:pk>/test/', views.webdav_test, name='webdav_test'),
path('webdav/<int:pk>/delete/', views.webdav_delete, name='webdav_delete'),
path('vault/<slug:name>/', views.vault, name='vault'),
]

70
accounts/views.py
View file

@ -12,7 +12,7 @@ from django.views.decorators.http import require_http_methods
from radio import lastfm as lastfm_module
from .models import ApiToken, WebDAVSource
from .models import ApiToken, UserVault, WebDAVSource
from .webdav import WebDAVError, list_directory
User = get_user_model()
@ -97,7 +97,7 @@ def lastfm_connect(request):
return render(request, 'accounts/settings.html', {
'profile': request.user.profile,
'has_lastfm': request.user.profile.has_lastfm(),
'lastfm_error': f"Could not connect to Last.fm: {exc}",
'lastfm_error': f"Verbindung zu Last.fm fehlgeschlagen: {exc}",
})
@ -283,6 +283,7 @@ def webdav_add(request):
username=request.POST.get('username', '').strip()[:200],
password=request.POST.get('password', '')[:500],
root_path=request.POST.get('root_path', '').strip()[:500],
inbox=request.POST.get('inbox') == '1',
)
level, message = _probe_source(source)
@ -290,6 +291,36 @@ def webdav_add(request):
return redirect('settings')
@login_required
@require_http_methods(['POST'])
def webdav_edit(request, pk):
source = WebDAVSource.objects.filter(pk=pk, user=request.user).first()
if not source:
messages.error(request, 'Verbindung nicht gefunden.')
return redirect('settings')
label = request.POST.get('label', '').strip()
base_url = request.POST.get('base_url', '').strip()
if not label or not base_url:
messages.error(request, 'Name und Server-URL sind erforderlich.')
return redirect('settings')
source.label = label[:100]
source.base_url = base_url[:500]
source.username = request.POST.get('username', '').strip()[:200]
source.root_path = request.POST.get('root_path', '').strip()[:500]
source.inbox = request.POST.get('inbox') == '1'
# The stored password is never sent to the page: empty means keep it.
password = request.POST.get('password', '')
if password:
source.password = password[:500]
source.save()
level, message = _probe_source(source)
messages.add_message(request, level, message)
return redirect('settings')
@login_required
@require_http_methods(['POST'])
def webdav_test(request, pk):
@ -310,3 +341,38 @@ def webdav_delete(request, pk):
if deleted:
messages.success(request, 'Verbindung entfernt.')
return redirect('settings')
# ---------------------------------------------------------------------------
# Vault: small device settings, end-to-end encrypted (docs/ai.md)
# ---------------------------------------------------------------------------
VAULT_MAX = 64 * 1024
@csrf_exempt
@require_http_methods(['GET', 'POST'])
def vault(request, name):
"""GET {ct, iv} ('' while there is none); POST {ct, iv} replaces it, {ct: ''} removes it.
The blob is encrypted on the devices with the book key; the server only keeps it."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
if request.method == 'GET':
v = UserVault.objects.filter(user=request.user, name=name).first()
return JsonResponse({'ct': v.ct if v else '', 'iv': v.iv if v else ''})
if len(request.body) > VAULT_MAX:
return JsonResponse({'error': 'too large'}, status=413)
try:
body = json.loads(request.body or b'{}')
except (json.JSONDecodeError, ValueError):
return JsonResponse({'error': 'invalid JSON'}, status=400)
ct = body.get('ct') if isinstance(body, dict) else None
iv = body.get('iv') if isinstance(body, dict) else None
if ct == '':
UserVault.objects.filter(user=request.user, name=name).delete()
return JsonResponse({'ok': True})
if not isinstance(ct, str) or not isinstance(iv, str) or not iv or len(iv) > 64:
return JsonResponse({'error': 'ct and iv required'}, status=400)
UserVault.objects.update_or_create(user=request.user, name=name, defaults={'ct': ct, 'iv': iv})
return JsonResponse({'ok': True})

43
books/migrations/0005_progress_last_read_wins.py Normal file
View file

@ -0,0 +1,43 @@
from django.db import migrations, models
def keep_what_we_know(apps, schema_editor):
"""The continue point so far was the furthest place: it is that, and it was
last saved at updated_at."""
EBookProgress = apps.get_model('books', 'EBookProgress')
for p in EBookProgress.objects.all().iterator():
p.furthest_fraction = p.scroll_fraction
p.furthest_anchor = p.position_anchor
p.read_at = int(p.updated_at.timestamp() * 1000)
p.save(update_fields=['furthest_fraction', 'furthest_anchor', 'read_at'])
class Migration(migrations.Migration):
dependencies = [
('books', '0004_ebook_is_read'),
]
operations = [
migrations.AddField(
model_name='ebookprogress',
name='read_at',
field=models.BigIntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='ebookprogress',
name='device',
field=models.CharField(blank=True, default='', max_length=40),
),
migrations.AddField(
model_name='ebookprogress',
name='furthest_fraction',
field=models.FloatField(default=0.0),
),
migrations.AddField(
model_name='ebookprogress',
name='furthest_anchor',
field=models.CharField(blank=True, default='', max_length=30),
),
migrations.RunPython(keep_what_we_know, migrations.RunPython.noop),
]

36
books/migrations/0006_covers_and_prefs.py Normal file
View file

@ -0,0 +1,36 @@
# Generated by Django 6.1 on 2026-10-16 18:21
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('books', '0005_progress_last_read_wins'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddField(
model_name='ebook',
name='cover_ct',
field=models.TextField(blank=True, default=''),
),
migrations.AddField(
model_name='ebook',
name='cover_iv',
field=models.CharField(blank=True, default='', max_length=32),
),
migrations.CreateModel(
name='LibraryPrefs',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('prefs_ct', models.TextField(blank=True, default='')),
('prefs_iv', models.CharField(blank=True, default='', max_length=32)),
('updated_at', models.DateTimeField(auto_now=True)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='library_prefs', to=settings.AUTH_USER_MODEL)),
],
),
]

23
books/migrations/0007_list_indexes.py Normal file
View file

@ -0,0 +1,23 @@
# Generated by Django 6.1 on 2026-10-19 13:24
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('books', '0006_covers_and_prefs'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddIndex(
model_name='ebook',
index=models.Index(fields=['user', 'uploaded_at', 'is_read'], name='ebook_list_idx'),
),
migrations.AddIndex(
model_name='ebook',
index=models.Index(fields=['user', 'cover_iv'], name='ebook_cover_idx'),
),
]

38
books/migrations/0008_ebook_file.py Normal file
View file

@ -0,0 +1,38 @@
# Generated by Django 6.1 on 2026-10-19 15:02
# The books' files move out of their rows into a table of their own (docs/database.md):
# copied in one statement, then the two columns are dropped (ALTER TABLE DROP COLUMN).
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('books', '0007_list_indexes'),
]
operations = [
migrations.CreateModel(
name='EBookFile',
fields=[
('book', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='file', serialize=False, to='books.ebook')),
('data_ct', models.TextField()),
('data_iv', models.CharField(max_length=32)),
],
),
migrations.RunSQL(
'INSERT INTO books_ebookfile (book_id, data_ct, data_iv) SELECT id, data_ct, data_iv FROM books_ebook',
reverse_sql='UPDATE books_ebook SET '
'data_ct = (SELECT f.data_ct FROM books_ebookfile f WHERE f.book_id = books_ebook.id), '
'data_iv = (SELECT f.data_iv FROM books_ebookfile f WHERE f.book_id = books_ebook.id)',
),
migrations.RemoveField(
model_name='ebook',
name='data_ct',
),
migrations.RemoveField(
model_name='ebook',
name='data_iv',
),
]

68
books/models.py
View file

@ -2,27 +2,91 @@ from django.db import models
from django.contrib.auth.models import User
class EBookManager(models.Manager):
def create(self, data_ct=None, data_iv=None, **kwargs):
"""A book, and with data_ct/data_iv its file (EBookFile) in the same go."""
book = super().create(**kwargs)
if data_ct is not None:
EBookFile.objects.create(book=book, data_ct=data_ct, data_iv=data_iv or '')
return book
class EBook(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ebooks')
meta_ct = models.TextField() # base64 AES-GCM ciphertext of {title, author, filename}
meta_iv = models.CharField(max_length=32) # hex IV for metadata
data_ct = models.TextField() # base64 AES-GCM ciphertext of raw EPUB bytes
data_iv = models.CharField(max_length=32) # hex IV for EPUB data
# The file itself is in EBookFile (book.file), not here (docs/database.md).
uploaded_at = models.DateTimeField(auto_now_add=True)
is_read = models.BooleanField(default=False)
# The cover, end-to-end encrypted like the rest (docs/library.md): a client that has
# the file extracts it, encrypts it with the user's key and sends it once.
cover_ct = models.TextField(blank=True, default='')
cover_iv = models.CharField(max_length=32, blank=True, default='')
objects = EBookManager()
class Meta:
ordering = ['uploaded_at']
# The list reads these from indexes (book_rows()); they made it fast while the file
# was still in this row (docs/database.md), and cost nothing now.
indexes = [
models.Index(fields=['user', 'uploaded_at', 'is_read'], name='ebook_list_idx'),
models.Index(fields=['user', 'cover_iv'], name='ebook_cover_idx'),
]
def __str__(self):
return f"EBook #{self.pk} (user={self.user_id})"
class EBookFile(models.Model):
"""A book's file, encrypted like everything else: base64 AES-GCM of the EPUB/PDF bytes.
In a table of its own (docs/database.md): with the file in the book's row, every change of a
title, a folder or "gelesen" rewrote megabytes, and the list read past every file."""
book = models.OneToOneField(EBook, on_delete=models.CASCADE, primary_key=True, related_name='file')
data_ct = models.TextField()
data_iv = models.CharField(max_length=32)
def __str__(self):
return f"EBookFile of #{self.book_id}"
def book_rows(user):
"""Every book of the user as {id, meta_ct, meta_iv, uploaded_at (ISO), is_read, has_cover},
oldest first — read without passing a book's file: meta_ct and meta_iv stand before it in
the row, the rest comes from the covering indexes ebook_list_idx and ebook_cover_idx."""
books = {b['id']: b for b in EBook.objects.filter(user=user).order_by().values('id', 'meta_ct', 'meta_iv')}
for book_id, at, read in EBook.objects.filter(user=user).order_by().values_list('id', 'uploaded_at', 'is_read'):
if book_id in books:
books[book_id].update(uploaded_at=at, is_read=read)
covered = set(EBook.objects.filter(user=user).exclude(cover_iv='').order_by().values_list('id', flat=True))
out = sorted((b for b in books.values() if 'uploaded_at' in b), key=lambda b: (b['uploaded_at'], b['id']))
for b in out:
b['uploaded_at'] = b['uploaded_at'].isoformat()
b['has_cover'] = b['id'] in covered
return out
class LibraryPrefs(models.Model):
"""The library's own settings, encrypted with the user's key (the folder order:
folder names are private, docs/library.md). One blob, replaced as a whole."""
user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='library_prefs')
prefs_ct = models.TextField(blank=True, default='')
prefs_iv = models.CharField(max_length=32, blank=True, default='')
updated_at = models.DateTimeField(auto_now=True)
class EBookProgress(models.Model):
"""Where to continue a book: the place it was last read, on any device
(docs/progress-sync.md), plus the furthest place ever reached."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ebook_progress')
book = models.ForeignKey(EBook, on_delete=models.CASCADE, related_name='progress')
scroll_fraction = models.FloatField(default=0.0)
position_anchor = models.CharField(max_length=30, blank=True, default='')
# When the reader was at that place (ms since epoch, server clock); null before 2026-10-12.
read_at = models.BigIntegerField(null=True, blank=True)
device = models.CharField(max_length=40, blank=True, default='')
furthest_fraction = models.FloatField(default=0.0)
furthest_anchor = models.CharField(max_length=30, blank=True, default='')
updated_at = models.DateTimeField(auto_now=True)
class Meta:

292
books/tests.py Normal file
View file

@ -0,0 +1,292 @@
"""The reading position: the place last read wins (docs/progress-sync.md)."""
import json
from unittest.mock import patch
from django.contrib.auth.models import User
from django.test import TestCase
from accounts.models import ApiToken
from .models import EBook, EBookProgress
NOW = 1_760_270_000_000 # the server's clock in these tests (ms)
MIN = 60_000
class ProgressTests(TestCase):
def setUp(self):
self.user = User.objects.create_user(username='alice', password='pw12345678')
self.auth = {'HTTP_AUTHORIZATION': f'Bearer {ApiToken.objects.create(user=self.user).token}'}
self.book = EBook.objects.create(user=self.user, meta_ct='m', meta_iv='i', data_ct='d', data_iv='i')
self.now = NOW
clock = patch('books.views._now_ms', side_effect=lambda: self.now)
clock.start()
self.addCleanup(clock.stop)
def save(self, anchor, frac=None, read_at=None, sent_at=None, device='Handy', **extra):
body = {'scroll_fraction': frac if frac is not None else 0.1, 'position_anchor': anchor, **extra}
if read_at is not None:
body.update(read_at=read_at, sent_at=sent_at if sent_at is not None else self.now, device=device)
resp = self.client.post(f'/books/{self.book.id}/progress/', json.dumps(body),
content_type='application/json', **self.auth)
self.assertEqual(resp.status_code, 200)
return resp.json()
def state(self):
return EBookProgress.objects.get(book=self.book)
# -- the new rule
def test_newer_place_wins_even_if_it_is_earlier_in_the_book(self):
self.save('300:0.5', read_at=NOW - 10 * MIN, device='Handy')
r = self.save('12:0.0', read_at=NOW - 1 * MIN, device='diora Desktop')
self.assertTrue(r['stored'])
p = self.state()
self.assertEqual((p.position_anchor, p.device, p.read_at), ('12:0.0', 'diora Desktop', NOW - MIN))
self.assertEqual(p.furthest_anchor, '300:0.5')
def test_an_older_write_arriving_late_does_not_win(self):
# The phone read offline at 10:00 and syncs at 12:00; the desktop read at 11:00.
self.save('40:0.0', read_at=NOW - 60 * MIN, device='diora Desktop')
r = self.save('90:0.0', read_at=NOW - 120 * MIN, device='Handy')
self.assertFalse(r['stored'])
self.assertEqual(r['position_anchor'], '40:0.0')
p = self.state()
self.assertEqual((p.position_anchor, p.device), ('40:0.0', 'diora Desktop'))
self.assertEqual(p.furthest_anchor, '90:0.0', 'the late write still counts as reached')
def test_resaving_an_unchanged_place_keeps_its_time_and_cannot_win(self):
self.save('50:0.0', read_at=NOW - 5 * MIN, device='Handy')
self.save('70:0.0', read_at=NOW - 2 * MIN, device='diora Desktop')
# The phone, idle since, saves its place again on going to the background.
self.now += MIN
r = self.save('50:0.0', read_at=NOW - 5 * MIN, device='Handy')
self.assertFalse(r['stored'])
self.assertEqual(self.state().position_anchor, '70:0.0')
def test_the_device_clock_error_is_taken_out(self):
# A tolino 3 hours behind: it read one minute ago by its clock, and sends now by its clock.
behind = 3 * 60 * MIN
self.save('10:0.0', read_at=NOW - 30 * MIN, device='Handy')
r = self.save('20:0.0', read_at=NOW - behind - MIN, sent_at=NOW - behind, device='tolino')
self.assertTrue(r['stored'])
self.assertEqual(self.state().read_at, NOW - MIN)
def test_nothing_wins_from_the_future(self):
self.save('10:0.0', read_at=NOW + 24 * 60 * MIN, sent_at=NOW)
self.assertEqual(self.state().read_at, NOW)
self.now += MIN
self.assertTrue(self.save('11:0.0', read_at=self.now, sent_at=self.now)['stored'])
def test_equal_times_take_the_new_write(self):
self.save('10:0.0', read_at=NOW - MIN)
self.assertTrue(self.save('11:0.0', read_at=NOW - MIN)['stored'])
def test_force_sets_the_place_now(self):
self.save('300:0.0', read_at=NOW - MIN)
self.assertTrue(self.save('0:0.0', frac=0.0, force=True)['stored'])
p = self.state()
self.assertEqual((p.position_anchor, p.read_at), ('0:0.0', NOW))
self.assertEqual(p.furthest_anchor, '300:0.0')
def test_furthest_only_grows(self):
self.save('80:0.0', read_at=NOW - 3 * MIN)
self.save('20:0.0', read_at=NOW - 2 * MIN)
self.save('81:0.0', read_at=NOW - 9 * MIN)
p = self.state()
self.assertEqual((p.position_anchor, p.furthest_anchor), ('20:0.0', '81:0.0'))
def test_pdfs_compare_by_fraction(self):
self.save('', frac=0.7, read_at=NOW - 2 * MIN)
self.save('', frac=0.3, read_at=NOW - MIN)
p = self.state()
self.assertEqual((p.scroll_fraction, p.furthest_fraction), (0.3, 0.7))
def test_response_is_the_state_after_the_write(self):
r = self.save('5:0.25', frac=0.05, read_at=NOW - MIN, device='Browser')
self.assertEqual(r['server_now'], NOW)
for key in ('scroll_fraction', 'position_anchor', 'read_at', 'device',
'furthest_fraction', 'furthest_anchor', 'last_read'):
self.assertIn(key, r)
self.assertEqual(r['read_at'], NOW - MIN)
def test_implausible_times_count_as_an_old_app(self):
self.save('10:0.0', read_at=NOW - MIN)
# read_at=True or seconds instead of ms are not times: only forward, like an old app.
self.assertFalse(self.save('5:0.0', read_at=True)['stored'])
self.assertFalse(self.save('5:0.0', read_at=1_760_270_000)['stored'])
# -- apps from before the change (no read_at)
def test_old_apps_only_move_forward(self):
self.save('50:0.0', read_at=NOW - MIN)
self.assertFalse(self.save('20:0.0')['stored'])
self.assertTrue(self.save('60:0.0')['stored'])
self.assertEqual(self.state().read_at, NOW)
def test_old_apps_resaving_the_same_place_change_nothing(self):
self.save('50:0.0', read_at=NOW - 5 * MIN, device='Handy')
self.now += MIN
self.assertFalse(self.save('50:0.0')['stored'])
p = self.state()
self.assertEqual((p.read_at, p.device), (NOW - 5 * MIN, 'Handy'))
def test_first_write_of_an_old_app_is_stored(self):
self.assertTrue(self.save('0:0.0', frac=0.0)['stored'])
def test_old_force_still_works(self):
self.save('50:0.0', read_at=NOW - MIN)
self.assertTrue(self.save('1:0.0', force=True)['stored'])
# -- reading
def test_book_list_and_snapshot_carry_the_state(self):
self.save('7:0.5', frac=0.2, read_at=NOW - MIN, device='Browser')
resp = self.client.get('/books/', **self.auth)
b = resp.json()[0]
self.assertEqual(resp['X-Server-Now'], str(NOW))
self.assertEqual((b['position_anchor'], b['read_at'], b['device'], b['furthest_anchor']),
('7:0.5', NOW - MIN, 'Browser', '7:0.5'))
self.assertTrue(b['last_read'].startswith('2025-10-12T'))
snap = self.client.get('/api/sync/?only=books', **self.auth).json()
p = snap['book_progress'][0]
self.assertEqual((p['book_id'], p['read_at'], p['device']), (self.book.id, NOW - MIN, 'Browser'))
self.assertIn('server_now', snap)
def test_rows_from_before_have_their_save_time_as_read_at(self):
p = EBookProgress.objects.create(user=self.user, book=self.book, scroll_fraction=0.4, position_anchor='9:0.0')
b = self.client.get('/books/', **self.auth).json()[0]
self.assertEqual(b['read_at'], int(p.updated_at.timestamp() * 1000))
def test_a_never_opened_book(self):
b = self.client.get('/books/', **self.auth).json()[0]
self.assertEqual((b['read_at'], b['last_read'], b['position_anchor']), (None, None, ''))
class CoverAndPrefsTests(TestCase):
"""Covers and the library's settings travel encrypted (docs/library.md)."""
def setUp(self):
self.user = User.objects.create_user(username='alice', password='pw12345678')
self.auth = {'HTTP_AUTHORIZATION': f'Bearer {ApiToken.objects.create(user=self.user).token}'}
self.book = EBook.objects.create(user=self.user, meta_ct='m', meta_iv='i', data_ct='d', data_iv='i')
def post(self, url, body, **auth):
return self.client.post(url, json.dumps(body), content_type='application/json', **(auth or self.auth))
def test_a_cover_is_kept_and_listed(self):
url = f'/books/{self.book.id}/cover/'
self.assertEqual(self.client.get(url, **self.auth).status_code, 404)
self.assertFalse(self.client.get('/books/', **self.auth).json()[0]['has_cover'])
self.assertEqual(self.post(url, {'cover_ct': 'Q0lQSEVS', 'cover_iv': 'a1b2c3d4e5f60718293a4b5c'}).status_code, 200)
self.assertEqual(self.client.get(url, **self.auth).json(),
{'cover_ct': 'Q0lQSEVS', 'cover_iv': 'a1b2c3d4e5f60718293a4b5c'})
self.assertTrue(self.client.get('/books/', **self.auth).json()[0]['has_cover'])
def test_bad_or_huge_covers_are_refused(self):
url = f'/books/{self.book.id}/cover/'
self.assertEqual(self.post(url, {'cover_ct': 'x', 'cover_iv': 'not hex!'}).status_code, 400)
self.assertEqual(self.post(url, {'cover_ct': 'x' * 300_000, 'cover_iv': 'a1b2c3d4'}).status_code, 400)
def test_covers_are_ones_own(self):
bob = User.objects.create_user(username='bob', password='pw12345678')
bob_auth = {'HTTP_AUTHORIZATION': f'Bearer {ApiToken.objects.create(user=bob).token}'}
url = f'/books/{self.book.id}/cover/'
self.assertEqual(self.post(url, {'cover_ct': 'x', 'cover_iv': 'a1b2c3d4'}, **bob_auth).status_code, 404)
self.assertEqual(self.client.get(url, **bob_auth).status_code, 404)
def test_prefs_are_one_encrypted_blob(self):
self.assertEqual(self.client.get('/books/prefs/', **self.auth).json(), {'prefs_ct': '', 'prefs_iv': ''})
self.post('/books/prefs/', {'prefs_ct': 'AAA', 'prefs_iv': 'a1b2c3d4e5f6'})
self.post('/books/prefs/', {'prefs_ct': 'BBB', 'prefs_iv': 'a1b2c3d4e5f7'})
self.assertEqual(self.client.get('/books/prefs/', **self.auth).json(), {'prefs_ct': 'BBB', 'prefs_iv': 'a1b2c3d4e5f7'})
class BookRowsReadPastNoFileTests(TestCase):
"""The list must not read a column that stands after the file (data_ct) in a book's row:
SQLite would walk through the whole file to get there (docs/database.md)."""
def test_the_list_queries_use_covering_indexes(self):
from django.contrib.auth.models import User
from django.db import connection
from django.test.utils import CaptureQueriesContext
from books.models import EBook, book_rows
user = User.objects.create_user('rows', password='x')
EBook.objects.create(user=user, meta_ct='m', meta_iv='i', data_ct='d' * 100000, data_iv='j', cover_iv='c', cover_ct='cc')
EBook.objects.create(user=user, meta_ct='m2', meta_iv='i2', data_ct='d', data_iv='j')
with CaptureQueriesContext(connection) as q:
rows = book_rows(user)
self.assertEqual([r['meta_ct'] for r in rows], ['m', 'm2'])
self.assertEqual([r['has_cover'] for r in rows], [True, False])
late = ('uploaded_at', 'is_read', 'cover_iv', 'cover_ct', 'data_ct', 'data_iv')
with connection.cursor() as cur:
for query in q.captured_queries:
sql = query['sql']
if any(col in sql for col in late):
cur.execute('EXPLAIN QUERY PLAN ' + sql.replace('%s', '?'), [user.id] if '?' in sql or '%s' in sql else [])
plan = ' '.join(str(r) for r in cur.fetchall())
self.assertIn('COVERING INDEX', plan, sql)
class EBookFileTests(TestCase):
"""The file lives in EBookFile; upload, data, replace-data and rekey go there."""
def setUp(self):
from django.contrib.auth.models import User
self.user = User.objects.create_user('filer', password='pw-filer-123')
self.client.force_login(self.user)
def test_upload_read_replace_rekey_delete(self):
import base64, json
from books.models import EBook, EBookFile
data = base64.b64encode(b'x' * 100).decode()
r = self.client.post('/books/upload/', json.dumps({'meta_ct': 'm', 'meta_iv': 'i', 'data_ct': data, 'data_iv': 'd'}),
content_type='application/json')
pk = r.json()['id']
self.assertEqual(EBookFile.objects.get(book_id=pk).data_ct, data)
self.assertEqual(self.client.get(f'/books/{pk}/data/').json(), {'data_ct': data, 'data_iv': 'd'})
self.client.post(f'/books/{pk}/replace-data/', json.dumps({'data_ct': 'bmV3', 'data_iv': 'd2'}), content_type='application/json')
self.assertEqual(self.client.get(f'/books/{pk}/data/').json()['data_ct'], 'bmV3')
self.client.post(f'/books/{pk}/rekey/', json.dumps({'meta_ct': 'm3', 'meta_iv': 'i3', 'data_ct': 'cmVr', 'data_iv': 'd3'}),
content_type='application/json')
self.assertEqual(EBook.objects.get(pk=pk).meta_ct, 'm3')
self.assertEqual(EBookFile.objects.get(book_id=pk).data_iv, 'd3')
self.client.post(f'/books/{pk}/delete/')
self.assertFalse(EBookFile.objects.filter(book_id=pk).exists())
def test_someone_elses_file_is_not_found(self):
from django.contrib.auth.models import User
from books.models import EBook
other = User.objects.create_user('other-filer', password='pw-other-123')
b = EBook.objects.create(user=other, meta_ct='m', meta_iv='i', data_ct='ZA==', data_iv='i')
self.assertEqual(self.client.get(f'/books/{b.id}/data/').status_code, 404)
class StaleMetaTests(TestCase):
"""A meta write that starts from an older meta (base_iv) is refused with the current one."""
def setUp(self):
from django.contrib.auth.models import User
from books.models import EBook
self.user = User.objects.create_user('metauser', password='pw-meta-123')
self.client.force_login(self.user)
self.book = EBook.objects.create(user=self.user, meta_ct='m1', meta_iv='aa', data_ct='ZA==', data_iv='i')
def post(self, body):
import json
return self.client.post(f'/books/{self.book.id}/meta/', json.dumps(body), content_type='application/json')
def test_current_base_is_taken(self):
self.assertEqual(self.post({'meta_ct': 'm2', 'meta_iv': 'bb', 'base_iv': 'aa'}).status_code, 200)
self.book.refresh_from_db()
self.assertEqual((self.book.meta_ct, self.book.meta_iv), ('m2', 'bb'))
def test_old_base_is_refused_with_the_current_meta(self):
self.post({'meta_ct': 'm2', 'meta_iv': 'bb', 'base_iv': 'aa'})
r = self.post({'meta_ct': 'old', 'meta_iv': 'cc', 'base_iv': 'aa'})
self.assertEqual(r.status_code, 409)
self.assertEqual((r.json()['meta_ct'], r.json()['meta_iv']), ('m2', 'bb'))
self.book.refresh_from_db()
self.assertEqual(self.book.meta_ct, 'm2')
def test_without_base_as_before(self):
self.assertEqual(self.post({'meta_ct': 'm3', 'meta_iv': 'dd'}).status_code, 200)

4
books/urls.py
View file

@ -5,6 +5,8 @@ urlpatterns = [
path('', views.book_list, name='book_list'),
path('upload/', views.upload_book, name='upload_book'),
path('cloud/sources/', webdav.cloud_sources, name='cloud_sources'),
path('cloud/inbox/', webdav.cloud_inbox, name='cloud_inbox'),
path('cloud/<int:pk>/done/', webdav.cloud_done, name='cloud_done'),
path('cloud/<int:pk>/browse/', webdav.cloud_browse, name='cloud_browse'),
path('cloud/<int:pk>/fetch/', webdav.cloud_fetch, name='cloud_fetch'),
path('metadata-lookup/', views.lookup_book_metadata, name='lookup_book_metadata'),
@ -12,6 +14,8 @@ urlpatterns = [
path('<int:pk>/delete/', views.delete_book, name='delete_book'),
path('<int:pk>/read/', views.set_book_read, name='set_book_read'),
path('<int:pk>/meta/', views.update_book_meta, name='update_book_meta'),
path('<int:pk>/cover/', views.book_cover, name='book_cover'),
path('prefs/', views.library_prefs, name='library_prefs'),
path('<int:pk>/replace-data/', views.replace_book_data, name='replace_book_data'),
path('<int:pk>/rekey/', views.rekey_book, name='rekey_book'),
path('<int:pk>/progress/', views.save_progress, name='save_book_progress'),

238
books/views.py
View file

@ -1,15 +1,18 @@
import base64
import json
import re
import time
import xml.etree.ElementTree as ET
from datetime import datetime, timezone as dt_timezone
import requests
from django.conf import settings
from django.db import transaction
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from .models import EBook, EBookProgress, EBookHighlights, EBookBookmarks
from .models import EBook, EBookFile, EBookProgress, EBookHighlights, EBookBookmarks, LibraryPrefs, book_rows
def _require_auth(request):
@ -247,11 +250,40 @@ def _progress_is_further(new_anchor, new_frac, old_anchor, old_frac):
"""True if the new reading position is at least as far into the book as the
old one. Compares by anchor block index (precise, decoupled from layout);
falls back to scroll_fraction only when an anchor is missing (PDFs)."""
nb, ni = _anchor_parts(new_anchor)
ob, oi = _anchor_parts(old_anchor)
if nb >= 0 and ob >= 0:
return ni >= oi if nb == ob else nb >= ob
return new_frac >= old_frac
return _cmp_progress(new_anchor, new_frac, old_anchor, old_frac) >= 0
def _cmp_progress(a_anchor, a_frac, b_anchor, b_frac):
"""> 0 if A is further into the book than B, < 0 if behind, 0 if level."""
ab, ai = _anchor_parts(a_anchor)
bb, bi = _anchor_parts(b_anchor)
if ab >= 0 and bb >= 0:
if ab != bb:
return ab - bb
return (ai > bi) - (ai < bi)
return (a_frac > b_frac) - (a_frac < b_frac)
def _now_ms():
return int(time.time() * 1000)
def progress_state(p):
"""A book's reading state as every client sees it (docs/progress-sync.md);
p may be None (never opened)."""
if p is None:
return {'scroll_fraction': 0.0, 'position_anchor': '', 'read_at': None, 'device': '',
'furthest_fraction': 0.0, 'furthest_anchor': '', 'last_read': None}
read_at = p.read_at if p.read_at is not None else int(p.updated_at.timestamp() * 1000)
return {
'scroll_fraction': p.scroll_fraction,
'position_anchor': p.position_anchor,
'read_at': read_at,
'device': p.device,
'furthest_fraction': p.furthest_fraction,
'furthest_anchor': p.furthest_anchor,
'last_read': datetime.fromtimestamp(read_at / 1000, tz=dt_timezone.utc).isoformat(),
}
@require_http_methods(['GET'])
@ -259,26 +291,18 @@ def book_list(request):
err = _require_auth(request)
if err:
return err
books = list(
request.user.ebooks.values('id', 'meta_ct', 'meta_iv', 'uploaded_at', 'is_read')
)
for b in books:
b['uploaded_at'] = b['uploaded_at'].isoformat()
# Include saved scroll_fraction for each book
progress_map = {
p.book_id: (p.scroll_fraction, p.updated_at, p.position_anchor)
for p in EBookProgress.objects.filter(user=request.user)
}
books = book_rows(request.user)
progress_map = {p.book_id: p for p in EBookProgress.objects.filter(user=request.user)}
highlighted_ids = set(
EBookHighlights.objects.filter(user=request.user).values_list('book_id', flat=True)
)
for b in books:
prog = progress_map.get(b['id'])
b['scroll_fraction'] = prog[0] if prog else 0.0
b['last_read'] = prog[1].isoformat() if prog else None
b['position_anchor'] = prog[2] if prog else ''
b.update(progress_state(progress_map.get(b['id'])))
b['has_highlights'] = b['id'] in highlighted_ids
return JsonResponse(books, safe=False)
response = JsonResponse(books, safe=False)
# The list is a bare array (old clients expect that): the server clock goes in a header.
response['X-Server-Now'] = str(_now_ms())
return response
@csrf_exempt
@ -326,12 +350,77 @@ def update_book_meta(request, pk):
if not meta_ct or not meta_iv:
return JsonResponse({'error': 'meta_ct, meta_iv required'}, status=400)
# base_iv: the IV of the meta the client changed. Another one now (changed elsewhere
# since): refused with the current meta, for the client to apply its change to that.
# A queued write from a device with an old list (a title or folder set meanwhile)
# used to put the old meta back. Clients that don't send base_iv write as before.
base_iv = body.get('base_iv')
if base_iv and base_iv != book.meta_iv:
return JsonResponse({'error': 'stale', 'meta_ct': book.meta_ct, 'meta_iv': book.meta_iv}, status=409)
book.meta_ct = meta_ct
book.meta_iv = meta_iv
book.save(update_fields=['meta_ct', 'meta_iv'])
return JsonResponse({'ok': True})
MAX_COVER = 200_000 # characters of base64: a 300×450 JPEG is far below
MAX_PREFS = 100_000
def _hex_iv(value):
return isinstance(value, str) and 8 <= len(value) <= 32 and all(c in '0123456789abcdefABCDEF' for c in value)
@csrf_exempt
@require_http_methods(['GET', 'POST'])
def book_cover(request, pk):
"""A book's cover, encrypted on the device with the user's key (docs/library.md):
GET {cover_ct, cover_iv} (404 without one), POST the same to set it."""
err = _require_auth(request)
if err:
return err
book = EBook.objects.filter(pk=pk, user=request.user).only('id', 'cover_ct', 'cover_iv').first()
if book is None:
return JsonResponse({'error': 'not found'}, status=404)
if request.method == 'GET':
if not book.cover_ct:
return JsonResponse({'error': 'no cover'}, status=404)
return JsonResponse({'cover_ct': book.cover_ct, 'cover_iv': book.cover_iv})
try:
body = json.loads(request.body)
except (json.JSONDecodeError, ValueError):
return JsonResponse({'error': 'invalid JSON'}, status=400)
ct, iv = body.get('cover_ct'), body.get('cover_iv')
if not isinstance(ct, str) or not ct or len(ct) > MAX_COVER or not _hex_iv(iv):
return JsonResponse({'error': 'cover_ct, cover_iv required (cover at most 150 KB)'}, status=400)
book.cover_ct, book.cover_iv = ct, iv
book.save(update_fields=['cover_ct', 'cover_iv'])
return JsonResponse({'ok': True})
@csrf_exempt
@require_http_methods(['GET', 'POST'])
def library_prefs(request):
"""The library's settings, encrypted with the user's key ({folder_order: [...]}):
GET {prefs_ct, prefs_iv} ('' while there are none), POST the same, replacing them."""
err = _require_auth(request)
if err:
return err
if request.method == 'GET':
p = LibraryPrefs.objects.filter(user=request.user).first()
return JsonResponse({'prefs_ct': p.prefs_ct if p else '', 'prefs_iv': p.prefs_iv if p else ''})
try:
body = json.loads(request.body)
except (json.JSONDecodeError, ValueError):
return JsonResponse({'error': 'invalid JSON'}, status=400)
ct, iv = body.get('prefs_ct'), body.get('prefs_iv')
if not isinstance(ct, str) or not ct or len(ct) > MAX_PREFS or not _hex_iv(iv):
return JsonResponse({'error': 'prefs_ct, prefs_iv required'}, status=400)
LibraryPrefs.objects.update_or_create(user=request.user, defaults={'prefs_ct': ct, 'prefs_iv': iv})
return JsonResponse({'ok': True})
@csrf_exempt
@require_http_methods(['POST'])
def upload_book(request):
@ -360,16 +449,17 @@ def upload_book(request):
return JsonResponse({'error': 'invalid base64 in data_ct'}, status=400)
if raw_size > max_bytes:
return JsonResponse({'error': 'file too large (max 50 MB)'}, status=400)
return JsonResponse({'error': f'file too large (max {settings.EBOOK_MAX_BYTES // (1024 * 1024)} MB)'}, status=400)
book = EBook.objects.create(
user=request.user,
meta_ct=meta_ct,
meta_iv=meta_iv,
data_ct=data_ct,
data_iv=data_iv,
)
EBookProgress.objects.create(user=request.user, book=book)
with transaction.atomic():
book = EBook.objects.create(
user=request.user,
meta_ct=meta_ct,
meta_iv=meta_iv,
data_ct=data_ct,
data_iv=data_iv,
)
EBookProgress.objects.create(user=request.user, book=book)
return JsonResponse({'ok': True, 'id': book.id})
@ -379,12 +469,10 @@ def get_book_data(request, pk):
if err:
return err
try:
book = EBook.objects.get(pk=pk, user=request.user)
except EBook.DoesNotExist:
f = EBookFile.objects.filter(book_id=pk, book__user=request.user).first()
if f is None:
return JsonResponse({'error': 'not found'}, status=404)
return JsonResponse({'data_ct': book.data_ct, 'data_iv': book.data_iv})
return JsonResponse({'data_ct': f.data_ct, 'data_iv': f.data_iv})
@csrf_exempt
@ -412,14 +500,12 @@ def replace_book_data(request, pk):
if not all([data_ct, data_iv]):
return JsonResponse({'error': 'data_ct and data_iv required'}, status=400)
update_fields = ['data_ct', 'data_iv']
book.data_ct = data_ct
book.data_iv = data_iv
if meta_ct and meta_iv:
book.meta_ct = meta_ct
book.meta_iv = meta_iv
update_fields += ['meta_ct', 'meta_iv']
book.save(update_fields=update_fields)
with transaction.atomic():
EBookFile.objects.update_or_create(book=book, defaults={'data_ct': data_ct, 'data_iv': data_iv})
if meta_ct and meta_iv:
book.meta_ct = meta_ct
book.meta_iv = meta_iv
book.save(update_fields=['meta_ct', 'meta_iv'])
return JsonResponse({'ok': True})
@ -448,11 +534,11 @@ def rekey_book(request, pk):
if not all([meta_ct, meta_iv, data_ct, data_iv]):
return JsonResponse({'error': 'meta_ct, meta_iv, data_ct, data_iv required'}, status=400)
book.meta_ct = meta_ct
book.meta_iv = meta_iv
book.data_ct = data_ct
book.data_iv = data_iv
book.save(update_fields=['meta_ct', 'meta_iv', 'data_ct', 'data_iv'])
with transaction.atomic():
EBookFile.objects.update_or_create(book=book, defaults={'data_ct': data_ct, 'data_iv': data_iv})
book.meta_ct = meta_ct
book.meta_iv = meta_iv
book.save(update_fields=['meta_ct', 'meta_iv'])
return JsonResponse({'ok': True})
@ -497,22 +583,50 @@ def save_progress(request, pk):
if isinstance(raw_anchor, str) and re.fullmatch(r'\d{1,7}:\d(\.\d{1,6})?', raw_anchor):
position_anchor = raw_anchor
force = bool(body.get('force', False))
force = body.get('force') is True
device = body.get('device') if isinstance(body.get('device'), str) else ''
now = _now_ms()
read_at = _client_ms(body.get('read_at'))
sent_at = _client_ms(body.get('sent_at'))
progress, _ = EBookProgress.objects.get_or_create(
user=request.user,
book=book,
)
# Always advance to the furthest-read position (by anchor block index, so a
# transiently wrong scroll_fraction can't freeze the position) unless the
# client explicitly forces a reset (e.g. "start over" button in the reader).
if force or _progress_is_further(position_anchor, scroll_fraction,
progress.position_anchor, progress.scroll_fraction):
progress.scroll_fraction = scroll_fraction
progress.position_anchor = position_anchor
progress.save(update_fields=['scroll_fraction', 'position_anchor', 'updated_at'])
with transaction.atomic():
progress, created = EBookProgress.objects.select_for_update().get_or_create(user=request.user, book=book)
if created:
progress.read_at = None
if force:
when, store = now, True
elif read_at is not None:
# The device's clock may be off: move its time onto ours by the
# difference at sending, and nothing counts as later than now.
when = read_at + (now - sent_at) if sent_at is not None else read_at
when = min(when, now)
store = progress.read_at is None or when >= progress.read_at
else:
# An app from before 2026-10-12: only forward, as it used to be — but
# strictly, so re-saving the same place changes nothing.
when = now
store = created or _cmp_progress(position_anchor, scroll_fraction,
progress.position_anchor, progress.scroll_fraction) > 0
if store:
progress.scroll_fraction = scroll_fraction
progress.position_anchor = position_anchor
progress.read_at = when
progress.device = device[:40]
if _cmp_progress(position_anchor, scroll_fraction,
progress.furthest_anchor, progress.furthest_fraction) > 0 or created:
progress.furthest_fraction = scroll_fraction
progress.furthest_anchor = position_anchor
progress.save()
return JsonResponse({'ok': True})
return JsonResponse({'ok': True, 'stored': store, 'server_now': now, **progress_state(progress)})
def _client_ms(value):
"""A time from a client (ms since epoch) or None if it isn't a plausible one."""
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
value = int(value)
return value if 10 ** 12 <= value < 10 ** 14 else None
@csrf_exempt

91
books/webdav.py
View file

@ -12,12 +12,16 @@ browser and only then POSTed to /books/upload/ as ciphertext, exactly like a
local upload — nothing readable is ever stored server-side.
"""
import json
from datetime import timedelta
from django.conf import settings
from django.http import HttpResponse, JsonResponse
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from accounts.models import WebDAVSource
from accounts.models import WebDAVSeen, WebDAVSource
from accounts.webdav import WebDAVError, WebDAVInputError, list_directory, fetch_file
from .views import _require_auth
@ -34,11 +38,94 @@ def cloud_sources(request):
sources = request.user.webdav_sources.all()
return JsonResponse({'sources': [
{'id': s.id, 'label': s.label, 'host': s.base_url}
{'id': s.id, 'label': s.label, 'host': s.base_url, 'inbox': s.inbox}
for s in sources
]})
# ---- "Eingang" (docs/library.md): new books of a source into the library by themselves ----
INBOX_DEPTH = 3 # sub-folders looked into
INBOX_MAX = 20 # files handed out per call
INBOX_CLAIM = timedelta(minutes=10)
def _inbox_files(source):
"""The source's book files, its sub-folders too (INBOX_DEPTH deep)."""
out, todo = [], [('', 0)]
while todo:
path, depth = todo.pop(0)
for e in list_directory(source, path):
if e['is_dir'] and depth < INBOX_DEPTH:
todo.append((e['path'], depth + 1))
elif e['is_book']:
out.append(e)
return out
@require_http_methods(['GET'])
def cloud_inbox(request):
"""The files of the user's "Eingang" sources not taken into the library yet, claimed for
this device for ten minutes (another device asking meanwhile doesn't get them):
{files: [{source, label, path, name, size, modified}], errors: [{source, label, error}]}.
A file that changed (other size) counts as new again."""
unauthorized = _require_auth(request)
if unauthorized:
return unauthorized
now = timezone.now()
files, errors = [], []
for source in request.user.webdav_sources.filter(inbox=True):
try:
entries = _inbox_files(source)
except (WebDAVError, WebDAVInputError) as exc:
errors.append({'source': source.id, 'label': source.label, 'error': str(exc)})
continue
seen = {s.path: s for s in source.seen.all()}
for e in entries:
if len(files) >= INBOX_MAX:
break
s = seen.get(e['path'])
if s and s.size == e['size']:
if s.done_at or (s.claimed_at and now - s.claimed_at < INBOX_CLAIM):
continue
WebDAVSeen.objects.update_or_create(
source=source, path=e['path'][:1000],
defaults={'size': e['size'], 'modified': e['modified'][:100], 'claimed_at': now,
'done_at': None, 'result': ''})
files.append({'source': source.id, 'label': source.label, 'path': e['path'],
'name': e['name'].rsplit('/', 1)[-1], 'size': e['size'], 'modified': e['modified']})
source.last_used_at = now
source.save(update_fields=['last_used_at'])
return JsonResponse({'files': files, 'errors': errors})
@csrf_exempt
@require_http_methods(['POST'])
def cloud_done(request, pk):
"""{path, result}: a device finished with an "Eingang" file — imported, known (already in the
library), too_big: not offered again; failed: offered again after the claim ran out."""
unauthorized = _require_auth(request)
if unauthorized:
return unauthorized
source = _get_source(request, pk)
if not source:
return JsonResponse({'error': 'unknown source'}, status=404)
try:
body = json.loads(request.body or b'{}')
except (json.JSONDecodeError, ValueError):
return JsonResponse({'error': 'invalid JSON'}, status=400)
result = body.get('result') if isinstance(body, dict) else None
if result not in ('imported', 'known', 'too_big', 'failed'):
return JsonResponse({'error': 'result: imported, known, too_big or failed'}, status=400)
seen = WebDAVSeen.objects.filter(source=source, path=str(body.get('path', ''))[:1000]).first()
if not seen:
return JsonResponse({'error': 'unknown file'}, status=404)
seen.result = result
seen.done_at = None if result == 'failed' else timezone.now()
seen.save(update_fields=['result', 'done_at'])
return JsonResponse({'ok': True})
@require_http_methods(['GET'])
def cloud_browse(request, pk):
unauthorized = _require_auth(request)

2
diora/__init__.py
View file

@ -0,0 +1,2 @@
# SQLite in WAL mode for every connection (diora/sqlite.py).
from . import sqlite # noqa: E402,F401

101
diora/search.py Normal file
View file

@ -0,0 +1,101 @@
"""The search across diora (docs/search.md): one query, what the server knows of every area —
podcasts and audiobooks one follows, podcast episodes (the full-text index, podcasts/discover.py),
news and blog sources, their articles, saved radio stations. The books aren't here: their titles
are encrypted, the page searches them itself (static/js/search.js).
Every word of the query has to be there, as the start of a word ("plattf" finds "Plattform");
case and diacritics don't count."""
from django.db.models import Q
from django.http import JsonResponse
from django.views.decorators.http import require_http_methods
from feeds.models import Item, Source
from podcasts import discover
from podcasts.models import PodcastFeed
from radio.models import SavedStation
LIMIT = 8 # per area; the area's own view has the rest
ARTICLE_SCAN = 400 # articles looked at after the database's rough match
def _words(q):
return [w for w in discover._words(q) if w][:8]
def _matches(words, *texts):
"""Every word as the start of a word in the texts ("plattf" finds "Plattform")."""
have = discover._words(' '.join(t or '' for t in texts))
return bool(words) and all(any(h.startswith(w) for h in have) for w in words)
def _rank(words, title):
"""Title starting with the query first, then title words, then the rest."""
t = discover._words(title)
if t[:len(words)] == words or (t and t[0].startswith(words[0])):
return 0
return 1 if all(any(h.startswith(w) for h in t) for w in words) else 2
def _feeds(user, words):
out = {'podcasts': [], 'audiobooks': []}
rows = PodcastFeed.objects.filter(user=user).values('id', 'title', 'author', 'artwork_url', 'kind', 'rss_url',
'favorite', 'new_episodes')
found = [f for f in rows if _matches(words, f['title'], f['author'])]
found.sort(key=lambda f: (_rank(words, f['title']), (f['title'] or '').lower()))
for f in found:
key = 'audiobooks' if f['kind'] == PodcastFeed.AUDIOBOOK else 'podcasts'
out[key].append(f)
return {k: v[:LIMIT] for k, v in out.items()} | {'podcasts_total': len(out['podcasts']),
'audiobooks_total': len(out['audiobooks'])}
def _sources(user, words):
rows = Source.objects.filter(user=user).values('id', 'title', 'url', 'kind', 'icon_url', 'category')
found = [s for s in rows if _matches(words, s['title'], s['url'], s['category'])]
found.sort(key=lambda s: (_rank(words, s['title']), (s['title'] or '').lower()))
return found[:LIMIT]
def _articles(user, words):
"""Articles whose title or author has every word, the newest first. The database narrows by
the longest word (LIKE, as case-blind as SQLite is), Python decides."""
longest = max(words, key=len)
rough = Q(title__icontains=longest) | Q(author__icontains=longest)
if longest != longest.capitalize():
rough |= Q(title__contains=longest.capitalize())
rows = (Item.objects.filter(user=user, gone=False).filter(rough).order_by('-published')
.values('id', 'title', 'author', 'kind', 'published', 'image_url', 'read_at', 'source__title')[:ARTICLE_SCAN])
found = [i for i in rows if _matches(words, i['title'], i['author'])]
return [{'id': i['id'], 'title': i['title'], 'author': i['author'], 'kind': i['kind'],
'published': i['published'].isoformat() if i['published'] else None, 'image_url': i['image_url'],
'read': i['read_at'] is not None, 'source': i['source__title']} for i in found[:LIMIT * 2]], len(found)
def _stations(user, words):
rows = SavedStation.objects.filter(user=user).values('id', 'name', 'url', 'tags', 'country', 'favicon_url',
'is_favorite')
found = [s for s in rows if _matches(words, s['name'], s['tags'], s['country'])]
found.sort(key=lambda s: (_rank(words, s['name']), not s['is_favorite'], s['name'].lower()))
return found[:LIMIT]
def search(user, q):
words = _words(q)
if not words or len(''.join(words)) < 2:
return {'q': q, 'empty': True}
out = {'q': q}
out.update(_feeds(user, words))
ids = discover.search_ids(user, q, limit=LIMIT * 2)
out['episodes'] = discover._rows(user, ids)
out['sources'] = _sources(user, words)
out['articles'], out['articles_total'] = _articles(user, words)
out['stations'] = _stations(user, words)
return out
@require_http_methods(['GET'])
def search_view(request):
"""?q=…: what matches in every area (see the module's text)."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
return JsonResponse(search(request.user, request.GET.get('q', '')[:200]))

14
diora/settings.py
View file

@ -31,12 +31,14 @@ INSTALLED_APPS = [
'books',
'gpodder',
'tts',
'feeds',
'projects',
]
EBOOK_MAX_BYTES = 50 * 1024 * 1024 # 50 MB
EBOOK_MAX_BYTES = 100 * 1024 * 1024 # 100 MB (big scanned PDFs, complete editions)
# Encrypted uploads are base64-encoded (~33% overhead) so allow ~75 MB body
DATA_UPLOAD_MAX_MEMORY_SIZE = 75 * 1024 * 1024
# Encrypted uploads are base64-encoded (~33% overhead) so allow ~140 MB body
DATA_UPLOAD_MAX_MEMORY_SIZE = 140 * 1024 * 1024
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
@ -76,7 +78,9 @@ DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR / 'data' / os.environ.get('DIORA_DB_NAME', 'db.sqlite3'),
'OPTIONS': {'timeout': 20},
# WAL (diora/sqlite.py); a transaction takes the write lock when it begins, so one that
# reads first never fails on upgrading it (WAL's busy timeout can't help there).
'OPTIONS': {'timeout': 20, 'transaction_mode': 'IMMEDIATE'},
}
}
@ -89,7 +93,7 @@ AUTH_PASSWORD_VALIDATORS = [
{'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'},
]
LANGUAGE_CODE = 'en-us'
LANGUAGE_CODE = 'de'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_TZ = True

23
diora/sqlite.py Normal file
View file

@ -0,0 +1,23 @@
"""SQLite in write-ahead-log mode (docs/database.md): readers no longer wait for a writer.
In the old rollback-journal mode a long write (the podcast refresh every 15 minutes, a book of
30 MB going in) kept every reader out until it was done, and requests failed with "database is
locked" — even GET /books/. WAL is set once per database file and stays; synchronous=NORMAL is
the safe pairing for it. A connection that can't switch (another one holds the file) just goes
on as it is; the next one switches.
"""
from django.db.backends.signals import connection_created
def _sqlite_wal(sender, connection, **kwargs):
if connection.vendor != 'sqlite':
return
with connection.cursor() as cursor:
try:
cursor.execute('PRAGMA journal_mode=WAL')
except Exception: # noqa: BLE001 - locked right now; another connection will do it
pass
cursor.execute('PRAGMA synchronous=NORMAL')
connection_created.connect(_sqlite_wal, dispatch_uid='diora_sqlite_wal')

62
diora/test_search.py Normal file
View file

@ -0,0 +1,62 @@
"""The search across diora (diora/search.py)."""
from datetime import datetime, timezone
from django.contrib.auth.models import User
from django.test import TestCase
from feeds.models import Item, Source
from podcasts import discover
from podcasts.models import PodcastEpisode, PodcastFeed
from radio.models import SavedStation
class SearchTests(TestCase):
def setUp(self):
self.user = User.objects.create_user(username='sucher', password='pw12345678')
self.client.force_login(self.user)
other = User.objects.create_user(username='andere', password='pw12345678')
when = datetime(2026, 3, 1, tzinfo=timezone.utc)
self.pod = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/p', title='Ökonomie heute',
author='Peter Decker')
self.book = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/b', title='Das Kapital',
author='Karl Marx', kind=PodcastFeed.AUDIOBOOK)
PodcastFeed.objects.create(user=other, rss_url='https://example.org/p', title='Ökonomie heute')
self.ep = PodcastEpisode.objects.create(feed=self.pod, guid='e1', title='Die Ökonomie der Plattformen',
audio_url='https://example.org/e1.mp3', pub_date=when)
PodcastEpisode.objects.create(feed=self.book, guid='k1', title='Kapitel 1: Die Ware',
audio_url='https://example.org/k1.mp3', pub_date=when)
discover.index_new()
blog = Source.objects.create(user=self.user, url='https://blog.example/feed', title='Plattform-Blog', kind=Source.BLOG)
Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='a', title='Plattformarbeit und Streik',
published=when)
Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='b', title='Gartenarbeit', published=when)
Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='c', title='Plattformen, alt',
published=when, gone=True)
SavedStation.objects.create(user=self.user, name='Radio Plattform', url='https://radio.example/s', tags='jazz')
def find(self, q):
return self.client.get('/search/', {'q': q}).json()
def test_every_area(self):
r = self.find('plattf')
self.assertEqual([e['id'] for e in r['episodes']], [self.ep.id])
self.assertEqual([a['title'] for a in r['articles']], ['Plattformarbeit und Streik'])
self.assertEqual([s['title'] for s in r['sources']], ['Plattform-Blog'])
self.assertEqual([s['name'] for s in r['stations']], ['Radio Plattform'])
def test_podcasts_and_audiobooks_apart_case_and_accents_blind(self):
r = self.find('okonomie')
self.assertEqual([f['id'] for f in r['podcasts']], [self.pod.id])
self.assertEqual(r['audiobooks'], [])
r = self.find('marx kap')
self.assertEqual([f['id'] for f in r['audiobooks']], [self.book.id])
self.assertEqual(r['episodes'], [], 'an audiobook\'s chapters are no podcast episodes')
def test_every_word_and_nothing_for_too_little(self):
self.assertEqual(self.find('plattform garten')['articles'], [])
self.assertTrue(self.find('a')['empty'])
self.assertEqual(self.client.get('/search/', {'q': 'x'}).status_code, 200)
def test_only_signed_in(self):
self.client.logout()
self.assertEqual(self.client.get('/search/', {'q': 'plattform'}).status_code, 401)

4
diora/urls.py
View file

@ -5,15 +5,19 @@ from django.urls import path, include
from django.views.static import serve as serve_static
from accounts.sync import sync_snapshot
from diora.search import search_view
urlpatterns = [
path('admin/', admin.site.urls),
path('accounts/', include('accounts.urls')),
path('podcasts/', include('podcasts.urls')),
path('books/', include('books.urls')),
path('feeds/', include('feeds.urls')),
path('projects/', include('projects.urls')),
path('tts/', include('tts.urls')),
path('api/2/', include('gpodder.urls')),
path('api/sync/', sync_snapshot, name='api_sync'),
path('search/', search_view, name='search'),
# Served at the root (not /static/js/sw.js) so its default scope covers
# the whole app — a service worker's scope is limited to its own script's
# directory unless the server sends Service-Worker-Allowed, so registering

57
docs/ai.md Normal file
View file

@ -0,0 +1,57 @@
# KI: Claude schlägt vor, der Nutzer entscheidet
Since 2026-10-24. Organisation help with Claude (Anthropic) across diora: books and magazines
first; podcasts, news/blogs and projects follow (same building blocks).
## The key: the user's own, end-to-end encrypted, only on the devices
Two providers, chosen in the settings (decided 2026-10-25): **Anthropic** (the official SDK) or
**OpenRouter** (the same Claude model, `anthropic/claude-opus-5.5`). OpenRouter's
Anthropic-compatible endpoint refuses the headers a browser call needs (`anthropic-version` …,
checked by a CORS preflight), so diora talks to OpenRouter's chat completions instead: plain
`fetch`, `response_format` json_schema (strict), `reasoning.effort`; no fallback on refusals, and
OpenRouter sees what is sent too. The vault keeps `{provider, anthropic_key, openrouter_key}`
(an old `{anthropic_key}` still reads as Anthropic); a key is checked before it is kept
(Anthropic: the model's info; OpenRouter: `GET /api/v1/key`).
Decided with the user: the API key is entered in the app ("KI-Assistent", the ✦ of a
feature or the app menu) and stored like the books — encrypted with the book key on the device,
kept by the server as a blob it can't read:
- `GET/POST /accounts/vault/<name>/` (`accounts.UserVault`, name `ai`): `{ct, iv}`; `{ct: ''}`
removes it. Its own store, not the library prefs (`/books/prefs/` is rewritten whole when
the folder order is saved).
- The devices decrypt it into memory only and call Claude **directly** from the page
(browser, the apps' WebView, the desktop). The diora server sees neither the key nor what is
sent; Anthropic sees what a feature needs (titles, authors, folder names …). Anthropic bills
the user.
- Without the book key on a device (or after a password change, which changes the book key)
the key can't be read: the settings ask for it again. Not on diora Reader (e-readers).
- Server-side jobs (in the background, without an app open) were left out on purpose: they
would need the key on the server. The project Radar has its own server key (docs/projects.md).
## Building blocks (`static/js/ai.js`, an ES module)
- The official SDK, `@anthropic-ai/sdk` 0.133.0, bundled for the browser with esbuild 0.28.2:
`static/vendor/anthropic-sdk-0.133.0.mjs` (licence beside it). `dangerouslyAllowBrowser`:
the key is the user's own. To update: `npm install @anthropic-ai/sdk@<v> esbuild` in an empty
folder, `esbuild entry.mjs --bundle --format=esm --platform=browser --target=es2020 --minify`
with `export { default } from '@anthropic-ai/sdk'`, and change the import in ai.js.
- `DioraAI.ask({system, prompt, schema, effort})`: model `claude-opus-5-5`, structured output
(`output_config.format` json_schema), effort `low` by default, streamed, `fallbacks: "default"`
(beta `server-side-fallback-2026-07-01`) for declined requests; refusals and over-long
answers become plain German messages.
- `DioraAI.review({title, intro, items, onApply})`: the suggestions with checkboxes — nothing
changes until the user ticks and presses "Übernehmen".
- `DioraAI.settings()`, `withKey(fn)`, `progress()`, `showError()`, `toast()`; the sheets close
with Android's back key; ⚙ in each sheet opens the settings.
## Features
| area | where | what Claude gets | what it suggests |
|---|---|---|---|
| Bücher, Zeitschriften | ✦ in the library's head (`static/js/app/55-library-tools.js`) | per book: id, title, author, folders, series, magazine mark; the existing folders | folders to add (existing first; new ones only for ≥ 3 books), series with volume, cleaned title/author where broken, magazine mark |
| Podcasts | ✦ in Abos (diora-android `web-native/static/native-podcast.js`, copied to `static/podcasts/`) | per subscription: title, author, folders, level, heard in 12 months, publishing interval, three newest episode titles | folders to add (≤ 2), level Stamm / Strom / Auf Abruf (only with a clear reason) |
| Nachrichten, Blogs | ✦ in the feeds' bar (`static/js/feeds.js`; the apps' section bar presses it) | per source: title, address, kind, Ressort or folders, recent titles shown | Ressort for news, folders for blogs, news ↔ blog when clearly wrong |
Applied through `_updateBookMeta` (the encrypted meta, as when editing by hand).

112
docs/audiobooks.md Normal file
View file

@ -0,0 +1,112 @@
# Audiobooks (Hörbücher & Hörspiele)
A podcast subscription can be marked as an audiobook. It then leaves the
podcast world (no "Neu", no queue automation, not in the podcast lists) and
shows up in the library ("Bücher") next to the e-books, where it is listened to
like a book: its episodes are chapters in order, "Weiterhören" goes on where
the last chapter stopped, the whole book has one progress, places can be kept
as bookmarks with a note, and the sleep timer can stop at the end of a chapter.
## Server (diora-web)
### The mark
`PodcastFeed.kind`: `"podcast"` (default) or `"audiobook"`. A feed setting like
the others: in `state.feeds[]` (`/podcasts/state/`), set with
`POST /podcasts/feeds/<id>/settings/ {"kind": "audiobook"}`.
Marked as an audiobook:
- its episodes leave "Neu" (dismissed, not played); new ones never get there
and are never queued, whatever `new_episodes` says;
- it is left out of "Folgen" (`/podcasts/episodes/` without `feed_id`);
- nothing else changes: places, played marks, downloads, settings stay.
Episodes of it already in the queue stay there until played or removed.
### Chapters and progress
Chapter order: `episode_number` (if set), then `pub_date`, then the title in
natural order ("Kapitel 2" before "Kapitel 10"), then id.
`GET /podcasts/audiobooks/` → `{"books": [book, …]}`, the last listened first:
```
book = {
"feed_id", "title", "author", "artwork_url",
"chapters": 24, "done": 6, // played chapters
"total_seconds": 32400, "heard_seconds": 11400,
"next": {"episode_id", "index", "position_seconds"} | null, // null: finished
"last_played_at": ISO | null
}
```
`heard_seconds` = played chapters' durations + the places in the others.
Durations the feed doesn't have count as 0 until a player reports them
(`progress/save` with `duration_seconds`, see youtube.md).
`GET /podcasts/audiobooks/<feed_id>/` → `{"book": book, "chapters": [ch, …],
"bookmarks": [bm, …]}` with
```
ch = { // the queue entry's shape, so it can be handed to the player as it is
"episode__id", "episode__title", "episode__audio_url", "episode__duration_seconds",
"episode__artwork_url", "episode__feed__id", "episode__feed__title",
"episode__feed__artwork_url", "episode__pub_date", "position_seconds", "played",
"index"
}
bm = {"id", "episode_id", "position_ms", "note", "created_at"} // in chapter order, then place
```
### Bookmarks
- `POST /podcasts/bookmarks/ {"episode_id", "position_ms", "note"}` → `{"ok", "bookmark": bm}`
- `POST /podcasts/bookmarks/<id>/ {"note"}` → `{"ok", "bookmark": bm}`
- `POST /podcasts/bookmarks/<id>/delete/` → `{"ok"}`
Any episode can have bookmarks; the library shows those of audiobooks.
## The player (both hosts: podcasts-web.js and Android)
One new host call, next to `podcastPlayEpisode`:
`podcastPlayBook(json)` with `json = {"feed_id", "chapters": [ch, …], "index", "position_ms"}`
(`position_ms` optional: else the chapter's own place)
- plays `chapters[index]` **without putting anything in the queue**;
- the book is what plays next: when a chapter ends it counts as played and the
next chapter follows from its own place; after the last one playing stops;
"next" (⏭, media keys) goes to the next chapter without marking the one left
as played;
- playing anything else (a queue episode, the radio's own handover aside) ends
the book context; the book context survives a restart of the page/app;
- the sleep timer needs nothing new: "end of chapter" is `podcastSleep(-1)`
(end of the episode) or, for an episode with chapters of its own, the minutes
until that chapter's end (`podcastSleep(ms / 60000)`, fractions are fine).
`playerState()` (unchanged) tells the library which chapter plays where.
## The library (diora-web `static/js/audiobooks.js`, everywhere)
One file for the web page, the desktop app and the Android app (like
feeds.js; full flavour only, it needs the podcast host). It draws into
`#tab-books`, above `#book-list`, in its own container `#audiobook-shelf`:
- "Weiterhören": the audiobook listened to last (unless finished);
- "Hörbücher": a card per audiobook — picture, title, author,
"3 h 10 von 9 h · Kapitel 7/24" and a progress bar;
- "+ Hörbuch": pick one of the podcast subscriptions to mark as an audiobook;
- the book view: picture, progress, ▶ Weiterhören / Von vorn, the chapter list
(tap: play it from its place; played ones ✓), bookmarks (tap: play from there;
note editable; ✕), "+ Lesezeichen" (at the place where this book plays now),
"Schlummer: Kapitelende", and ⋮ → "Wieder als Podcast", "Alles als ungehört".
It talks to the server with fetch and to the player through the host
(`window.DioraNative` on Android, `window.DioraPodcastHost` elsewhere):
`podcastState`, `podcastFeedSettings`, `podcastPlayBook`, `playerState`,
`podcastSleep`, `podcastMarkPlayedAll`.
## The podcast UI (native-podcast.js, diora-android)
- feed settings sheet: "Als Hörbuch" (sets `kind`);
- audiobook feeds left out of Abos, Neu, Folgen, search results of the
subscriptions; a hint "In Bücher → Hörbücher" where one was moved;
- the player shows a book chapter as "Kapitel 7/24 · <book>" when it plays one.

47
docs/database.md Normal file
View file

@ -0,0 +1,47 @@
# The database
One SQLite file, `data/db.sqlite3` (on the server `/home/marwin/diora-prod-data`, mounted into
`diora-prod` at `/app/data`). The books live in it too, encrypted, so it is large (2.5 GB in
October 2026) and some writes are big (a scanned book of 30 MB).
## Write-ahead log (since 2026-10-19)
`diora/sqlite.py` puts every connection in WAL mode (`PRAGMA journal_mode=WAL`, set once per
file, it stays) with `synchronous=NORMAL`; `settings.DATABASES` has `transaction_mode:
IMMEDIATE`, so a transaction takes the write lock when it begins.
Why: in the rollback-journal mode before, a long write kept all readers out. The podcast
refresh (cron, every 15 minutes) and big uploads made requests fail with "database is
locked", GET /books/ among them — the apps then saw no new books and no changed titles.
With WAL, readers read on while one writer writes; writers still take turns (20 s timeout).
Next to `db.sqlite3` there are now `db.sqlite3-wal` and `db.sqlite3-shm`. They belong to the
database: a copy of `db.sqlite3` alone may miss the latest writes. To copy it safely:
docker exec diora-prod python -c "import sqlite3; s=sqlite3.connect('data/db.sqlite3'); d=sqlite3.connect('data/backup.sqlite3'); s.backup(d)"
or stop the container first. Back to the old mode, if ever needed:
`PRAGMA journal_mode=DELETE` (with no other connection open) and remove `diora/sqlite.py`'s hook.
## The book list never reads past the file
A book's row is `user, meta_ct, meta_iv, data_ct (the file), data_iv, uploaded_at, is_read,
cover_ct, cover_iv`. SQLite stores a big value in a chain of overflow pages, and to reach a
column *after* it, it walks the whole chain: listing 427 books read gigabytes and took 37 s
(the desktop app gives up after 10 s, so it never synced). Since then `books.models.book_rows()`
reads `id, meta_ct, meta_iv` from the rows (they stand before the file) and `uploaded_at,
is_read` / whether there is a cover from the covering indexes `ebook_list_idx` and
`ebook_cover_idx` — no list query touches a file. `books/tests.py` checks the query plans.
## The files in a table of their own (since 2026-10-19)
`books.models.EBookFile` (one row per book, `book_id` its key) holds `data_ct`/`data_iv`;
`EBook` has only the small fields. Before, every change of a title, a folder or "gelesen"
rewrote the whole row with its file (megabytes, for a scanned volume tens of them), and every
`EBook.objects.get()` loaded the file. `EBook.objects.create(data_ct=…, data_iv=…)` still
works (the manager makes the EBookFile too); the API is unchanged.
Migration `books/0008_ebook_file` copied 430 files in one statement and dropped the two
columns: about 2 minutes on the server (rehearsed on a copy), the site down meanwhile. The
pages the old columns used are free now and get reused; `VACUUM` (exclusive, minutes, at a
quiet time) would give them back to the disk.

226
docs/library.md Normal file
View file

@ -0,0 +1,226 @@
# Folders and the library (redesign)
## Folders everywhere: tags shown as filter chips
One idea of a folder for every area — Bücher, Zeitschriften, Hörbücher,
Podcasts, Blogs (and the news' Ressorts, which already work this way):
- A thing can be in **several folders** (they work like tags); none is fine.
- Folders are **filters**, not rooms: a row of chips on top —
`Alle · <folder> n · … · Ohne Ordner` — tapping one filters in place; no
"inside a folder" view, no back button. On a wide window the row wraps.
- **Assigning**: mark one or more things (long press / ☐ select mode, or ⋮ on
one), then "In Ordner …": a sheet with every folder of the area as checkboxes
and "Neuer Ordner". No typing a name that must match.
- **Managing** (⋮ on the chip row → "Ordner verwalten"): rename, merge (rename
onto an existing name), delete (the things stay, only the folder goes),
reorder by dragging. The order is the chip order.
- **Automatic groups** stay automatic and look different from folders (no
chip; a stack/group heading): audiobook series, magazine periodicals.
### Where folders are kept
| Area | Field | Notes |
|------|-------|-------|
| E-books, magazines | encrypted meta `folders: [names]` | `folder` (the first) kept for older clients; read `folders` or else `[folder]` |
| Audiobooks, podcasts | `PodcastFeed.folders` (JSON list) | feed setting `folders`; `folder` (string) still accepted |
| Blogs | `Source.folders` (exists) | |
| News | `Source.category` (one Ressort) | already chips in Nachrichten |
The folder order per area: e-books in the encrypted user settings blob (names
are private), the others on the server (`/podcasts/folders/`, `/feeds/folders/`,
`{order: [names]}`).
### Podcasts: a category's episodes, by time
The folder chips stand over „Folgen“ too: with one chosen it lists every episode of
that category (`GET /podcasts/episodes/?folder=<name>`, empty: the podcasts in none).
„Folgen“, „Neu“ and one podcast's page group their episodes by time — Heute, Gestern,
Diese Woche (the last 7 days), Diesen Monat (30 days), then the months — after the
Kino feed. In „Folgen“ (sort „Nach Zeitraum, seltene zuerst“, the default) and „Neu“
the podcasts that seldom come go first within a group: `interval_days` of each feed
in `/podcasts/state/`, the mean gap between its newest 10 episodes (one episode: its
age, at least 30 days, the same when all came within an hour; kept an hour). Heard is a filter chip, not an order.
### Shared code: `static/js/folders.js` (diora-web, every platform)
```
DioraFolders.chips(el, {folders: [{name, count}], active, none: count|0, onPick(name|''|null), onManage()})
DioraFolders.assign({title, folders: [names], selected: [names], onDone(names)}) // the sheet
DioraFolders.manage({folders: [names], onRename(old, new), onDelete(name), onOrder(names)})
```
Each area keeps its own storage and calls these for the look and the dialogs.
## The library (Bücher, Zeitschriften)
- **Header**: title, search (title, author), sort (zuletzt gelesen · Titel ·
Autor · hinzugefügt · kürzeste / längste zuerst), ⊞/☰ grid or list, `+` (Buch hinzufügen, Aus Cloud).
- **Length** (since 2026-10-25): the pages in the encrypted meta (`pages`) — a PDF's pages, an
EPUB's text by 1800 characters a page (`bookPagesOfFile`, 40-epub.js; the reader's own
estimate). Set on upload and when a book is opened; for older books the device works it out
once when a length sort is chosen ("Länge wird ermittelt … 3 von 80": file from the offline
copy or the server, decrypted on the device). Unknown lengths come last; the cards show
"· 320 S." while sorted by length.
- **Filter chips**: Alle · Angefangen · Ungelesen · Gelesen — then the folders.
- **Weiterlesen**: one card on top (cover, title, "Seite 132 von 186 · gestern",
▶), not repeated in the list below.
- **Grid** (default on phone and desktop): covers with a progress line, title.
**List** (default on e-ink / diora Reader): one row per book, small cover,
title, author, % / ✓.
### Covers, end-to-end encrypted
A client that has a book's file extracts its cover (EPUB: the OPF cover image;
PDF: page 1), scales it to 240 px wide JPEG, **encrypts it with the user's key**
(AES-GCM like the meta) and uploads it once: `POST /books/<id>/cover/
{cover_ct, cover_iv}`. `GET /books/` lists `has_cover`; `GET /books/<id>/cover/`
returns `{cover_ct, cover_iv}`. Clients cache the decrypted image locally. The
server never sees a cover.
### Editions
Two or more books can be **editions of one work** (the PDF scan and the EPUB, two
translations …): the same random key in each one's encrypted meta, `work: "w…"`
(`''` or missing = none). The list shows a work once — the edition read last, else an EPUB,
else the first added — with its formats on the cover ("EPUB · PDF", in the list view after
the author). Each edition keeps its own file, position and highlights.
- Joining: choose several books → "Als Ausgaben verbinden", or a book's ⋮ → "Als Ausgabe
verbinden …" (other books listed, similar titles first, with a search).
- ⋮ → "Ausgaben (n) …" lists the editions: open one, "Lösen" sets one apart (the last one
left is set apart with it), "Weitere Ausgabe verbinden …".
- For the apps' own book sheets: `window.bookEditions(id)` opens that sheet,
`window.bookEditionsOf(id)` returns the editions (the book alone if it has none).
### Generated covers
A book without a cover (or one set to `cover: 'generated'` in its meta) gets one
drawn by `static/js/gencover.js`: an SVG, the same on every device, never stored.
A **concept** (motif, palettes, typeface) is chosen by score:
- words in the title (3), the shelf category (2) and the author (2, only at a
name's start). A stem of five letters or more counts anywhere (German
compounds), a shorter one at a word's start, one of three letters as a whole
word; `^` / `$` in the list force a word's start / end.
- the folders, by name (`FOLDER_THEMES`): the first counts 2.5 — more than an
author or shelf word, less than a title word — further ones 1.
A work of Marx or Engels (author, or a volume of the MEW series) is always a **blue volume**
(blau: blue cloth, gold rules and capitals, the spine on the left with the volume number);
no other book is.
Concepts: konstruktiv (Marxist and political titles), geometrie (Philosophie: rings, turning squares,
triangle/circle/square), psyche (Psychoanalyse: two ellipses, the Borromean
knot, a spiral), karte (Geschichte), kosmos (Science-Fiction), ornament
(Fantasy), raster (Politik, Soziologie), schatten (Krimi: a red line, a lit
doorway), grusel (Horror), klassiker (patterned paper with a title plate),
sakral (Religion & Mythologie), spiel (Kinder & Jugend), code (Informatik),
typo, wellen, botanik, berge, zellen, and since 2026-10-25 funk (talk and radio: waveform,
mast, speech bubbles), musik (record, staff), kunst (colour fields, brushstrokes), essen (plate
on a checked cloth, citrus), sport (track, pitch), geld (price chart, coins), reise (road,
route with compass) — and bauhaus when nothing matches (shapes, quarter-circle tiles, a
circle crossed by diagonals). Most concepts have two or three motifs. A hash of title and
author picks motif, palette, layout and every detail.
**Square** (`{square: true}`, since 2026-10-25): audiobooks and podcasts without a picture, or
whose picture doesn't load (audiobooks.js; diora-android native-podcast.js `art()`, before the
initial letter). 300 × 300, three title lines at most; the band of the motif shown is centred
in what the text leaves free. Podcasts (`podcast: true`) nothing matches get funk, not
bauhaus; a Marx/Engels audiobook gets konstruktiv (the blue volume is a binding).
### Export as XML (since 2026-10-24)
⇩ in the library's head (`static/js/app/55-library-tools.js`) saves `diora-bibliothek-<date>.xml`:
made on the device from the list it has decrypted (nothing new goes to the server), every book
and magazine issue with its diora metadata — title, author, file name, ISBN, pages, shelf
category, series and volume, work (editions), magazine mark, cover choice, folders —, the
reading position (`<progress percent anchor read read-at last-read device furthest-percent
furthest-anchor>`), when it was uploaded, whether it has highlights, and an issue's articles with
their state. The folder order heads the file. Root `<diora-library version="1">`.
✦ beside it: ordering with Claude (docs/ai.md).
### Eingang: books from a cloud folder by themselves (since 2026-10-24)
A cloud source (Einstellungen → Cloud-Verbindungen, WebDAV: `accounts.WebDAVSource`,
`accounts/webdav.py`) can be marked **Eingang**: its new books come into the library without
anyone importing them. The server can't do it alone — the books are encrypted with the key
on the devices — so a device does:
1. `static/js/app/57-cloud-inbox.js`: after the book list has loaded (at most every 5
minutes; only with the real key in `localStorage`, never on diora Reader) it asks
`GET /books/cloud/inbox/` → `{files: [{source, label, path, name, size, modified}], errors}`:
the `.epub`/`.pdf` files of the Eingang sources (three folders deep) no device has taken.
The server claims them for this device for ten minutes (another device asking meanwhile
doesn't get them).
2. Each file: fetched through the server (`/books/cloud/<id>/fetch/`), imported like any file
(`_importEbookBuffer`, `skipKnown`: same title and author → left out), then
`POST /books/cloud/<id>/done/ {path, result}` with `imported`, `known`, `too_big` (not
offered again) or `failed` (again once the claim ran out).
3. The server keeps them in `accounts.WebDAVSeen` (source, path, size, modified): a file whose
size changed counts as new. The folder itself is never changed.
The first such source: Shelfmark's e-book downloads on the Hetzner server, served read-only by
dufs (set up by the "Media" session; its notes: `C:\Users\m\homelab-handoff\04-hoerbuecher.md`).
## Zeitschriften: das Heft und seine Artikel
An issue is a unit (the editors' selection and order, its date, its theme) and
so is each article in it. Neither is reduced to the other
(`static/js/app/54-issues.js`):
- **The issue stays the document.** It opens and reads through as before, in
the editors' order. The reader marks the article one is in: a running head
under the bar ("Schwerpunkt · Artikel 8 von 23 · Titel"), in the apps' footer
"Artikel 8/23", and a **pause at each article's end** (`aside.iss-end`):
"✓ gelesen", the issue's count, "Weiter im Heft" (the next article not read
yet, skipping those read or skipped), "Zum Inhalt", "überspringen",
"Doch nicht gelesen". In a PDF the pause is a bar that comes up when one
turns from an article's last page onto the next.
- **The states determine each other.** An article one was in and whose end one
reached counts as read (a jump — from the contents, "Weiter" — reads
nothing). The issue counts "7 von 23 gelesen · 2 übersprungen"; "Im Heft
weiterlesen" goes to the article one was in, else the next one open.
"Heft abschließen" marks the rest as skipped, never as read.
- **In the library** an issue shows one bar with a segment per article (read,
skipped, open, the one one is in). A tap opens its contents: Rubriken, the
articles with their state, a teaser and the reading time; the ⋮ / mark
sets read / unread / skipped by hand.
**The index** comes from the file, on the device:
- **EPUB**: the table of contents, by file — an article is a file (the first
entry pointing into it); the entries after it into the same file are its
sections, whatever their depth (GegenStandpunkt puts some at the top level);
an entry whose children point into other files is a Rubrik. A one-file EPUB:
the leaves. No table of contents: each spine file with text. The teaser is
the headline under the title (the headings before the first paragraph that
the title doesn't say, not numbered sections), else the first paragraph.
- **PDF**: a real outline (word processors' `_Hlk…`/`_Toc…` leftovers don't
count); most magazines' PDFs have none, so mostly **the contents page
printed in the issue**: among the first 16 pages the one with most numbered
rows (numbers that merely count 1, 2, 3 … are a list, not a contents), plus
its neighbours with entries. A row: the page number at the start
("14 Der Krieg gegen Palästina", Jacobin), or at the end, maybe with the
author in a column before the title ("Nathan Sperber | Beyond Neoliberalism?
| 23", NLR); lines under it in its type size continue the title, an author
line ("Von …", a name) gives the author, lines in other type the teaser; a
title in capitals may begin on the lines above its number, a lone word on top
is its kicker (LuXemburg: GESPRÄCH, INFOGRAFIK); "Name: Titel"
(manuskripte). Capitals in the contents' type are Rubriken. The printed
numbers are mapped to the file's pages by the offset most titles agree on
(found in the page's text, a headline in capitals preferred); running heads
and the contents page's own head are dropped.
Reading times count 200 words a minute. It is kept in the encrypted meta as
`issue: {v: 2, a: [{h, t, r, x, m, a?, p?, e?}]}` (href, title, Rubrik, teaser,
minutes, author, a PDF's first and last page), or `{v: 2, a: [], none: true}`
for an issue without a usable table of contents (read as a whole). An index of
another version is built anew. Built in the
background for issues without one (not on e-ink: there when the issue is
opened, or it comes from another device).
**The states** are in the meta too: `issueRead: {href: 'r' | 's'}`,
`issueAt: href` (the article one was in). A change goes to the server after
2.5 s; offline it waits in `localStorage.diora_issue_pending` and goes with the
next save (or when the connection is back).

111
docs/navigation.md Normal file
View file

@ -0,0 +1,111 @@
# Navigation: Lesen and Hören
diora's areas are sorted by what one does, not where things come from:
| Group | Sections |
|-------|----------|
| — | Home |
| Lesen | Bücher · Zeitschriften · Nachrichten · Blogs |
| Hören | Hörbücher · Podcasts · Radio · Ambient (· Fokus, web page only) |
| Projekte | Projekte (docs/projects.md; `side`) |
| — (foot) | Suche (docs/search.md; `footer`, `transient`) · Abos & Quellen (docs/sources.md; `footer`) |
- **Desktop / wide window (sidebar):** every section is its own entry, under
its group's heading. Nothing inside a section repeats the choice.
- **Phone (tab bar):** four tabs, **Home · Lesen · Hören · Projekte** (since 2026-10-19).
Lesen and Hören show a row of their sections at the top (segments); each group remembers
the section last used. The search is a magnifier in the bar on top (the apps' section bars,
the page's narrow bar), never a tab; Abos & Quellen is in the app menu. The mini player
stays below as it is.
- **Sidebar / wide bar:** `side` sections (Projekte) and then the `footer` ones (Suche,
Abos & Quellen) stand at its foot. A `transient` section (Suche) is never remembered as
the one to open next.
- **diora Reader** (tolino, e-readers): only Lesen, without Home. **TV:** Hören.
Anything new goes into one of the groups. Managing (subscribing, settings,
folders) is in "Abos & Quellen" (docs/sources.md; done).
## One definition: `static/js/nav.js` (diora-web)
Loaded by the page on every platform, before the other area scripts. It owns
the sections, which panel shows each, switching between them, the URL hash
and the remembered choice. The bars are drawn from it. **It is the one source for the
navigation.**
Underneath, `static/js/app/20-tabs.js` keeps `TOP_TABS`: the list of *panels*
(`#tab-<name>`) that `showTab(name)` can show — home, radio, focus, podcasts, books, feeds,
ambient, audiobooks, sources, suche, projekte. That is not a second navigation: nav.js
maps each section to one of these panels (several sections share one: Bücher/Zeitschriften
→ books, Nachrichten/Blogs → feeds) and calls `showTab`. A new area needs both: its panel in
`TOP_TABS` (and the template) and its section in nav.js.
```
window.DioraNav = {
sections(), // [{id, group, label, icon, available}] in order (only those the page has)
groups(), // [{id: 'home'|'lesen'|'hoeren'|'projekte', label, icon, sections: [id…]}]
icon(name), // a section's or group's icon as SVG (also 'menu')
current(), // the section id shown now
go(id), // show a section ('lesen'/'hoeren': that group's last section)
onChange(fn), // fn(sectionId) after every change
}
```
| Section id | Group | Label | Panel | View inside the panel |
|------------|-------|-------|-------|-----------------------|
| `home` | — | Home | `#tab-home` | |
| `books` | lesen | Bücher | `#tab-books` | e-books (not magazines) |
| `magazines` | lesen | Zeitschriften | `#tab-books` | magazines |
| `news` | lesen | Nachrichten | `#tab-feeds` | news |
| `blogs` | lesen | Blogs | `#tab-feeds` | blogs |
| `audiobooks` | hoeren | Hörbücher | `#tab-audiobooks` | (audiobooks.js) |
| `podcasts` | hoeren | Podcasts | `#tab-podcasts` | |
| `radio` | hoeren | Radio | `#tab-radio` | |
| `ambient` | hoeren | Ambient | `#tab-ambient` | |
| `focus` | hoeren | Fokus | `#tab-focus` | web page only (`webOnly`) |
| `projekte` | projekte | Projekte | `#tab-projekte` | (projects.js) |
| `suche` | (foot) | Suche | `#tab-suche` | (search.js); never remembered |
| `sources` | (foot) | Abos & Quellen | `#tab-sources` | (sources.js) |
`go()` calls the page's `showTab(panel)` as before (so everything hooked onto
showTab keeps working) and then the panel's view hook:
`window.__dioraBooksView('books'|'magazines')`,
`window.__dioraFeedsView('news'|'blogs')`. The panels' own switches for that
choice (feeds.js News/Blogs) are hidden. `_currentTopTab` stays the panel name.
Hash: `#books`, `#magazines`, `#news`, `#blogs`, `#audiobooks`, … (old hashes
`#feeds`, `#radio/saved` keep working). Remembered: `localStorage`
`diora_nav_section` and `diora_nav_last_<group>`.
**Zeitschriften**: the books marked as magazines (encrypted meta `magazine`), or, unmarked,
those in a library folder called "Zeitschriften" (or "Magazine"); issues and articles:
docs/library.md.
### The segment row (phone)
nav.js draws `#nav-segments` above the panels: the current group's sections,
when the layout is the tab bar (not `html.nui-sidebar`) and the group has more
than one available section. Styled with the theme tokens; the native CSS may
restyle it.
## Who draws what
| Where | Bar | Drawn by |
|-------|-----|----------|
| Android (phone) | `#nui-tabbar`: Home · Lesen · Hören · Projekte | native-radio.js, from `DioraNav.groups()` |
| Android tablet, desktop (`html.nui-sidebar`) | rail: Home, headings Lesen/Hören with their sections, at the foot Projekte · Suche · Abos & Quellen, Menü | native-radio.js, from `DioraNav.sections()` |
| Apps, every section | a bar on top (`.nui-tab-bar`; `.nui-section-bar` for the generic ones) with the section's name, its actions and the search magnifier | native-ui.js `SECTION_BARS`, native-radio.js, native-podcast.js |
| Web page | `#tabs` (narrow: groups + segments; wide: all sections under headings) | nav.js |
A tab/rail button calls `DioraNav.go(id)`; the active state comes from
`DioraNav.onChange`.
## Phases
1. Navigation (this document), Hörbücher as their own panel, Zeitschriften by folder.
2. "Abos & Quellen": podcasts, YouTube channels, news sources, blogs and the
audiobook mark in one place; the everyday views without managing.
3. Library (done): audiobook series are found in the titles ("Das Kapital –
Erster Band" …) and shown as one stack; audiobooks have folders
(`PodcastFeed.folders`, a list; `folder` is its first, for older clients); a book is marked as a magazine in its encrypted
meta (`magazine: true/false`; unset: the "Zeitschriften" folder decides), and
Zeitschriften groups the issues by periodical.

46
docs/news-text.md Normal file
View file

@ -0,0 +1,46 @@
# Lesehilfen für Nachrichten (feeds/textproc.py)
Klassische Textverfahren, kein Sprachmodell; reines Python auf dem Server, für die Nachrichten
eines Nutzers auf einmal. Ausprobiert an echten Nachrichten (Oktober 2026), übernommen wurde,
was dort trug.
## Floskeln
Sätze, die eine Quelle unter mindestens drei ihrer letzten 40 Artikel setzt (Spendenboxen,
„Diese Nachricht wurde … gesendet“, Abo-Hinweise), gehören zu keinem Artikel. `GET
/feeds/items/<id>/` liefert den Artikel ohne die Absätze, die nur aus solchen Sätzen bestehen; der
Überblick sieht sie gar nicht erst. Je Quelle sechs Stunden zwischengespeichert.
## Überblick einer Meldung
`GET /feeds/stories/<id>/overview/` – nur für Meldungen aus mindestens zwei Quellen:
- `summary`: drei Sätze aus allen Berichten, die am meisten von dem sagen, was alle sagen
(LexRank), ohne einander zu wiederholen (MMR); `[{source, text}]`;
- `novel`: je Quelle bis zu zwei Sätze, denen in keinem anderen Bericht etwas ähnelt (tf-idf-Kosinus
< 0,2), ohne die schon im Überblick; `{source: [text]}`;
- `words`: je Quelle ihre eigenen Wörter für das Ereignis (Log-Odds gegen die übrigen Berichte, mit
Dirichlet-Prior; nur großgeschriebene Formen, also Substantive und Namen); `{source: [word]}`;
- `related`: dieselbe Meldung in der anderen Sprache (Deutsch/Englisch) – Meldungen der letzten zwei
Tage, die mindestens drei seltene Namen teilen (nach idf gewichtet, Summe ≥ 17); Story-Objekte wie
in `/feeds/news/`.
Die App zeigt das über dem Artikel als „Überblick aus N Quellen“, darunter die Meldungen in der
anderen Sprache. Android lädt die Überblicke ungelesener Meldungen mit vor (Repo.kt), damit sie auch
offline da sind.
## Themen
Die erste Seite von `GET /feeds/news/` hat `topics: [{label, ids}]`: Meldungen der letzten zwei
Tage, die von derselben Sache handeln, nicht nur vom selben Ereignis. Gruppiert nach Titel und
Anriss (Leader-Clustering gegen die Mittelpunkte, drei Runden). Ein Thema hat mindestens drei
Meldungen aus mindestens zwei Quellen, liegt dicht beieinander (mittlere Ähnlichkeit zum Mittelpunkt
≥ 0,3) und hat ein Etikett aus mindestens zwei großgeschriebenen Wörtern, die die meisten seiner
Meldungen teilen. Mit `category` nur der Anteil der Kategorie. `?ids=1,2,3` liefert genau diese
Meldungen (ein Thema antippen).
## Verworfen
Stimmung (SentiWS), steigende Begriffe, Fortsetzungsketten (Nachrichten laufen nach zwei Tagen ab),
Clickbait-Erkennung: im Versuch zu ungenau oder ohne Nutzen. Persönliche Reihenfolge: erst, wenn
genug gelesen ist, um daraus zu lernen.

184
docs/podcasts.md Normal file
View file

@ -0,0 +1,184 @@
# Podcasts: Strom, Merkliste, Wiedergabeliste
The plan agreed on 2026-10-19 (with the user, from their listening history). It replaces
the inbox model ("Neu" to work through, "Folgen" beside it, one queue that is both a store
and the play order). Server: diora-web (podcasts/). The podcast UI is diora-android's
`web-native/static/native-podcast.js`, copied into `static/podcasts/` — built there.
## Why
- The subscriptions (about 540) are a self-chosen part of the podcast world: episodes to see
go by and dip into now and then — not a list to work through. "Neu" had 3360 unplayed
episodes; nobody works through that.
- The queue held 74 episodes (100+ hours): a store of "want to hear some day", yet the
player played it in order as if it were the play order.
- In AntennaPod "download automatically" and "add to queue automatically" were one thing
for the user: a regular podcast is one whose new episodes are on the phone. diora split
them (177 feeds downloading, 8 queueing).
- "Silent" subscriptions came from AntennaPod fetching every feed on the phone, slowly. The
server fetches all feeds anyway now (conditional, every 15 minutes).
## The lists
**Wiedergabeliste** — short, what plays next. The player goes through it in order and
**stops when it is empty** (it does not take from the Merkliste). Every episode in it is
downloaded. Filled by hand: ▶ on an episode (plays it, puts it first), or "In die
Wiedergabeliste" from the Merkliste or the Strom.
**Merkliste** — long, "want to hear". It may grow; it **never expires**. Grouped by podcast,
each podcast **collapsible** (newest first inside). New episodes of Stamm subscriptions land
here and are downloaded; so is what the user adds with ＋. An episode played from it moves
to the front of the Wiedergabeliste; finished, it leaves both.
When an episode leaves both lists (heard, removed), its download is deleted (setting
`delete_dequeue`). Downloads follow the lists, not the subscriptions.
## Subscriptions: three levels
| level | new episodes | today's mapping |
|---|---|---|
| **Stamm** | into the Merkliste and downloaded (per feed: or straight into the Wiedergabeliste — an option, used by none yet) | `new_episodes='queue'` (8) plus the regularly heard ones (about 50, list below) |
| **Strom** (default) | in the Strom; nothing to work through, nothing downloaded | `'inbox'`, and the former silent `'ignore'` ones |
| **Auf Abruf** | not in the Strom; on the podcast's page and in search | — (for exceptions) |
The per-feed "download automatically" switch goes: Stamm means it.
**Fetch interval** per subscription: *Automatisch* (by publishing rhythm, `interval_days`:
daily publishers hourly, rare ones daily), *stündlich*, *täglich*, *wöchentlich*. The former
silent subscriptions become Strom with *täglich*.
## The Strom (replaces "Neu" and "Folgen")
All episodes of all Strom and Stamm subscriptions, by period (Heute, Gestern, Diese Woche,
…). Within a period the podcasts are ordered by **how much the user has listened to them**
(episodes heard): every podcast with at least one heard episode before every one without —
those count for infinitely more (heard in the last 12 months, `heard_365`); among the never-heard ones, rarely publishing first.
Each podcast appears **once per period, always bundled**: its newest episode, and "+n ›" for the others. Episodes new since
they were last seen carry a **small mark** (no counts). Per episode: ▶ (Wiedergabeliste,
first), ＋ (Merkliste), ★. Filters stay: Favoriten, Verlauf, Angefangen, Gehört.
Tabs: **Wiedergabeliste · Merkliste · Strom · Abos**.
## Favourite subscriptions
★ on a subscription (`PodcastFeed.favorite`): a mark and a filter (Abos: only the favourites;
episode lists: `filter=fave_feeds`), whatever its level — it puts nothing on the Merkliste by
itself. Started with the Stamm subscriptions marked.
## Stöbern (decided 2026-10-19)
Within the subscriptions:
- **Search** in titles and shownotes: `GET /podcasts/discover/search/?q=…&stream=1` (all
words, the last as a prefix; titles weigh more). SQLite FTS5 index, contentless
(`podcasts/discover.py`), filled after every refresh; `manage.py index_episodes` fills it.
- **Similar episodes** from the other podcasts: `GET /podcasts/discover/similar/<id>/`.
- **People** (who talks, who gives a talk), from episode titles only (`podcasts/people.py`):
`GET /podcasts/discover/people/` → `[{name, episodes, feeds, score}]`, the ones in the most
listened podcasts first; `GET /podcasts/discover/person/?name=` → their episodes.
- **Topics** of the Strom's last days, chips as for the news: `GET /podcasts/discover/topics/?days=7`
→ `[{label, ids}]`.
- **Forgotten**: `GET /podcasts/discover/forgotten/` → `{loved: [...], rare: [...]}` — unheard
episodes older than 60 days of the podcasts listened to most, and of the rare ones; the same
all day.
One controlled step out (Apple's episode search and fyyd; no key):
- **People elsewhere**: `GET /podcasts/discover/elsewhere/?name=` → episodes naming the person
in podcasts the user doesn't have (`{source, podcast, feed_url, title, audio_url, guid,
pub_date, duration_seconds, artwork_url, link}`). `POST /podcasts/discover/take/` with such an
item puts the episode on the Merkliste; its podcast becomes an *Auf Abruf* subscription.
- **Recommendations**: `GET /podcasts/discover/recommend/` → podcasts like the most listened ones
(fyyd), recommended for several first: `{title, feed_url, artwork_url, description, link,
because: [titles], count}`. Subscribing: the usual feeds/add/ (then *Auf Abruf* or Strom).
Show search stays on Apple (best coverage).
## Podcast-Radio (since 2026-10-25)
Listening without committing: one button plays the Strom like a station.
- **The rule**: the radio always plays the **newest** (publication date) Strom episode that is
neither played, nor started (normally), nor done for the radio. So an episode that comes out
while one plays is next (the one playing isn't interrupted); otherwise it goes back in time,
without limit. Auf Abruf podcasts and audiobooks aren't in the Strom, so not in the radio.
- **Mixed**: several episodes of one podcast in a row are mixed with the others waiting — the
next is the newest of *another* podcast than the one before, if there is one (within the
newest 200).
- What comes next is asked anew at every end and skip (so the newest really comes next); the
list fetched before is for "Danach: …" in the player and for going on offline.
- **Leaves no trace**: no progress (no Verlauf, no "Angefangen", no played mark, not counted in
`heard_365`), the Wiedergabeliste and Merkliste unchanged, nothing downloaded (it streams; a
downloaded file plays if there is one), the Strom's marks unchanged.
- **Done for the radio**: run to its end or skipped (⏭, notification, headset, car). Pausing
isn't: the radio keeps its episode and place for itself, ▶ goes on there.
- **Committing**: in the player while the radio plays, ＋ (Merkliste) and "Normal weiterhören"
(first in the Wiedergabeliste at the current place; from then on it plays as usual, with
progress). Playing anything normally switches the radio off (as an audiobook does).
- **Server** (`podcasts/radio.py`): `PodcastRadioHeard(user, episode, at)`, kept apart from
progress and kept for good (pruned, an old episode would come back when the radio goes back
that far). `GET /podcasts/radio/?after=<playing>&skip=<ids>&limit=5` → the next episodes by
the rule (`{episodes: [...]}`, the rows of the episode lists; `after` isn't among them and its
podcast not first; `skip`: ones the device is done with whose /heard/ may still be on its
way); `POST /podcasts/radio/heard/ {"episode_id"}`. Shared by all devices. Filter `untouched`
(episode lists): neither played nor listened into.
- **Player sync**: a report carries `podcast_radio: true` while the radio plays
(`PlayerState.podcast_radio`); taking it over ("Hier abspielen", a transfer) or adopting it goes
on with the radio on that device.
- **Clients**: the web player (`podcasts-web.js`: an entry with `radio: true`, the radio's own
`podradio` in localStorage — next ones, its place, what it was done with lately; host
`podcastRadio()`, `podcastRadioCommit()`, `podcastRadioLater()`; `playerState().radio =
{next}`) and Android (`PodcastRadio.kt` beside `BookPlay.kt`; RadioService's `save()` /
`markPlayed()` / `followQueue()` leave radio items alone). UI (native-podcast.js): 📻 at the top
of the Strom, "Podcast-Radio · <podcast>" in the mini player and the player, there a box with
"Danach: …", ＋ Merkliste and "Normal weiterhören"; ⏭ only skips.
- Later perhaps: a radio of one folder, a longest length.
## Moving over
- Today's queue (74) becomes the Merkliste; the Wiedergabeliste starts empty.
- Stamm: the 8 automatic subscriptions and the regularly heard ones (≥ 8 episodes a year or
≥ 3 in 90 days, 2026-10-19): Ones and Tooze, Die Neuen Zwanziger – Der Salon, Surplus,
The Deprogram, junge Welt – Tagesüberblick, Speakeasy-Bar, jwpod – Forschung, RLS
Audiodokumente, The Reaction Pod, Geschichten aus der Geschichte, Märkte Mächte Emissionen,
friendlyjordies, Second Thought, Neben der Spur, Forschungsquartett, GegenStandpunkt –
Artikel, Hörsaal, Hakim, Marxistische Vorträge, Blätter, Kontextfenster, New Left Review,
Kommunisten Kneipe, Jung & Naiv, HealthyGamerGG, Dekarldent, Daily Dose Of Dekarldent,
Eine Stunde History, Übertage, Eigentlich, Zwischen Zwei Deckeln, UNFASSBAR, Über Rechts,
TitosLeftLung, New Books in History, ARTEde, Irgendwas mit ARTE, Zwei wie Pech & Schwafel,
Geschichte der kommenden Welten, Vollbild, Zeitfragen, Zeitfragen. Feature, Die Filmanalyse,
Reaction, Wirtschaft am Mittag.
- Everything else: Strom; the silent ones with *täglich*.
## Order of work
1. Wiedergabeliste and Merkliste (server: a second list, moving the queue over, the player's
order; UI: two tabs, grouped Merkliste).
2. The three levels and the fetch interval; downloads following the lists.
3. The Strom replacing "Neu" and "Folgen".
4. Browsing — after more thought.
## API (server, since 2026-10-19)
- `PodcastSettings.lists_v2` turns the two lists on. `POST /podcasts/lists/ {"on": true}`
moves the queue into the Merkliste (the Wiedergabeliste starts empty); `{"on": false}` puts
the Merkliste back at the end of the queue. Off, everything works as before (apps not
updated yet).
- **Wiedergabeliste** = the queue as it was: `state.queue`, `/podcasts/queue/add|remove|reorder/`.
The player goes through it and stops at its end.
- **Merkliste** = `PodcastLater`: `state.later` (rows like the queue's, newest first, plus
`added_at`, `auto` — added by a Stamm feed — and `in_queue`), `POST /podcasts/later/add/` and
`/podcasts/later/remove/` `{"episode_ids": [...]}`. Heard (progress at the end, mark-played)
and "Folge löschen" take an episode off it. Episode lists carry `in_later`; filter `later`.
- **Levels**: `PodcastFeed.new_episodes` — `queue` = Stamm into the Merkliste (with lists_v2;
without it: the queue), `play` = Stamm into the Wiedergabeliste, `inbox` = Strom,
`ignore` = Auf Abruf. `auto_queue` stays true for queue and play.
- **Fetch interval**: `PodcastFeed.refresh_interval` — `''` (by rhythm: every `interval_days`
hours, within 15 minutes and 24 hours), `hourly`, `daily`, `weekly`; the cron's
`refresh_feeds` only fetches feeds that are due (`views.refresh_due`).
- **Strom**: `GET /podcasts/episodes/?filter=stream` (all but Auf Abruf; with `folder=`, `sort=`);
episodes carry `discovered_at`. Per feed in `state.feeds`: `heard_365` (episodes heard in the
last 12 months: played, or listened into for more than a minute) and `interval_days`.
`POST /podcasts/stream/seen/` → `{before, now}`; `state.stream_seen_at`. An episode is marked
when its `discovered_at` is after the time before the current look.
- Web host (`static/js/podcasts-web.js`, `DioraPodcastHost`): `podcastLaterAdd(json episodes)`,
`podcastLaterRemove(json ids)`, `podcastListsSwitch(on)`, `podcastStreamSeen()` (keeps the
previous time as `state.stream_seen_before` for the marks).

71
docs/progress-sync.md Normal file
View file

@ -0,0 +1,71 @@
# Reading position: the last place read wins
Since 2026-10-12 the place to continue a book is **where it was last read, on any device**, not the
furthest place ever reached. The furthest place is still kept, as a hint ("Weiteste Stelle").
This file is the contract between the server (books/views.py `save_progress`), the web reader
(static/js/app/46-readpos.js, with the reader in 62-reader.js), diora-android (Ops.kt / SyncEngine.kt / native-*.js) and diora-desktop
(src/main/ops.js / sync.js). They must apply the same rules.
## The state per book
| field | meaning |
|---|---|
| `scroll_fraction`, `position_anchor` | the continue point (anchor `"block:inner"`, empty for PDFs) |
| `read_at` | when the reader was **at** that place, ms since epoch, **server clock** |
| `device` | who read there (`"Browser"`, `"diora Desktop"`, `"Handy"`, `"tolino"` …), for "auf X weitergelesen" |
| `furthest_fraction`, `furthest_anchor` | the furthest place ever reported (only ever grows) |
`last_read` in `/books/` is `read_at` as ISO time (it used to be the time of the last save attempt).
## Writing: `POST /books/<id>/progress/`
```json
{"scroll_fraction": 0.42, "position_anchor": "187:0.62", "read_at": 1760270000000,
"sent_at": 1760270004000, "device": "diora Desktop"}
```
* `read_at` is **the moment the reader arrived at this position** (the last time the visible position
changed because of the reader), in the **device's** clock, not the moment of saving. Saving an
unchanged position again — every 10 s, on hide, on close, on unload — therefore carries the old time
and can never beat a newer position from another device. A device that only *shows* a position it
got from the server (just opened, nothing moved) reports the server's `read_at` back unchanged.
* `sent_at` is the device clock **when the request leaves** (an outbox sets it at flush time, not
when queuing). The server corrects the device's clock error with it:
`t = read_at + (server_now − sent_at)`, then `t = min(t, server_now)`.
So a tolino whose clock is hours off still orders correctly, and nothing can win "from the future".
* The server stores the position if `t >= stored read_at` (or nothing is stored, or `force`).
Older writes (an offline device catching up late) are **not** stored as the continue point.
* Every write — stored or not — raises `furthest_*` if it is further (same comparison as before:
anchor block, then inner fraction; `scroll_fraction` when an anchor is missing).
* `force: true` (the explicit "Diese Stelle als Fortschritt setzen") stores regardless, with `t = server_now`.
* **Old clients** (no `read_at`): stored only if **strictly further** than the stored position
(the old rule, but `>` so an idle old client re-saving its place changes nothing), with `t = server_now`.
This keeps not-yet-updated apps from pulling the continue point back.
* The response is the state after the write:
`{"ok": true, "stored": bool, "scroll_fraction", "position_anchor", "read_at", "device",
"furthest_fraction", "furthest_anchor", "server_now"}`.
## Reading
`/books/` and `/api/sync/` (`book_progress`) carry all fields of the state, `read_at` in ms (server clock),
plus `server_now` (ms) so a client can estimate its clock offset `server_now − device_now`.
## Clients
* **Local apply / outbox** (Android, desktop, web cache): the same rule as the server, in server time:
a local write converts its `read_at` with the last known offset (0 if none), stores it if newer
than or equal to what the local list holds. Two queued writes for the same book: keep the one with
the later `read_at` (force stays sticky). After a pull, a pending write is reapplied with that rule:
newer than the server's → it shows (and is sent); older → the server's position shows (it's still sent,
for `furthest`).
* **Never save before the position is restored** after opening a book, and never save a position the
reader didn't reach by reading: while a jump is tentative ("Zurück zum Text" after a footnote or link,
"Weit gescrollt — Zurückspringen"), the previous position and its `read_at` are what gets saved, until
the reader returns (nothing changes) or carries on reading there (then the jump target counts, with
its own time).
* **Opening a book** uses the newer of server state and local (not yet sent) state, by `read_at`.
* **Coming back to an open book** (window/app in front again, or after a pull): if the server holds a
newer position from another device than the one shown, offer it: "Auf <device> weitergelesen —
Springen". Do not move the page without asking.
* If `furthest_*` is clearly ahead of the continue point when a book opens, offer
"Weiteste Stelle: 63 % — Springen".

96
docs/projects.md Normal file
View file

@ -0,0 +1,96 @@
# Projekte
Projects of a preset: the first is **Forschung** (a research project: sources, citations,
notes; Radar and Schreiben follow). In the navigation (`nav.js`, section and group `projekte`,
flag `side`): a tab beside Home · Lesen · Hören on a phone, at the foot of the sidebar and of
the wide bar above Suche · Abos & Quellen. The search moved out of the phone's tab bar: a
magnifier in the bar on top (`suche` is a footer section; the apps' section bars show it).
## End-to-end encrypted
Like the books (docs/library.md): a project's data (`{name, description}`) and every item's
data are AES-GCM blobs with the user's key (`32-crypto.js`, iv hex, ciphertext base64). The
server stores them and knows only: a project exists, its preset, archived, and what an item
points to — `kind` + `ref` (a book's, episode's or article's id, never its title).
## Items
| kind | ref | data |
|---|---|---|
| `paper` | – | `{key, csl, note, doi}` (CSL-JSON from doi.org) |
| `book` | book id | `{key, csl, note, type}` (CSL from the book's meta) |
| `episode` | episode id | `{key, csl, note, podcast, feed, position}` (position: seconds, "ab 12:34") |
| `article` | feeds item id | `{key, csl, note}` |
| `note` | – | `{title, body (Markdown), folder}` |
A book, episode or article is in a project once (adding again answers `existing: true`).
`key` is the citation key for `[@key]` / `[@key, S. 12]` in notes: first author's family name
(or the podcast's), year, first word of the title — unique in the project, editable.
Notes: Markdown (headings, lists, quotes, bold/italic, links); the view shows citations as
"(Amodio & Jost 2007, S. 3)" and a list of the works cited. Saved as one types (0.9 s).
Export: BibTeX and CSL-JSON of the sources (all or the chosen kind).
## Linking from elsewhere
`DioraProjects.link(kind, ref, info)` opens the sheet "Mit Projekt verknüpfen" (the projects,
the ones it is in ticked, ＋ Neues Projekt). Called from the book sheet (library, apps), the
article reader (◇ in its bar), the episode sheet and the player (apps, native-podcast.js).
`DioraProjects.of(kind, ref)` → `[{id, name}]` the projects it's in; `onChange(fn)` after changes.
## Desktop: a window of its own
The project's menu (⋯) has "In neuem Fenster öffnen" when `dioraDesktop.openWindow` is there:
the desktop opens the page with `?project=<id>`; `projects.js` then shows only that project
(`html.prj-solo` hides the navigation and the dock).
## Writing and conflicts
Every write sends `base_iv` (the IV it changed). Changed elsewhere since: 409 with the current
state. A note's text then goes beside it as a copy "(Konflikt)"; other data reloads.
## API (signed in)
- `GET /projects/` → `{projects: [{id, preset, data_ct, data_iv, archived, position, items, …}], links: [{project, kind, ref}], presets}`
- `POST /projects/create/` `{preset, data_ct, data_iv}` → `{project}`
- `GET /projects/<id>/` → `{project, items: [{id, kind, ref, data_ct, data_iv, created_at, updated_at}]}`
- `POST /projects/<id>/` `{data_ct, data_iv, base_iv}` and/or `{archived}` / `{position}`
- `POST /projects/<id>/delete/`
- `POST /projects/<id>/items/add/` `{kind, ref?, data_ct, data_iv}` → `{item, existing}`
- `POST /projects/items/<id>/` `{data_ct, data_iv, base_iv}` → `{item}` (409 `{item}` when stale)
- `POST /projects/items/<id>/delete/`
- `GET /projects/doi/?q=<DOI | doi.org link | arXiv link or id>` → `{doi, csl}` — fetched from
doi.org (Crossref, DataCite …) as CSL-JSON, cached a week, nothing else kept.
## Radar
A project can have a Radar (`projects/radar.py`, `projects/papers.py`): every week the new papers
from PubMed, arXiv, bioRxiv and medRxiv, sorted by Claude into the config's groups (the first
one — "lesenswert" — with what it's about, how solid it is, the political hook), the week's
news (web search), a basic term explained. One Claude batch (Opus, half price): about 0.40 $
a week. Ported from the "Neuropolitik Grundlagen" session's `F:
europolitik ools`
(fetch_papers.py, digest_batch.py); the digest is their bundle format (version 1).
- Config per radar (`Radar.config`, presets in `projects/radar_presets.py`): PubMed queries per
topic, arXiv query, keyword pattern for the preprint servers, groups and their labels, the
prompts, weekday and hour (Europe/Berlin), days. **Plain on the server**: it searches with them.
The papers and digests are public material, plain too (`RadarDigest.bundle`).
- The decisions (gelesen / später / verworfen / in Quellen) are the user's: an encrypted item
of kind `radar` (`{decisions: {paper id: state}}`). "→ Quellen" makes a `paper` source
(citation data from doi.org when there's a DOI, else from the record).
- Runs from the host's crontab every half hour: `docker exec diora-prod python manage.py radar`
(submits what is due, collects finished batches). Needs `ANTHROPIC_API_KEY` in the container.
`radar --setup <project> neuropolitik` gives a project its Radar; `radar --import <project>
<file>` takes a digest made elsewhere; `radar --seen <project> <file>` ids reported before.
- API: `GET /projects/<id>/radar/` → `{radar: {enabled, title, groups, group_labels, weekday,
hour, days, sources, running, submitted, last_run_at, last_error, next_run, key}, digests:
[{id, generated, bundle}]}` (the latest 12); `POST …/radar/settings/` `{enabled}`;
`POST …/radar/run/` (fetch at the next cron run); `POST …/radar/import/` (a bundle, e.g.
`digest_batch.py --diora-url`).
## Next
Papers as books (a paper's PDF into the library, its highlights
into the project's notes), Schreiben (drafts with the bibliography in a citation style),
the projects in the search, more presets.

137
docs/radio.md Normal file
View file

@ -0,0 +1,137 @@
# Radio, Ambient, Fokus
The oldest part of diora (it began as a radio player): internet radio with saved stations,
history and track titles, the generative Ambient player, and the web page's Fokus timer.
Written down 2026-10-24 from the code; it is the reference for the three apps.
## Server: the `radio` app
Models (`radio/models.py`):
| model | what |
|---|---|
| `SavedStation` | a user's station: name, url (unique per user), bitrate, country, tags, `favicon_url` (the logo), `is_favorite`, notes |
| `StationPlay` | one listening: station, started/ended; hour, weekday, weekend filled on save (for the recommendations) |
| `TrackHistory` | a track title heard (user, or `session_key` signed out), `scrobbled` |
| `FeaturedStation` | stations picked in the admin for everybody (order, active) |
| `FocusSession` | a finished Fokus round |
Also: `podcasts.PlayerState` with `kind='radio'` (the player sync between devices) and
`accounts.UserProfile.focus_station_url/_name` (the station the Fokus timer plays).
Endpoints (`radio/urls.py`):
- `GET /` — the page (`templates/radio/player.html`). The saved stations, the history (50) and
the featured stations come rendered into it (`#saved-tbody`, `#history-tbody`,
`.featured-section`) and as `INITIAL_SAVED` (with `play_count`), `INITIAL_FEATURED`,
`USER_FOCUS_STATION`. The apps take these parts from it (see below).
- `GET /radio/sse/?url=&station=` — server-sent events with the track title, `data: {"track": …}` when it
changes, `{"error": "stream ended"}` at the end. The server opens the stream itself with
`Icy-MetaData: 1` and reads `StreamTitle` (`radio/icy.py`). Which addresses it opens
(`radio/streamcheck.py`, since 2026-10-24): never one in a private network (loopback,
10.x, 172.16–31.x, 192.168.x, link-local, the Docker network); **without an account only the
directory's stations** — the page's curated lists (read from `16-stations.js`), the featured
stations, the creamfresh stream, and what radio-browser.info lists (`station=` its station
id, which the page sends for search results; else by the address; kept a day). An address of
one's own is for saving, and saving needs an account: signed out, a pasted address offers
signing in instead of ▶.
- `POST /radio/record/` — a track into the history; `scrobble: true` sends it to Last.fm when the
account has it linked (`radio/lastfm.py`; "Artist - Title", else the station as artist).
- `POST /radio/save/` (by url: create or update), `/radio/remove/<pk>/`, `/radio/favorite/<pk>/`,
`/radio/notes/<pk>/`; `GET /radio/history/` (last 200), `POST /radio/history/<pk>/delete/`.
- `POST /radio/play/start/` → `play_id`, `/radio/play/stop/`; `GET /radio/recommendations/` —
the five stations most played around this hour on this kind of day, with a German label.
- `POST /radio/import/` — an `.m3u`/`.m3u8` file into the saved stations (`#EXTINF` names) →
`{added, skipped}`.
- `POST /radio/focus/record/`, `GET /radio/focus/stats/`.
- `GET /radio/affiliate/?track=` — artwork/album from iTunes (an Amazon link only with
`AMAZON_AFFILIATE_ENABLED`).
- `GET /radio/creamfresh-stream/`, `POST /radio/creamfresh-feedback/` — the user's own station
(radio.creamfresh.xyz) through the server, which adds its login; 👍/👎 for the DJ.
- `GET /radio/stream-player/` — an old standalone player page; nothing links to it.
**Station search** is not on the server: the page asks radio-browser.info directly
(`de1.api.radio-browser.info/json/stations/search`, 50, by click count). A pasted `http(s)://`
address is offered to play or save as it is.
## Web page (`static/js/app/`)
| file | what |
|---|---|
| `01-core.js` | the globals: `audio` (one `<audio>`), `currentStation`, `currentTrack`, `sseSource`, `isPlaying`, `currentPlayId` |
| `10-radio.js` | `playStation(url, name, id)`, `stopPlayback`, `togglePlayStop`, the SSE (`startMetadataSSE` → `recordTrack`), volume (0–255, `diora_volume`), search, saving, favourites, recommendations, the creamfresh votes, Media Session |
| `16-stations.js` | mood chips (they search a tag), curated lists, featured stations, the occasional donation hint, notes |
| `29-m3u.js` | M3U import |
| `80-radio-sidebar.js` | the compact radio player of the sidebar |
| `24-ambient.js` | the Ambient tab (below) and `showRadioTab` |
| `14-focus.js` | Fokus (below) |
Sub-tabs `RADIO_SUB_TABS = ['search', 'saved', 'history']` (`20-tabs.js`), hash `#radio/saved`,
remembered in `diora_active_radio_tab`. On an https page an `http://` stream would be blocked:
the page opens it in a new tab — unless the host plays it (`window.DIORA_PLAYS_HTTP_STREAMS`,
set by the apps). No HLS: what `<audio>` plays.
## Ambient (`static/ambient/index.html`, "Drift")
No sound files: a generative WebAudio synthesizer. Modes (`MODES`): ambient music (chords,
bells, grains), noise colours, nature (rain, storm, sea, wind, stream, fire, crickets),
machines (fan, cabin, train), focus and sleep (binaural delta–beta, singing bowls,
heartbeat); each with two sliders and seeded presets in the URL hash.
The page holds it in an iframe, loaded when first needed (`24-ambient.js`); they talk by
`postMessage` (`{type: 'ambient', cmd: 'start'|'stop'|'mode'|'volume'}` in,
`{state: 'playing'|'stopped'}` out). Mode remembered in `diora_ambient_mode`.
Hooks for the apps: `__dioraAmbientStart/Stop/Cmd/Info/Playing`.
**One sound at a time:** Ambient starting stops the radio (`stopPlayback(false)`) and pauses a
podcast (`__dioraPausePlayer()`; Android: `podcastPause()`, pause only, never toggle); the
radio starting stops Ambient.
## Fokus (web page only)
A Pomodoro timer (25/5 min) in its own section (`nav.js`: `focus`, `webOnly`): it plays the
chosen focus station, records a `FocusSession` after each work phase, pauses the sound for the
break; with a do-not-disturb full-screen mode (`toggleDND`) and a statistics table.
## Android (diora-android)
- **One player** for radio, podcasts and audiobooks: `RadioService.kt`, a Media3
`MediaLibraryService` with one ExoPlayer (15 s connect / 20 s read timeout; diora's own
server gets the login headers — the creamfresh proxy). After a pause of more than 3 s a
station jumps back to live. Android Auto: Saved, Featured, Queue, voice search by name.
Resuming after a restart plays the last station. Radio always streams on mobile data (the
mobile-data rules are for episodes).
- `Radio.kt` — the page's controller: play, stop, `track(title)` (new metadata without
rebuffering), state to the page (`window.__dioraRadio`).
- `RadioPage.kt` — the saved stations, history and featured stations exist only in the
rendered `/`: the app fetches it with its login, cuts those parts out into `radio-page.json`
and splices them into its own page at markers that `tools/build_web.py` set
(`<!--diora:saved|history|featured-->`, `__DIORA_RADIO_*_JSON__`). So they show offline too.
- `LocalServer.kt` passes `/radio/sse/` through as a stream (padded with SSE comments: WebView
hands intercepted data over only when its buffer is full); POSTs go through `native.js`.
- `web-native/static/native-radio.js` — turns the page's `audio` into a remote of the native
player (`radioPlay`/`radioStop`, counters against stale echoes), shows errors ("Sender nicht
erreichbar"), draws the dock/rail (from `nav.js`), the mini player and the radio bar.
- `AmbientService.kt` — Ambient as a foreground media service (notification with Stopp, audio
focus, stops when headphones are pulled); started and stopped from the page
(`ambientState`). Widgets for Ambient and resuming (`CardWidgets.kt`).
- Sleep timer (`SleepTimer.kt`, `SleepRules.kt`): counts only playing time (Ambient too), fades
out over the last 10 s, shaking resets it, can stop after N episodes, can switch itself on at
night (22–6 h); one that counted nothing for 3 h is dropped.
- Data usage: the stream counts as `RADIO_STREAM`, `/radio/` and `/` as `RADIO_DATA`
(`DataUsage.kt`).
## Desktop (diora-desktop)
A copy of `native-radio.js`; `src/web/desktop.js` plays with a hidden `Audio()` of its own
(`DioraNative.radioPlay/Stop/State/Refresh/Track`; media keys: pause = stop) and re-reads `/`
like RadioPage.kt (at most every 20 s). `http://` streams go through the local router
(`/__stream?u=`, `src/main/router.js`) without their ICY data; the SSE is passed through with the
login; `backgroundThrottling: false` keeps the titles coming.
## Known weak spots
- The login for the creamfresh stream is written into `radio/views.py`; it belongs in the
environment like the other secrets.
- Every listener's titles cost the server a second connection to the stream.
- No recording, no alarm; station logos only from radio-browser's favicon.

53
docs/refactoring.md Normal file
View file

@ -0,0 +1,53 @@
# Refactoring (started 2026-10-14)
diora grew fast across four code bases that share one web front end:
diora-web (server + app.js and friends), diora-android (Kotlin + web-native/*.js),
diora-desktop (Electron + desktop.js) and the vendored copies between them.
This is the plan for tidying it up without breaking anything.
## Rules
* **No behaviour changes in a refactoring step.** Features and fixes go in their own commits.
* **Tests before moving code.** A step only starts when what it touches is covered.
* **Small steps, each one releasable**, each one checked on web, desktop and Android.
* Whoever owns a file does its refactoring (web-native/* → the Android session;
app.js, home.js, templates, desktop → the desktop session); shared contracts live in docs/.
## The numbers (2026-10-14, before step 2; app.js is now `static/js/app/`)
| file | lines |
|---|---|
| static/js/app.js | 6 864, about 275 functions, about 120 globals, 45 sections |
| static/podcasts/native-podcast.js (copy from diora-android) | 4 554 |
| static/css/app.css | 2 010 |
| static/js/podcasts-web.js | 1 899 |
| podcasts/views.py | 2 505 |
| web-native/native.css | 2 150 |
Web JavaScript has no unit tests and four Playwright smoke tests that run nowhere.
The reading-position rules exist four times (books/views.py, app.js, desktop ops.js,
Android Ops.kt), each tested separately or not at all.
## Steps
1. **Safety net** (in progress)
* Playwright tests that run locally and in CI against a throwaway database: log in, open a
book (an EPUB fixture), read, the position is saved and restored, a highlight, a note
in the margin, the theme switch, radio and podcasts tabs load.
* Shared test vectors for the reading-position rules (`docs/progress-cases.json`), run by
the Django tests, desktop's node tests and Android's OpsTest — one contract, checked
in all four places.
2. **Split app.js** (done: 656520f, sw v106 — `static/js/app/NN-*.js`, 33 files; the
template lists them in order) by its sections into plain scripts loaded in order (radio, focus,
ambient, crypto, epub, books, reading position, reader, tts, highlights, margin, …). They
stay classic scripts sharing globals, so the native layers that wrap `window.*` keep
working unchanged; template, sw.js SHELL and Android's build_web list them. No code
changes inside the moved functions.
3. **Dead code**: features and helpers nothing calls any more (found by the split and by
coverage of the tests), removed one at a time.
4. **Server**: podcasts/views.py into modules by area (feeds, episodes, queue, player sync,
import); shared helpers (`_require_auth`, JSON body parsing) in one place.
5. **CSS**: the remaining fixed colours to the theme tokens (docs/theme.md); app.css split
like app.js.
6. **Copies between repositories**: one sync script per direction with a check in CI that
the copy matches its source revision (today: sync_native.sh, sync_podcast_ui.py).

32
docs/search.md Normal file
View file

@ -0,0 +1,32 @@
# Suche
One field for all of diora: a section of its own in the navigation (`static/js/nav.js`,
section and group `suche`, flag `side`):
- phone (the apps' tab bar, the web page's narrow bar): beside Home · Lesen · Hören;
- the apps' sidebar and the web page's wide bar: at the foot, above Abos & Quellen.
It's never the section the app opens in next time (`transient`: `diora_nav_section` isn't written for it).
## What is searched where
| Area | Where | Opens |
|---|---|---|
| Bücher, Zeitschriften | on the device (`_lastBookListData` / `bookMetaCache`): titles are encrypted, the server never sees them; title, author, series, file name, folders; editions of one work once | `openBook(id)` |
| Hörbücher, Podcasts (Abos) | server: title, author | `__dioraAudiobookOpen(id)` / `__dioraPodcastOpenFeed(id)` |
| Folgen | server: the full-text index (`podcasts/discover.py`, titles and shownotes, podcasts only) | `__dioraPodcastOpenEpisode(row)` if the podcast UI has it, else the podcast's page; ▶ plays (`podcastPlayEpisode`) |
| Artikel | server: titles and authors of news and blog items not expired | `__dioraFeedsOpenItem(id)` |
| Nachrichtenquellen, Blogs | server: title, address, category | the news / blogs |
| Radio | server: saved stations, name, tags, country | `playStation()` |
Every word has to be there as the start of a word ("plattf" finds "Plattform"); case and
diacritics don't count. "Darüber hinaus" hands the query to Abos & Quellen (new podcasts,
YouTube, feeds) or the radio directory search. The last eight queries (Enter or a hit opened)
are kept on the device. `DioraSearch.open(q)` opens it from anywhere.
## API
`GET /search/?q=…` (signed in) →
`{q, podcasts, podcasts_total, audiobooks, audiobooks_total, episodes, sources, articles,
articles_total, stations}`; `{q, empty: true}` for less than two letters. Episodes come as the
episode lists give them (`_episode_list_data`). Eight per area (articles sixteen).

47
docs/sources.md Normal file
View file

@ -0,0 +1,47 @@
# Abos & Quellen (navigation phase 2)
One place to manage every subscription: podcasts, YouTube channels, audiobooks,
news sources and blogs. The everyday views (Strom, Merkliste, Wiedergabeliste — docs/podcasts.md —,
Nachrichten, Blogs, Hörbücher) show content; adding, changing and removing sources happens
here (their own small entry points lead here).
## Where
`DioraNav` section `sources` ("Abos & Quellen", icon ⚙), panel `#tab-sources`,
`group: 'more'`, `footer: true`:
- not in `groups()` (no tab of its own on a phone);
- in `sections()` with `footer: true`: the sidebar puts it at the bottom, above
"Menü"; the phone's app menu has an entry; diora Reader's bar leaves it out;
- the web page's bar: at the end (wide), as ⚙ (narrow).
## The page (`static/js/sources.js`, diora-web, every platform)
- **Hinzufügen**: one field for an address or search words.
- a YouTube address → its channel (`/podcasts/search/?q=`), as Podcast or Hörbuch;
- another address → `/feeds/discover/`: every feed found, with the kind it
looks like (`suggested_kind`: `podcast` when most entries carry sound or
video, else `news`/`blog`) and whether it's subscribed already
(`subscribed` as a feed, `podcast_subscribed`); the kind can be changed:
Podcast · Hörbuch · Nachrichten · Blog;
- words → the podcast directory (`/podcasts/search/`).
Podcast/Hörbuch subscribe with `POST /podcasts/feeds/add/` (then
`settings {kind: 'audiobook'}`), Nachrichten/Blog with `POST /feeds/sources/add/`.
- **The list**: all sources, filter chips Alle · Podcasts · YouTube · Hörbücher ·
Nachrichten · Blogs · Probleme (fetching fails), and a search over the names.
- **A source's actions** (tapping it): open its content, its settings (the
existing editors), change its kind (Podcast ↔ Hörbuch, Nachrichten ↔ Blog),
unsubscribe.
- **Import & Export**: OPML for podcasts and feeds.
## Hooks it uses (each optional: without it, it goes to the section instead)
| Hook | From | Does |
|------|------|------|
| `__dioraPodcastFeedSettings(feedId)` | native-podcast.js | opens the podcast's settings sheet |
| `__dioraPodcastOpenFeed(feedId)` | native-podcast.js | shows the podcast's episode list (Podcasts section) |
| `__dioraFeedsEdit(sourceId)` | feeds.js | opens the source's editor (Nachrichten/Blogs, manage view) |
| `__dioraFeedsStale()` | feeds.js | its state is old: load again when shown |
| `__dioraAudiobookOpen(feedId)` | audiobooks.js | opens the book |
The host calls it uses: `podcastState`, `podcastFeedSettings`, `podcastRefresh`
(after subscribing), `podcastFeedRemoved` (after unsubscribing).

53
docs/theme.md Normal file
View file

@ -0,0 +1,53 @@
# Light and dark
diora has two colour schemes, **dark** (the old look) and **light**, chosen by the reader:
"Dunkel", "Hell" or "Automatisch" (as the operating system says). This file is the contract
between diora-web (app.css, home.js, feeds.js, templates, and static/podcasts/podcast-ui.css —
a copy of diora-android's native.css by tools/sync_podcast_ui.py), diora-android (native.css,
native-*.js; the apps ship diora-web's files too, through build_web.py) and diora-desktop
(desktop.css).
## Switching
* `<html data-theme="dark|light">` is the scheme in effect. Set as early as possible
(an inline script in base.html's `<head>`, before any CSS paints) so nothing flashes.
* The choice is `localStorage.diora_theme` = `"dark"` (default) | `"light"` | `"auto"`.
* `window.__dioraTheme()` → the choice; `window.__dioraSetTheme(choice)` sets and applies it;
`window.__dioraThemeChanged` (event `diora-theme` on `window`, `detail` = the scheme in
effect) lets native code follow (Android status/navigation bars, Electron's window colour).
* "auto" follows `prefers-color-scheme` live — except in the Android app, whose WebView
answers that query from the app's own (dark) theme, not the system's: there the page asks
`DioraNative.systemScheme()` ("dark" | "light") and the app calls
`window.__dioraSystemScheme(scheme)` when the system switches (uiMode change, or on return).
* The desktop app (Electron) sets `nativeTheme.themeSource` to the scheme for a fixed choice
(title bar, scrollbars, form controls match) but leaves it on `"system"` for "auto", so
`prefers-color-scheme` there is the system's; the page tells it the choice along with the
scheme (`dioraDesktop.setTheme(scheme, choice)`, desktop 0.1.31).
* E-ink mode (`html.nui-eink`) keeps its own black-on-white rules and ignores this.
* The reader has its own themes; its default is now "Wie die App" (`readerSettings.theme = 'app'`):
dark → reader dark, light → reader bright. Sepia, Dunkel and Hell stay choosable.
* A wallpaper (the radio page's background image) keeps its own contrast handling
(`body.bright-bg`, the ◑ button); without a wallpaper the scheme decides.
## Colours: only through these variables
| variable | dark | light | for |
|---|---|---|---|
| `--bg` | `#000` | `#f6f5f1` | the page |
| `--fg` | `#fff` | `#17171a` | text |
| `--muted` | `#9a9a9a` | `#66666b` | secondary text, icons |
| `--card` | `#0b0b0b` | `#ffffff` | cards, rows that stand out |
| `--panel` | `#111` | `#ffffff` | sheets, menus, docks, bars |
| `--hover` | `#141414` | `#ecebe6` | hover / pressed |
| `--raised` | `#1a1a1a` | `#e4e2dc` | inputs, tracks of sliders, chips |
| `--border` | `#2a2a2a` | `#d9d6ce` | lines |
| `--border-strong` | `#444` | `#b4b0a6` | lines that must show (inputs, focus off) |
| `--accent` | `#e63946` | `#d0263a` | the red: brand, active, progress |
| `--on-accent` | `#fff` | `#fff` | text on the red |
| `--scrim` | `rgba(0,0,0,.6)` | `rgba(20,20,20,.35)` | behind sheets |
| `--shadow` | `none` | `0 1px 3px rgba(0,0,0,.08)` | cards in light (no lines needed in dark) |
Old names stay as aliases: `--nui-muted: var(--muted)`, `--nui-panel: var(--panel)`,
`--text: var(--fg)`, `--text-muted: var(--muted)`, `--bg-card: var(--card)`.
Images, album art, covers and Ambient's own picture keep their colours. Status colours
(`--green`, `--yellow`, `--red`) may stay as they are.

76
docs/youtube.md Normal file
View file

@ -0,0 +1,76 @@
# YouTube channels as podcasts
A YouTube channel (or playlist) can be subscribed to like a podcast. Only its
sound is played, and **the server never downloads or streams a video or its
sound**: each app finds the audio stream itself.
## Server (diora-web, `podcasts/youtube.py`)
- `POST /podcasts/feeds/add/` takes a YouTube address as `rss_url`: a channel
(`/@name`, `/channel/UC…`, `/c/…`, `/user/…`), a playlist (`?list=…`), a video
(subscribes its channel) or the feed itself. The server reads the channel page
once (feed address, name, picture) and stores the feed
`https://www.youtube.com/feeds/videos.xml?channel_id=UC…` (or `?playlist_id=…`).
`GET /podcasts/search/?q=<YouTube address>` answers with that one channel.
- From then on the feed is refreshed like any other — except that YouTube answers its
channel feeds with a 404 (or 5xx) now and then for no reason (October 2026: two
requests in three), so a YouTube feed is asked up to 12 times, 0.25 s apart, before
its refresh counts as failed (`YOUTUBE_FEED_TRIES` in `podcasts/views.py`). It carries the latest 15
videos. **Shorts are left out.** There is no duration (`duration_seconds` 0)
until a player knows it: `POST /podcasts/progress/save/` takes an optional
`duration_seconds` (number, 1 s … 7 days), stored only while the episode has none.
- An episode from such a feed has
- `audio_url` = `https://www.youtube.com/watch?v=<11-character id>`. This is
**the** marker: a client recognises a YouTube episode by this prefix;
- `guid` = `yt:video:<id>`, `artwork_url` = the video's thumbnail,
`description` = the video description as HTML (links, line breaks; timestamps
like `00:00 Intro` work as usual).
- Nothing on the server touches the sound of such an episode: no ID3 chapters,
no transcripts, and `/podcasts/episodes/<id>/silences/` answers
`{"ready": true, "spans": [], "error": "youtube"}` (skip silence is up to the
app, as on Android anyway).
- `POST /podcasts/feeds/<id>/move/ {"rss_url": …}` gives an existing subscription a
new address and keeps everything (settings, episodes, places, played and favourite
marks, queue). Moved to a YouTube address, every episode whose file is named after
its video (`…/<11-character id>.mp3`, as podsync-like servers name them) becomes
that video (`guid` `yt:video:<id>`, `audio_url` its `watch?v=` page), so the
channel's feed finds it again instead of adding it twice. Used to move channels
that came through such a server (it downloaded the sound on the server) to the
channel itself.
## Ad and outro: SponsorBlock
What goes out of the sound — the advertisement and the closing words — is other people's
work: SponsorBlock (`sponsor.ajay.app`) lets listeners mark a video's parts by category.
`podcasts/sponsor.py` asks it once per episode (only `sponsor` and `outro`, only parts a
listener skips, never a part marked `full`) and keeps the answer on the episode
(`PodcastEpisode.sponsor`); `GET /podcasts/episodes/<id>/sponsor/` answers with it,
`{"ready": true, "v": 1, "spans": [[from, to], …]}`, empty for an episode that isn't a
video and with an `error` beside empty spans when SponsorBlock couldn't be asked. Nothing
else is fetched — no intro, self-promotion, interaction reminder or non-music section.
An answer counts for a week, a failure for an hour (segments are added later; a failure was
probably the network). A player jumps over those parts and never runs them faster: the
setting is `skip_sponsor` of this device (on by default, in the play and speed menus), and
it does something only for YouTube episodes. Nothing of the sound leaves the device for
this — the video id does, to sponsor.ajay.app, when the server asks (never the sound, and
never for an episode that isn't a video).
## Apps
A client that sees a `watch?v=` address resolves it **itself, on the device,
to an audio-only stream** (no video track), and plays or downloads that.
| Client | How |
|---------|-----|
| Android | NewPipeExtractor: the original audio track, progressive, opus before m4a, ≤ 160 kbit/s; resolved when the stream is opened and once more on a 403; downloads in 10 MB ranges (in one go YouTube throttles to ~50 KB/s) |
| Desktop | yt-dlp in the main process (`-f bestaudio`), Electron itself as its JavaScript runtime; yt-dlp is fetched from its GitHub release on first use and kept up to date (`src/main/youtube.js`) |
| Browser | can't (YouTube doesn't let a web page read its pages, and a stream address only works for the device that asked for it): the episode is listed but says it plays in the app |
Stream addresses are bound to the device's IP address and run out after a few
hours: resolve shortly before playing, and again when the player reports an
error (podcasts-web.js reloads after a media error anyway).
Progress, queue, played state etc. work as for any episode: they belong to the
episode on the server, not to the stream address.

7
e2e/env.js
View file

@ -1,6 +1,9 @@
// Shared constants between playwright.config.js and e2e/global-setup.js.
const PORT = 8000;
// E2E_PORT / E2E_PYTHON: a port that no dev server holds (the suite must never
// run against a real database), and the Python with the requirements installed.
const PORT = Number(process.env.E2E_PORT || 8000);
const BASE_URL = `http://127.0.0.1:${PORT}`;
const PYTHON = process.env.E2E_PYTHON || 'python3';
const SERVER_ENV = {
...process.env,
@ -15,4 +18,4 @@ const TEST_USER = {
password: 'e2e-test-pw-12345',
};
module.exports = { PORT, BASE_URL, SERVER_ENV, TEST_USER };
module.exports = { PORT, BASE_URL, PYTHON, SERVER_ENV, TEST_USER };

14
e2e/global-setup.js
View file

@ -11,7 +11,7 @@ const fs = require('fs');
const path = require('path');
const { execFileSync } = require('child_process');
const { chromium } = require('@playwright/test');
const { BASE_URL, SERVER_ENV, TEST_USER } = require('./env');
const { BASE_URL, PYTHON, SERVER_ENV, TEST_USER } = require('./env');
const ROOT = path.join(__dirname, '..');
const DB_PATH = path.join(ROOT, 'data', SERVER_ENV.DIORA_DB_NAME);
@ -22,14 +22,14 @@ module.exports = async function globalSetup() {
// Fresh DB every run.
fs.rmSync(DB_PATH, { force: true });
execFileSync('python3', ['manage.py', 'migrate', '--noinput'], {
execFileSync(PYTHON, ['manage.py', 'migrate', '--noinput'], {
cwd: ROOT,
env: SERVER_ENV,
stdio: 'inherit',
});
execFileSync(
'python3',
PYTHON,
[
'manage.py',
'shell',
@ -45,6 +45,14 @@ User.objects.create_user(${JSON.stringify(TEST_USER.username)}, "", ${JSON.strin
);
fs.mkdirSync(AUTH_DIR, { recursive: true });
// A big PDF (e2e/make_pdf.py): opening draws only the pages near the view.
execFileSync(PYTHON, [path.join(__dirname, 'make_pdf.py'), path.join(AUTH_DIR, 'big.pdf'), '400'], { stdio: 'inherit' });
// The book the reader tests read (e2e/make_epub.py).
execFileSync(PYTHON, [path.join(__dirname, 'make_epub.py'), path.join(AUTH_DIR, 'book.epub')], { stdio: 'inherit' });
// Two more for uploading several at once (navigation.spec.js).
for (const n of ['Eins', 'Zwei']) {
execFileSync(PYTHON, [path.join(__dirname, 'make_epub.py'), path.join(AUTH_DIR, `many-${n}.epub`), `Mehrfach ${n}`], { stdio: 'inherit' });
}
const browser = await chromium.launch();
const page = await browser.newPage({ baseURL: BASE_URL });

58
e2e/make_epub.py Normal file
View file

@ -0,0 +1,58 @@
"""A small EPUB for the e2e tests: 12 chapters of numbered paragraphs (long enough
to scroll and to have a reading position), one footnote. Written to the path in
argv[1]; e2e/global-setup.js calls it for every run."""
import sys
import zipfile
CONTAINER = """<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles>
</container>"""
def chapter(n):
paras = []
for p in range(1, 41):
text = f'Kapitel {n}, Absatz {p}. ' + ' '.join(
f'Wort{w} im Absatz {p} des Kapitels {n} steht hier, damit der Absatz einige Zeilen lang wird.'
for w in range(1, 6))
if n == 1 and p == 3:
text += ' Mit Fußnote<a href="ch12.xhtml#fn1" epub:type="noteref"><sup>1</sup></a>.'
paras.append(f'<p id="c{n}p{p}">{text}</p>')
notes = ''
if n == 12:
notes = '<aside epub:type="footnote" id="fn1"><p>1 Der Text der Fußnote.</p></aside>'
return f"""<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops">
<head><title>Kapitel {n}</title></head>
<body><h1>Kapitel {n}</h1>{''.join(paras)}{notes}</body></html>"""
def main(path, title='E2E Testbuch'):
chapters = range(1, 13)
manifest = ''.join(f'<item id="c{n}" href="ch{n}.xhtml" media-type="application/xhtml+xml"/>' for n in chapters)
spine = ''.join(f'<itemref idref="c{n}"/>' for n in chapters)
nav_items = ''.join(f'<li><a href="ch{n}.xhtml">Kapitel {n}</a></li>' for n in chapters)
opf = f"""<?xml version="1.0" encoding="utf-8"?>
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="id">
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
<dc:identifier id="id">diora-e2e-book</dc:identifier>
<dc:title>{title}</dc:title><dc:creator>diora</dc:creator><dc:language>de</dc:language>
</metadata>
<manifest><item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>{manifest}</manifest>
<spine>{spine}</spine>
</package>"""
nav = f"""<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops">
<head><title>Inhalt</title></head><body><nav epub:type="toc"><ol>{nav_items}</ol></nav></body></html>"""
with zipfile.ZipFile(path, 'w') as z:
z.writestr(zipfile.ZipInfo('mimetype'), 'application/epub+zip', compress_type=zipfile.ZIP_STORED)
z.writestr('META-INF/container.xml', CONTAINER, compress_type=zipfile.ZIP_DEFLATED)
z.writestr('OEBPS/content.opf', opf, compress_type=zipfile.ZIP_DEFLATED)
z.writestr('OEBPS/nav.xhtml', nav, compress_type=zipfile.ZIP_DEFLATED)
for n in chapters:
z.writestr(f'OEBPS/ch{n}.xhtml', chapter(n), compress_type=zipfile.ZIP_DEFLATED)
if __name__ == '__main__':
main(sys.argv[1], *sys.argv[2:3])

12
e2e/make_pdf.py Normal file
View file

@ -0,0 +1,12 @@
"""A PDF of many pages for the reader tests (big scanned books): make_pdf.py OUT [PAGES]."""
import sys
from pypdf import PdfWriter
out, pages = sys.argv[1], int(sys.argv[2]) if len(sys.argv) > 2 else 400
w = PdfWriter()
for _ in range(pages):
w.add_blank_page(width=420, height=595)
w.add_metadata({'/Title': 'Dicker Band', '/Author': 'diora'})
with open(out, 'wb') as f:
w.write(f)

287
e2e/navigation.spec.js Normal file
View file

@ -0,0 +1,287 @@
// The navigation (docs/navigation.md): Home, Lesen (Bücher, Zeitschriften, Nachrichten,
// Blogs), Hören (Hörbücher, Podcasts, Radio, Ambient), on the web page's own bar.
const { test, expect } = require('@playwright/test');
const path = require('path');
test.use({ storageState: path.join(__dirname, '.auth', 'user.json') });
const visiblePanel = page => page.evaluate(() =>
[...document.querySelectorAll('section.tab-panel')].filter(p => p.style.display !== 'none').map(p => p.id));
test('wide: every section in the bar under its group, each shows its panel and view', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 800 });
await page.goto('/#books');
const wide = page.locator('#tabs .nav-wide');
await expect(wide.locator('.nav-group-label')).toHaveText(['Lesen', 'Hören']);
await expect(wide.locator('[data-nav]')).toHaveText(
['Home', 'Bücher', 'Zeitschriften', 'Nachrichten', 'Blogs', 'Hörbücher', 'Podcasts', 'Radio', 'Ambient', 'Fokus', 'Projekte', 'Suche', 'Abos & Quellen']);
await wide.locator('[data-nav="blogs"]').click();
await expect(page).toHaveURL(/#blogs$/);
expect(await visiblePanel(page)).toEqual(['tab-feeds']);
expect(await page.evaluate(() => localStorage.getItem('diora_feeds_view'))).toBe('blogs');
await expect(page.locator('#tab-feeds .fd-seg')).toBeHidden(); // the bar chooses, not the tab
await expect(page.locator('#nav-segments')).toBeHidden();
await wide.locator('[data-nav="audiobooks"]').click();
expect(await visiblePanel(page)).toEqual(['tab-audiobooks']);
await expect(page.locator('#audiobook-shelf')).toBeVisible();
await expect(wide.locator('[data-nav="audiobooks"]')).toHaveClass(/active/);
});
test('narrow: three groups, the sections as a row on top, each group remembers its last', async ({ page }) => {
await page.setViewportSize({ width: 390, height: 800 });
await page.goto('/#news');
const narrow = page.locator('#tabs .nav-narrow');
await expect(narrow.locator('[data-nav]')).toHaveText(['Home', 'Lesen', 'Hören', 'Projekte', '', '']);
await expect(narrow.locator('[data-nav="suche"] svg.dnav-ic')).toBeVisible(); // the magnifier
await expect(narrow.locator('[data-nav="sources"] svg.dnav-ic')).toBeVisible();
await expect(narrow.locator('[data-nav="lesen"]')).toHaveClass(/active/);
const segs = page.locator('#nav-segments');
await expect(segs.locator('[data-nav]')).toHaveText(['Bücher', 'Zeitschriften', 'Nachrichten', 'Blogs']);
await expect(segs.locator('[data-nav="news"]')).toHaveClass(/active/);
await narrow.locator('[data-nav="hoeren"]').click();
await segs.locator('[data-nav="radio"]').click();
expect(await visiblePanel(page)).toEqual(['tab-radio']);
await narrow.locator('[data-nav="lesen"]').click();
await expect(page).toHaveURL(/#news$/); // back where Lesen was left
await narrow.locator('[data-nav="hoeren"]').click();
await expect(page).toHaveURL(/#radio/);
});
test('old links and direct panel switches land in their section', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 800 });
await page.goto('/#feeds');
expect(await visiblePanel(page)).toEqual(['tab-feeds']);
await expect(page.locator('#tabs .nav-wide [data-nav].active')).toHaveText(/Nachrichten|Blogs/);
await page.evaluate(() => showTab('podcasts'));
await expect(page.locator('#tabs [data-nav="podcasts"]')).toHaveClass(/active/);
await expect(page).toHaveURL(/#podcasts$/);
await page.reload();
expect(await visiblePanel(page)).toEqual(['tab-podcasts']); // remembered
});
test('Suche: books from this device, the rest from the server, a hit opens where it lives', async ({ page }) => {
await page.setViewportSize({ width: 390, height: 800 });
await page.route('**/search/?q=*', route => route.fulfill({json: {
q: 'kapital', podcasts: [{id: 7, title: 'Kapital-Lesekreis', author: 'Rote Hilfe', artwork_url: '', favorite: true}],
podcasts_total: 1, audiobooks: [], audiobooks_total: 0,
episodes: [{id: 70, feed__id: 7, feed__title: 'Kapital-Lesekreis', title: 'Folge 3: Das Kapital, Band 1', duration_seconds: 3600}],
sources: [], articles: [{id: 5, title: 'Kapital und Krise', source: 'Ein Blog', kind: 'blog', published: null, read: false}],
articles_total: 1, stations: []}}));
await page.goto('/#home');
await page.locator('#tabs .nav-narrow [data-nav="suche"]').click();
await expect(page).toHaveURL(/#suche$/);
expect(await visiblePanel(page)).toEqual(['tab-suche']);
await expect(page.locator('#tab-suche .sx-q')).toBeFocused();
await page.evaluate(() => {
_lastBookListData = [{id: 901, title: 'Das Kapital', author: 'Karl Marx', type: 'epub', keyOk: true},
{id: 902, title: 'Gartenbuch', author: 'Jemand', type: 'epub', keyOk: true}];
});
await page.locator('#tab-suche .sx-q').fill('kapital');
const groups = page.locator('#tab-suche .sx-h');
await expect(groups).toHaveText([/Bücher/, /Podcasts/, /Folgen/, /Artikel/]);
await expect(page.locator('#tab-suche .sx-group').first().locator('mark')).toHaveText('Kapital');
await page.locator('#tab-suche .sx-q').press('Enter');
await page.locator('#tab-suche .sx-q').fill('');
await expect(page.locator('#tab-suche .sx-recent .sx-chip')).toHaveText(['kapital']);
// Not where the app opens next time.
expect(await page.evaluate(() => localStorage.getItem('diora_nav_section'))).not.toBe('suche');
});
test('Projekte: a Forschung project, a paper by DOI, a linked book, a note citing them', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 900 });
await page.route('**/projects/doi/?q=*', route => route.fulfill({json: {doi: '10.1038/nn.1234', csl: {
type: 'article-journal', title: 'Neural correlates of political ideology', 'container-title': 'Nature Neuroscience',
author: [{family: 'Amodio', given: 'David'}, {family: 'Jost', given: 'John'}], issued: {'date-parts': [[2007, 9]]},
DOI: '10.1038/nn1979', volume: '10', page: '1246-1247'}}}));
await page.goto('/#projekte');
const tab = page.locator('#tab-projekte');
await tab.locator('[data-act="new"]').click();
await expect(tab.locator('.prj-preset.on')).toContainText('Forschung');
await tab.locator('input[name="name"]').fill('Neuropolitik');
await tab.locator('button[type="submit"]').click();
await expect(tab.locator('.prj-h1')).toHaveText('Neuropolitik');
await expect(page).toHaveURL(/#projekte\/\d+$/);
// The server keeps only ciphertext.
const raw = await page.evaluate(() => fetch('/projects/').then(r => r.json()));
expect(JSON.stringify(raw)).not.toContain('Neuropolitik');
await tab.locator('.prj-add input').fill('https://doi.org/10.1038/nn1979');
await tab.locator('.prj-add button').click();
await tab.locator('[data-add-doi]').click();
await expect(tab.locator('.prj-row .prj-title')).toHaveText(['Neural correlates of political ideology']);
await expect(tab.locator('.prj-key')).toHaveText(['amodio2007neural']);
// From a book's sheet elsewhere: DioraProjects.link.
await page.evaluate(() => DioraProjects.link('book', 4242, {title: 'Das politische Gehirn', author: 'Hans Muster'}));
await page.locator('.prj-picker [data-pick]').filter({hasText: 'Neuropolitik'}).click();
await expect(tab.locator('.prj-row')).toHaveCount(2);
expect(await page.evaluate(() => DioraProjects.of('book', 4242).map(p => p.name))).toEqual(['Neuropolitik']);
await tab.locator('[data-view="notes"]').click();
await tab.locator('[data-act="new-note"]').click();
await tab.locator('[data-note-title]').fill('These');
await tab.locator('[data-note-body]').fill('# Idee\nIdeologie zeigt sich im Gehirn [@amodio2007neural, S. 3] und bei [@muster0jdas].\n\n| Text | Zeit |\n|---|---|\n| Cobb | 2,5 h |\nDanach.');
await expect(tab.locator('.prj-saved')).toHaveText('gespeichert', {timeout: 5000});
await tab.locator('[data-act="preview"]').click();
await expect(tab.locator('.prj-md .prj-cite').first()).toHaveText('(Amodio & Jost 2007, S. 3)');
await expect(tab.locator('.prj-refs li').first()).toContainText('Nature Neuroscience');
await expect(tab.locator('.prj-md td')).toHaveText(['Cobb', '2,5 h']);
await tab.locator('[data-view="sources"]').click();
const [dl] = await Promise.all([page.waitForEvent('download'), tab.locator('[data-export="bib"]').click()]);
const bib = require('fs').readFileSync(await dl.path(), 'utf8');
expect(bib).toContain('@article{amodio2007neural,');
expect(bib).toContain('doi = {10.1038/nn1979}');
});
test('Radar: the week\'s digest, decisions kept encrypted, a paper into the sources', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 900 });
const digest = {id: 1, generated: '2026-10-19', bundle: {version: 1, generated: '2026-10-19',
auf_einen_blick: 'Ein Decoder kommt der Absicht zuvor.', grundlagenbegriff: {begriff: 'Decoder', erklaerung: 'Ein Modell …'},
news_markdown: '- **Paradromics:** Kooperation. [Quelle](https://example.org/n)',
stats: {records: 3, per_source: {PubMed: 2, arXiv: 1}, errors: [], cost_usd: 0.38},
items: [
{id: 'pmid:1', doi: '', source: 'PubMed', title: 'Intention decoded', journal: 'Nature', first_author: 'Verbaarschot',
n_authors: 5, date: '2026', url: 'https://example.org/1', gruppe: 'lesenswert', worum_es_geht: 'Ein Spiel.',
einordnung: 'Preprint.', politischer_haken: 'Libet.', kurz: 'Kurz.'},
{id: 'arxiv:2', source: 'arXiv', title: 'EEG artefacts', first_author: 'Adeika', n_authors: 2, url: 'https://example.org/2',
gruppe: 'technik', worum_es_geht: '', einordnung: '', politischer_haken: '', kurz: 'Muskeln statt Gedanken.'},
{id: 'pmid:3', source: 'PubMed', title: 'Off topic', gruppe: 'rauschen', kurz: ''}]}};
await page.route(/\/projects\/\d+\/radar\/$/, route => route.fulfill({json: {radar: {enabled: true, title: 'Neuropolitik-Digest',
groups: ['lesenswert', 'propriozeption', 'gesellschaft', 'technik', 'rauschen'], group_labels: {lesenswert: 'Lesenswert', technik: 'Technik im Überblick', rauschen: 'Rauschen'},
weekday: 6, hour: 18, days: 7, sources: ['PubMed', 'arXiv'], running: false, next_run: '2026-10-26T18:00:00+01:00',
last_error: '', key: true}, digests: [digest]}}));
await page.goto('/#projekte');
const tab = page.locator('#tab-projekte');
await tab.locator('[data-act="new"]').click();
await tab.locator('input[name="name"]').fill('Radarprojekt');
await tab.locator('button[type="submit"]').click();
await expect(tab.locator('.prj-tab.on')).toContainText('Radar');
await expect(tab.locator('.prj-radar-h')).toContainText('KW 43');
await expect(tab.locator('.prj-radar-group[open] > summary')).toHaveText([/Lesenswert/, /Technik/, /Branche/]);
await expect(tab.locator('.prj-paper').first()).toContainText('Politischer Haken: Libet.');
await tab.locator('[data-radar-decide="arxiv:2"][data-state="later"]').click();
await tab.locator('[data-radar-decide="pmid:1"][data-state="read"]').click();
await expect(tab.locator('[data-radar-later]')).toHaveText('Später 1');
// Kept, encrypted: after a reload still there.
await page.waitForTimeout(800);
await page.reload();
await expect(tab.locator('[data-radar-later]')).toHaveText('Später 1');
await expect(tab.locator('.prj-paper').first()).toContainText('gelesen');
const raw = await page.evaluate(() => fetch(location.pathname.replace(/\/$/, '') + '/projects/').then(r => r.json()));
expect(JSON.stringify(raw)).not.toContain('arxiv:2');
await tab.locator('[data-radar-source="arxiv:2"]').click();
await expect(tab.locator('.prj-paper').nth(1)).toContainText('✓ in Quellen');
await tab.locator('[data-view="sources"]').click();
await expect(tab.locator('.prj-row .prj-title')).toHaveText(['EEG artefacts']);
});
test('Abos & Quellen: one page for every subscription, reached from the feeds too', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 800 });
await page.goto('/#news');
await page.locator('#tab-feeds [data-act="manage"]').first().click();
await expect(page).toHaveURL(/#sources$/);
const tab = page.locator('#tab-sources');
await expect(tab.locator('.src-h1')).toHaveText('Abos & Quellen');
await expect(tab.locator('.src-add-q')).toBeVisible();
await expect(tab.locator('.src-chip.on')).toContainText('Nachrichten'); // came from the news
await tab.locator('[data-filter="all"]').click();
await expect(tab.locator('.src-chip.on')).toContainText('Alle');
});
test('the library: a header with search, sort, layout and +, the add menu', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 800 });
await page.goto('/#books');
const list = page.locator('#book-list');
await expect(list.locator('.lib-h')).toHaveText('Bücher');
await expect(list.locator('.lib-q')).toBeVisible();
await expect(list.locator('.lib-sort')).toBeVisible();
await expect(page.locator('#book-drop-zone')).toBeHidden(); // adding is behind +
await list.locator('[data-lib="add"]').click();
await expect(page.locator('.dfo-sheet')).toContainText('E-Book-Datei');
await page.locator('.dfo-sheet [data-x]').click();
await expect(page.locator('.dfo-sheet')).toHaveCount(0);
await page.goto('/#magazines');
await expect(list.locator('.lib-h')).toHaveText('Zeitschriften');
});
test('several books at once; one already there is left out', async ({ page }) => {
const files = ['Eins', 'Zwei'].map(n => path.join(__dirname, '.auth', `many-${n}.epub`));
await page.goto('/#books');
await page.waitForFunction(() => typeof _importMany === 'function' && Array.isArray(_lastBookListData));
const status = page.locator('#book-upload-status');
await page.locator('#book-file-input').setInputFiles(files);
await expect(status).toContainText('2 Bücher hinzugefügt', { timeout: 30000 });
await page.waitForFunction(() => _lastBookListData.filter(b => /^Mehrfach/.test(b.title)).length === 2);
await page.locator('#book-file-input').setInputFiles(files);
await expect(status).toContainText('2 schon da', { timeout: 30000 });
// Out again, so the other tests see the library as before.
const books = await page.evaluate(() => _lastBookListData.filter(b => /^Mehrfach/.test(b.title)).map(b => b.id));
for (const id of books) await page.request.post(`/books/${id}/delete/`);
});
test('editions: two books joined show once with their formats, set apart show twice', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 800 });
const files = ['Eins', 'Zwei'].map(n => path.join(__dirname, '.auth', `many-${n}.epub`));
await page.goto('/#books');
await page.waitForFunction(() => typeof _importMany === 'function' && Array.isArray(_lastBookListData));
await page.locator('#book-file-input').setInputFiles(files);
await page.waitForFunction(() => _lastBookListData.filter(b => /^Mehrfach/.test(b.title) && b.id).length === 2, null, { timeout: 30000 });
const ids = await page.evaluate(() => _lastBookListData.filter(b => /^Mehrfach/.test(b.title)).map(b => b.id));
const list = page.locator('#book-list');
// New books: as cards under "Ungelesen" (not in the "Zuletzt gelesen" area on top).
const unread = () => page.evaluate(() => { _lib.status = 'unread'; renderBookList(_lastBookListData); });
await unread();
const cards = () => list.locator('.lib-grid [data-lib-book]').filter({ hasText: 'Mehrfach' });
await expect(cards()).toHaveCount(2);
// Choose both, join them.
await page.evaluate(ids => { _lib.selecting = new Set(ids); renderBookList(_lastBookListData); }, ids);
await list.locator('[data-lib="sel-editions"]').click();
await page.waitForFunction(ids => {
const w = ids.map(id => _lastBookListData.find(b => b.id === id).work);
return w[0] && w[0] === w[1];
}, ids);
await expect(cards()).toHaveCount(1);
await expect(cards().locator('.lib-ed')).toHaveText('EPUB');
// The work survives a reload (it is in the encrypted meta).
await page.reload();
await page.waitForFunction(() => Array.isArray(_lastBookListData) && _lastBookListData.length);
await unread();
await expect(cards()).toHaveCount(1);
// Ausgaben: both listed; one set apart, and the other is on its own again.
await cards().locator('.lib-more').click({ force: true });
await page.locator('.dfo-sheet .lib-sheet-item', { hasText: 'Ausgaben (2)' }).click();
await expect(page.locator('.dfo-sheet .lib-ed-row')).toHaveCount(2);
await page.locator('.dfo-sheet [data-unlink]').first().click();
await page.waitForFunction(ids => ids.every(id => !_lastBookListData.find(b => b.id === id).work), ids);
await expect(cards()).toHaveCount(2);
await page.evaluate(() => { _lib.status = 'all'; });
for (const id of ids) await page.request.post(`/books/${id}/delete/`);
});
test('a meta change from an old list goes onto the current meta (409, then again)', async ({ page }) => {
const files = [path.join(__dirname, '.auth', 'many-Eins.epub')];
await page.goto('/#books');
await page.waitForFunction(() => typeof _importMany === 'function' && Array.isArray(_lastBookListData));
await page.locator('#book-file-input').setInputFiles(files);
await page.waitForFunction(() => _lastBookListData.some(b => b.title === 'Mehrfach Eins' && b.id && b.meta_iv), null, { timeout: 30000 });
const id = await page.evaluate(() => _lastBookListData.find(b => b.title === 'Mehrfach Eins').id);
// Elsewhere the title changes; this page still has the old list.
await page.evaluate(id => _updateBookMeta(id, {title: 'Anderswo umbenannt'}), id);
await page.evaluate(id => {
const b = _lastBookListData.find(x => x.id === id);
b.title = 'Mehrfach Eins'; b.meta_iv = '000000000000000000000000'; // the old list
bookMetaCache[id] = {...bookMetaCache[id], title: 'Mehrfach Eins'};
}, id);
await page.evaluate(id => _updateBookMeta(id, {folders: ['Probe']}), id);
const meta = await page.evaluate(async id => {
const row = (await (await fetch('/books/', {cache: 'no-store'})).json()).find(b => b.id === id);
return JSON.parse(new TextDecoder().decode(await decryptBytes(await getOrCreateEncKey(), row.meta_iv, row.meta_ct)));
}, id);
expect(meta.title).toBe('Anderswo umbenannt');
expect(meta.folders).toEqual(['Probe']);
await page.request.post(`/books/${id}/delete/`);
});

71
e2e/pdf.spec.js Normal file
View file

@ -0,0 +1,71 @@
// A big PDF (a scanned volume of 400 pages, e2e/make_pdf.py): it opens at once, only the
// pages near the view are drawn, the end can be reached, and opening can be cancelled.
const path = require('path');
const { test, expect } = require('@playwright/test');
test.use({ storageState: path.join(__dirname, '.auth', 'user.json') });
test.describe.configure({ mode: 'serial' });
const PDF = path.join(__dirname, '.auth', 'big.pdf');
let bookId = null;
let pageErrors = [];
test.beforeEach(({ page }) => {
pageErrors = [];
page.on('pageerror', e => pageErrors.push(String(e)));
});
test.afterEach(() => {
expect(pageErrors, 'errors in the page').toEqual([]);
});
async function ready(page) {
await page.goto('/#books');
await page.waitForLoadState('networkidle');
await page.waitForFunction(() => typeof openBook === 'function' && Array.isArray(_lastBookListData));
}
const drawn = page => page.evaluate(() =>
[...document.querySelectorAll('#reader-content canvas.pdf-page')].filter(c => c.width > 0).length);
test('the big PDF goes in', async ({ page }) => {
await ready(page);
await page.locator('#book-file-input').setInputFiles(PDF);
await page.waitForFunction(() => _lastBookListData.some(b => b.title === 'Dicker Band'), null, { timeout: 30000 });
bookId = await page.evaluate(() => _lastBookListData.find(b => b.title === 'Dicker Band').id);
});
test('it opens with every page placed, only those near the view drawn', async ({ page }) => {
await ready(page);
await page.evaluate(id => openBook(id), bookId);
await page.waitForFunction(() => _readPos && _readPos.ready, null, { timeout: 20000 });
await expect(page.locator('#reader-content .pdf-page-wrapper')).toHaveCount(400);
await expect(page.locator('.pdf-loading-overlay')).toHaveCount(0);
await expect.poll(() => drawn(page)).toBeGreaterThan(0);
expect(await drawn(page)).toBeLessThan(30);
// To the end: the last page gets drawn, the first ones are let go.
await page.evaluate(() => { const c = document.getElementById('reader-content'); c.scrollTop = c.scrollHeight; });
await expect.poll(() => page.evaluate(() => document.querySelector('#pdf-page-400 canvas').width > 0)).toBe(true);
await expect.poll(() => page.evaluate(() => document.querySelector('#pdf-page-1 canvas').width)).toBe(0);
expect(await drawn(page)).toBeLessThan(30);
await page.evaluate(() => closeReader());
});
test('opening can be cancelled, and another book opens after it', async ({ page }) => {
await ready(page);
// Slow the file down so the opening is still under way.
await page.route(`**/books/${bookId}/data/`, async route => {
await new Promise(r => setTimeout(r, 1500));
await route.continue().catch(() => {});
});
const p = page.evaluate(id => openBook(id).then(() => 'done', e => 'error ' + e), bookId);
await expect(page.locator('.reader-opening-cancel')).toBeVisible({ timeout: 5000 });
await page.locator('.reader-opening-cancel').click();
await expect(page.locator('#reader-overlay')).toBeHidden();
expect(await p).toBe('done');
await page.unroute(`**/books/${bookId}/data/`);
// Opening it again works, all the way.
await page.evaluate(id => openBook(id), bookId);
await page.waitForFunction(() => _readPos && _readPos.ready, null, { timeout: 20000 });
await page.evaluate(() => closeReader());
await page.request.post(`/books/${bookId}/delete/`);
});

169
e2e/reader.spec.js Normal file
View file

@ -0,0 +1,169 @@
// The reader end to end: a book goes in (encrypted in the browser), opens, is
// read, and its place, a highlight and the theme survive a reload. The safety
// net for refactoring app.js (docs/refactoring.md). One book for the whole file:
// the tests build on each other and run in order.
const path = require('path');
const { test, expect } = require('@playwright/test');
test.use({ storageState: path.join(__dirname, '.auth', 'user.json') });
test.describe.configure({ mode: 'serial' });
const BOOK = path.join(__dirname, '.auth', 'book.epub');
let bookId = null;
// Errors in the page fail the test that caused them.
let pageErrors = [];
test.beforeEach(({ page }) => {
pageErrors = [];
page.on('pageerror', e => pageErrors.push(String(e)));
});
test.afterEach(() => {
expect(pageErrors, 'errors in the page').toEqual([]);
});
async function openTestBook(page) {
await page.goto('/#books');
// A new service worker taking over reloads the page once: let that settle first.
await page.waitForLoadState('networkidle');
await page.waitForFunction(() => typeof openBook === 'function' && typeof _lastBookListData !== 'undefined' && Array.isArray(_lastBookListData));
await page.evaluate(id => openBook(id), bookId);
await page.waitForFunction(() => document.querySelectorAll('#reader-content [data-epub-src]').length === 12
&& typeof _readPos !== 'undefined' && _readPos && _readPos.ready);
}
async function serverBook(page) {
const books = await (await page.request.get('/books/')).json();
return books.find(b => b.id === bookId);
}
test('a book goes in and shows in the list', async ({ page }) => {
await page.goto('/#books');
await page.setInputFiles('#book-file-input', BOOK);
const found = await page.waitForFunction(() => {
const list = typeof _lastBookListData !== 'undefined' ? (_lastBookListData || []) : [];
const b = list.find(x => x.title === 'E2E Testbuch');
return b ? b.id : null;
}, null, { timeout: 20000 });
bookId = await found.jsonValue();
expect(bookId).toBeTruthy();
});
test('opening without reading sends no position', async ({ page }) => {
const posts = [];
page.on('request', r => { if (r.method() === 'POST' && /\/progress\/$/.test(r.url())) posts.push(r.url()); });
await openTestBook(page);
await page.waitForTimeout(3000);
await page.evaluate(() => saveReaderProgress());
await page.evaluate(() => closeReader());
await page.waitForTimeout(500);
expect(posts).toEqual([]);
});
test('reading moves the position, and it comes back after a reload', async ({ page }) => {
await openTestBook(page);
const content = page.locator('#reader-content');
await content.hover();
for (let i = 0; i < 12; i++) await page.mouse.wheel(0, 600);
const saved = page.waitForRequest(r => r.method() === 'POST' && /\/progress\/$/.test(r.url()));
const body = JSON.parse((await saved).postData());
expect(body.device).toBe('Browser');
expect(body.read_at).toBeGreaterThan(1e12);
const anchor = await page.evaluate(() => getPositionAnchor(document.getElementById('reader-content')));
await page.waitForTimeout(500);
const b = await serverBook(page);
expect(b.position_anchor).not.toBe('');
expect(b.device).toBe('Browser');
await page.reload();
await openTestBook(page);
const restored = await page.evaluate(() => getPositionAnchor(document.getElementById('reader-content')));
const block = a => Number(a.split(':')[0]);
expect(Math.abs(block(restored) - block(anchor))).toBeLessThanOrEqual(1);
});
test('a footnote shows its text', async ({ page }) => {
await openTestBook(page);
const ref = page.locator('#reader-content [data-footnote-ref]').first();
await ref.scrollIntoViewIfNeeded();
await ref.click();
await expect(page.locator('.footnote-popover')).toContainText('Der Text der Fußnote');
});
test('a highlight is kept, and shows in the margin', async ({ page }) => {
await openTestBook(page);
// Marker mode on (the ✎ button), select the start of a paragraph on screen
// the way a mouse would, then let go.
await page.evaluate(() => {
if (!markerModeActive) toggleMarkerMode();
const p = document.querySelector('#reader-content #c1p2') || document.querySelector('#reader-content p');
p.scrollIntoView({ block: 'center' });
const text = p.firstChild;
const range = document.createRange();
range.setStart(text, 0);
range.setEnd(text, 20);
const sel = window.getSelection();
sel.removeAllRanges();
sel.addRange(range);
const r = range.getBoundingClientRect();
p.dispatchEvent(new MouseEvent('mouseup', { bubbles: true, clientX: r.right, clientY: r.bottom }));
});
const saved = page.waitForRequest(r => r.method() === 'POST' && /\/highlights\/$/.test(r.url()), { timeout: 15000 });
await page.locator('.hl-color-btn[data-hl-color="yellow"]').click();
await page.evaluate(() => saveHighlights && saveHighlights());
await saved;
await page.reload();
await openTestBook(page);
await page.waitForFunction(() => document.querySelectorAll('#reader-content mark.epub-highlight, #reader-content .epub-highlight').length > 0);
// The margin remembers being open; open it if it isn't.
await page.evaluate(() => { if (!readerAnnotationsMarginOpen) toggleAnnotationsMargin(); });
await expect(page.locator('#reader-margin-markers .margin-note')).toHaveCount(1);
});
test('an open margin keeps the text column in the middle when there is room', async ({ page }) => {
const column = () => page.evaluate(() => {
const c = document.getElementById('reader-content');
const p = c.querySelector('p');
const r = p.getBoundingClientRect();
const m = document.getElementById('reader-margin').getBoundingClientRect();
return {left: r.left, right: r.right, width: r.width, marginRight: m.right, window: window.innerWidth};
});
await page.setViewportSize({ width: 1600, height: 900 });
await openTestBook(page);
const closed = await column();
await page.evaluate(() => { if (!readerAnnotationsMarginOpen) toggleAnnotationsMargin(); });
await page.waitForTimeout(400); // the margin and the padding slide together
const open = await column();
expect(Math.abs(open.left - closed.left)).toBeLessThanOrEqual(2);
expect(open.width).toBeCloseTo(closed.width, 0);
// Not enough room: the column moves aside, but keeps its width and clears the margin.
await page.setViewportSize({ width: 1000, height: 900 });
await page.waitForTimeout(400);
const narrow = await column();
expect(narrow.left).toBeGreaterThanOrEqual(narrow.marginRight);
await page.evaluate(() => { toggleAnnotationsMargin(); });
});
test('the theme switch turns the page and the reader light, and stays', async ({ page }) => {
await page.goto('/');
await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark');
await page.locator('#theme-toggle').click();
await expect(page.locator('html')).toHaveAttribute('data-theme', 'light');
await page.reload();
await expect(page.locator('html')).toHaveAttribute('data-theme', 'light');
await openTestBook(page);
await expect(page.locator('#reader-overlay')).toHaveClass(/reader-theme-bright/);
await page.evaluate(() => { closeReader(); __dioraSetTheme('dark'); });
await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark');
});
test('every tab opens without errors', async ({ page }) => {
await page.goto('/');
await page.waitForLoadState('networkidle');
await page.waitForFunction(() => typeof showTab === 'function' && typeof loadBookList === 'function');
for (const tab of ['home', 'radio', 'podcasts', 'books', 'feeds', 'focus', 'ambient']) {
await page.evaluate(t => showTab(t), tab);
await expect(page.locator(`#tab-${tab}`)).toBeVisible();
}
});

178
e2e/sponsor.spec.js Normal file
View file

@ -0,0 +1,178 @@
// Skipping the ad and the outro of a YouTube episode (docs/youtube.md, podcasts/sponsor.py).
//
// The sound of a YouTube episode plays only in an app, so the desktop is imitated here:
// window.dioraDesktop.youtubeAudio answers with a stream of our own (a silent WAV, served
// with byte ranges so the player can jump in it). The browser host (podcasts-web.js) is the
// desktop's podcast host, so this is the code that runs there.
const { test, expect } = require('@playwright/test');
const { execFileSync } = require('child_process');
const path = require('path');
const { PYTHON, SERVER_ENV, TEST_USER } = require('./env');
const ROOT = path.join(__dirname, '..');
const VID = 'aqz-KE-bpKQ';
const WATCH = `https://www.youtube.com/watch?v=${VID}`;
const SPANS = [[1, 3], [6, 7.5]]; // ad, outro
const SECONDS = 9;
const STREAM = '/e2e-silence.wav';
// A silent 8 kHz mono WAV: long enough to reach both parts and past them.
function wav(seconds, rate = 8000) {
const n = seconds * rate;
const buf = Buffer.alloc(44 + n * 2);
buf.write('RIFF', 0);
buf.writeUInt32LE(36 + n * 2, 4);
buf.write('WAVE', 8);
buf.write('fmt ', 12);
buf.writeUInt32LE(16, 16);
buf.writeUInt16LE(1, 20); // PCM
buf.writeUInt16LE(1, 22); // mono
buf.writeUInt32LE(rate, 24);
buf.writeUInt32LE(rate * 2, 28); // byte rate
buf.writeUInt16LE(2, 32); // block align
buf.writeUInt16LE(16, 34); // bits
buf.write('data', 36);
buf.writeUInt32LE(n * 2, 40);
return buf;
}
const BODY = wav(SECONDS);
function seed() {
const py = `
from django.contrib.auth import get_user_model
from django.utils import timezone
from podcasts.models import PodcastEpisode, PodcastFeed, PodcastQueue
import json
u = get_user_model().objects.get(username=${JSON.stringify(TEST_USER.username)})
PodcastQueue.objects.filter(user=u).delete()
PodcastEpisode.objects.filter(feed__user=u).delete()
PodcastFeed.objects.filter(user=u).delete()
f = PodcastFeed.objects.create(user=u, rss_url='https://www.youtube.com/feeds/videos.xml?channel_id=UCe2etest',
title='Kanal')
ep = PodcastEpisode.objects.create(feed=f, guid='yt:video:${VID}', title='Eine Folge',
audio_url=${JSON.stringify(WATCH)}, pub_date=timezone.now(),
duration_seconds=${SECONDS},
sponsor={'v': 1, 'spans': ${JSON.stringify(SPANS)},
'at': timezone.now().isoformat()})
PodcastQueue.objects.create(user=u, episode=ep, position=0)
print('episode', ep.id)
`;
execFileSync(PYTHON, ['manage.py', 'shell', '-c', py], { cwd: ROOT, env: SERVER_ENV, stdio: 'inherit' });
}
test.beforeAll(seed);
test.use({ storageState: 'e2e/.auth/user.json' });
// The stream, with byte ranges: without them Chromium may refuse to jump in a file.
async function serveStream(page) {
await page.route(`**${STREAM}`, route => {
const range = route.request().headers()['range'];
const head = { 'Content-Type': 'audio/wav', 'Accept-Ranges': 'bytes' };
if (!range) return route.fulfill({ status: 200, headers: head, body: BODY });
const m = /bytes=(\d*)-(\d*)/.exec(range);
const from = m && m[1] ? Number(m[1]) : 0;
const to = m && m[2] ? Number(m[2]) : BODY.length - 1;
return route.fulfill({
status: 206,
headers: { ...head, 'Content-Range': `bytes ${from}-${to}/${BODY.length}`, 'Content-Length': String(to - from + 1) },
body: BODY.subarray(from, to + 1),
});
});
}
// The desktop the host talks to, and a look at the player's own position over time.
async function imitateDesktop(page) {
await page.addInitScript(url => {
window.dioraDesktop = {
youtubeAudio: () => Promise.resolve({ url }),
deviceName: 'diora Desktop',
};
window.__times = [];
const Orig = window.Audio;
window.Audio = function () {
const el = new Orig(...arguments);
window.__el = el;
el.addEventListener('timeupdate', () => window.__times.push(el.currentTime));
el.addEventListener('playing', () => window.__times.push(el.currentTime));
return el;
};
window.Audio.prototype = Orig.prototype;
}, STREAM);
}
// The episode as the host itself holds it (the queue from /podcasts/state/).
async function episode(page) {
return page.evaluate(() => {
const s = JSON.parse(window.DioraPodcastHost.podcastState());
return (s.queue || []).find(q => /watch\?v=/.test(q.episode__audio_url || '')) || null;
});
}
function inPart(t) {
return SPANS.find(([from, to]) => t > from + 0.05 && t < to - 0.05) || null;
}
test('the ad and the outro are jumped over', async ({ page }) => {
const asked = [];
page.on('request', r => { if (/\/podcasts\/episodes\/\d+\/sponsor\//.test(r.url())) asked.push(r.url()); });
await serveStream(page);
await imitateDesktop(page);
await page.goto('/');
await page.waitForFunction(() => window.DioraPodcastHost, null, { timeout: 15000 });
await page.waitForFunction(() => {
const s = JSON.parse(window.DioraPodcastHost.podcastState());
return (s.queue || []).some(q => /watch\?v=/.test(q.episode__audio_url || ''));
}, null, { timeout: 15000 });
const ep = await episode(page);
expect(ep, 'the seeded episode is in the queue').not.toBeNull();
await page.evaluate(json => window.DioraPodcastHost.podcastPlayEpisode(json, 0), JSON.stringify(ep));
// Where the sound is over time, from here (a `timeupdate` comes only every 250 ms).
const times = await sample(page, 4.5);
expect(asked.length, 'the server was asked for the parts').toBeGreaterThan(0);
const inside = times.filter(inPart);
expect(inside, `no part was played: ${inside.join(', ')} of ${times.length} positions`).toEqual([]);
expect(Math.max(...times), 'the sound kept going past the ad').toBeGreaterThan(3.2);
});
// The player's position, read every 100 ms until it passed `until` (in seconds).
async function sample(page, until, limit = 250) {
const times = [];
for (let i = 0; i < limit; i++) {
const t = await page.evaluate(() => (window.__el && window.__el.readyState > 0
? window.__el.currentTime : null));
if (t !== null) times.push(t);
if (t !== null && t > until) break;
await page.waitForTimeout(100);
}
return times;
}
test('with the setting off nothing is asked and nothing is jumped over', async ({ page }) => {
seed(); // the first test left the episode played (and out of the queue)
let asked = 0;
page.on('request', r => { if (/\/podcasts\/episodes\/\d+\/sponsor\//.test(r.url())) asked++; });
await serveStream(page);
await imitateDesktop(page);
await page.goto('/');
// This device's setting, before the host reads it (localStorage, PlaybackPrefs.kt).
await page.evaluate(() => localStorage.setItem('dpw_prefs_' + window.USER_ID, JSON.stringify({skip_sponsor: false})));
await page.reload();
await page.waitForFunction(() => window.DioraPodcastHost, null, { timeout: 15000 });
await page.waitForFunction(() => {
const s = JSON.parse(window.DioraPodcastHost.podcastState());
return (s.queue || []).some(q => /watch\?v=/.test(q.episode__audio_url || ''));
}, null, { timeout: 15000 });
expect(await page.evaluate(() => JSON.parse(window.DioraPodcastHost.podcastPrefs()).skip_sponsor)).toBe(false);
const ep = await episode(page);
await page.evaluate(json => window.DioraPodcastHost.podcastPlayEpisode(json, 0), JSON.stringify(ep));
const times = await sample(page, 4.5);
expect(asked, 'nothing was asked for').toBe(0);
const inside = times.filter(inPart);
expect(inside.length, `the ad was played (${inside.join(', ')})`).toBeGreaterThan(0);
});

0
feeds/__init__.py Normal file
View file

6
feeds/apps.py Normal file
View file

@ -0,0 +1,6 @@
from django.apps import AppConfig
class FeedsConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'feeds'

149
feeds/cluster.py Normal file
View file

@ -0,0 +1,149 @@
"""Bundling news items about the same event into stories.
Each item is described by the words of its title (counted twice) and summary,
without stop words and with a crude German/English suffix cut. A new item joins
the most similar story of the last WINDOW hours (cosine of tf-idf vectors,
document frequencies over those stories) if that similarity reaches THRESHOLD
and they share at least MIN_SHARED words; otherwise it starts a story of its own.
A story many sources report on is a top story.
"""
import math
import re
from collections import Counter
from datetime import timedelta
from .models import Item, Story
WINDOW = timedelta(hours=36)
THRESHOLD = 0.25
MIN_SHARED = 2
STOP = set('''
aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher
da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese
diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für
gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer
in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen
nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind
so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter
weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier
heute gestern morgen laut sagt sagte erst mal neu alt video live ticker liveblog news meldung bericht
the and for with from that this have has are was were will would been into about after over their they
his her its our you your not but all can new says said more than when what who how why off out
tot tote toten getötet sterben stirbt gestorben menschen mensch leute millionen milliarden prozent euro
deutschland deutsche deutschen land länder stadt welt woche wochen monat monate tag tage tagen uhr
frau frauen mann männer kinder erste ersten zweite neuer neues großen große groß kleine wegen gibt geben
'''.split())
_WORD = re.compile(r'[a-zäöüß][a-zäöüß0-9\-]+')
_SUFFIXES = ('ungen', 'ung', 'en', 'er', 'es', 'e', 's', 'n')
def _stem(w):
if len(w) > 5:
for s in _SUFFIXES:
if w.endswith(s) and len(w) - len(s) >= 4:
return w[:-len(s)]
return w
def words(text):
out = []
for w in _WORD.findall((text or '').lower()):
w = w.strip('-')
if len(w) < 3 or w in STOP:
continue
out.append(_stem(w))
return out
def item_terms(title, summary):
return Counter(words(title) * 2 + words(summary[:300]))
def _cosine(a, b, idf):
dot = sum(a[w] * b[w] * idf.get(w, 1.0) ** 2 for w in a.keys() & b.keys())
if not dot:
return 0.0
na = math.sqrt(sum((n * idf.get(w, 1.0)) ** 2 for w, n in a.items()))
nb = math.sqrt(sum((n * idf.get(w, 1.0)) ** 2 for w, n in b.items()))
return dot / (na * nb)
class Clusterer:
"""Assigns one user's new news items to stories (load once, add items, done).
An item is compared with every item already in a story: the story's score
is the mean of the best match and the average match, so one stray shared
word can't pull an item in, and a story doesn't grow into a magnet the way
a pooled word list would. Items of the same source join only when very
close (a source rarely reports one event twice, but often two similar ones)."""
SAME_SOURCE = 0.5
def __init__(self, user, now):
self.user = user
self.stories = list(Story.objects.filter(user=user, last_at__gte=now - WINDOW))
self.members = {s.pk: [] for s in self.stories} # story id -> [(source id, terms)]
for i in Item.objects.filter(story__in=self.stories, gone=False).only('story_id', 'source_id', 'title', 'summary'):
self.members[i.story_id].append((i.source_id, item_terms(i.title, i.summary)))
self._df = Counter()
for ms in self.members.values():
for _, t in ms:
self._df.update(t.keys())
self._n = sum(len(ms) for ms in self.members.values())
def _idf(self):
n = self._n + 1
return {w: math.log((n + 1) / (d + 1)) + 1 for w, d in self._df.items()}
def score(self, t, source_id, members, idf):
sims = []
for sid, mt in members:
if len(t.keys() & mt.keys()) < MIN_SHARED:
sims.append(0.0)
continue
sim = _cosine(t, mt, idf)
if sid == source_id and sim < self.SAME_SOURCE:
sim = 0.0
sims.append(sim)
if not sims or max(sims) == 0:
return 0.0
return (max(sims) + sum(sims) / len(sims)) / 2
def add(self, item):
t = item_terms(item.title, item.summary)
best, best_sim = None, 0.0
if t:
idf = self._idf()
for s in self.stories:
if abs((s.last_at - item.published).total_seconds()) > WINDOW.total_seconds():
continue
sim = self.score(t, item.source_id, self.members[s.pk], idf)
if sim > best_sim:
best, best_sim = s, sim
if best is not None and best_sim >= THRESHOLD:
self._join(best, item)
else:
best = Story.objects.create(user=self.user, lead=item, first_at=item.published,
last_at=item.published, size=1, sources=1)
self.stories.append(best)
self.members[best.pk] = []
item.story = best
item.save(update_fields=['story'])
self.members[best.pk].append((item.source_id, t))
self._df.update(t.keys())
self._n += 1
return best
def _join(self, s, item):
item.story = s
item.save(update_fields=['story'])
s.size = Item.objects.filter(story=s).count()
s.sources = Item.objects.filter(story=s).values('source').distinct().count()
s.first_at = min(s.first_at, item.published)
s.last_at = max(s.last_at, item.published)
if s.lead is None or (not s.lead.image_url and item.image_url):
s.lead = item
s.save(update_fields=['size', 'sources', 'first_at', 'last_at', 'lead'])

253
feeds/fetch.py Normal file
View file

@ -0,0 +1,253 @@
"""Reading feeds: entries into items, the whole article from its page, finding a
site's feed. Downloading the feed itself is podcasts.views._fetch_feed (ETag,
body hash, redirects), shared with the podcast refresh."""
import calendar
import hashlib
import html
import logging
import re
from datetime import datetime, timezone as dt_timezone
from html.parser import HTMLParser
from urllib.parse import urljoin, urlparse
import requests
from django.utils import timezone
from podcasts.shownotes import _Sanitizer, html_to_text
log = logging.getLogger(__name__)
USER_AGENT = 'Mozilla/5.0 (compatible; diora feed reader; +https://diora.creamfresh.xyz)'
PAGE_TIMEOUT = 15
# Shorter text than this in the feed counts as a teaser: the article is fetched ("auto").
TEASER_CHARS = 700
SUMMARY_CHARS = 400
class _AbsoluteSanitizer(_Sanitizer):
"""The podcast sanitizer, with links and images made absolute first
(article HTML is full of relative ones, which the allowlist would drop)."""
def __init__(self, base):
super().__init__()
self.base = base
def _start(self, tag, attrs, closed):
if tag == 'img':
# Lazy-loaded images keep the real address in data-src, src is a placeholder.
a = dict(attrs)
real = a.get('data-src') or a.get('data-original')
if real:
attrs = [(n, v) for n, v in attrs if n not in ('src', 'data-src', 'data-original')] + [('src', real)]
fixed = []
for name, value in attrs:
if name in ('href', 'src') and value and not value.strip().startswith(('data:', 'javascript:', '#')):
value = urljoin(self.base, value.strip())
fixed.append((name, value))
super()._start(tag, fixed, closed)
def clean_html(text, base=''):
if not text:
return ''
s = _AbsoluteSanitizer(base)
s.feed(text)
return s.result().strip()
def plain(text):
return re.sub(r'\s+', ' ', html_to_text(text or '')).strip()
def _short(text, n=SUMMARY_CHARS):
if len(text) <= n:
return text
cut = text[:n].rsplit(' ', 1)[0]
return cut + ' …'
def _when(entry):
"""The entry's time (published, else updated), never in the future, or None."""
now = timezone.now()
for key in ('published_parsed', 'updated_parsed', 'created_parsed'):
t = entry.get(key)
if t:
try:
when = datetime.fromtimestamp(calendar.timegm(t), dt_timezone.utc)
except (OverflowError, ValueError, TypeError):
continue
return min(when, now)
return None
class _FirstImage(HTMLParser):
def __init__(self):
super().__init__()
self.src = ''
def handle_starttag(self, tag, attrs):
if tag == 'img' and not self.src:
a = dict(attrs)
w, h = a.get('width', ''), a.get('height', '')
if w in ('1', '0') or h in ('1', '0'):
return # tracking pixel
self.src = a.get('src') or ''
def _image(entry, content_html, base):
for m in entry.get('media_thumbnail') or []:
if m.get('url'):
return m['url']
for m in entry.get('media_content') or []:
if m.get('url') and (m.get('medium') == 'image' or (m.get('type') or '').startswith('image/')):
return m['url']
for enc in entry.get('enclosures') or []:
if (enc.get('type') or '').startswith('image/') and enc.get('href'):
return enc['href']
p = _FirstImage()
try:
p.feed(content_html or '')
except Exception:
pass
return urljoin(base, p.src) if p.src else ''
def guid_hash(entry):
key = entry.get('id') or entry.get('link') or f"{entry.get('title', '')}|{entry.get('published', '')}"
return hashlib.sha256(key.encode('utf-8', 'replace')).hexdigest()
def parse_entry(entry, feed_url):
"""A feedparser entry -> the fields of an Item (without source/user)."""
link = entry.get('link') or ''
base = link or feed_url
contents = [c.get('value') or '' for c in entry.get('content') or []]
content_html = max(contents, key=len) if contents else ''
summary_html = entry.get('summary') or ''
if len(plain(summary_html)) > len(plain(content_html)):
content_html = summary_html
content = clean_html(content_html, base)
text = plain(content)
summary = plain(summary_html) or text
title = plain(html.unescape(entry.get('title') or '')) or _short(text, 80) or 'Ohne Titel'
when = _when(entry)
return {
'guid_hash': guid_hash(entry),
'url': link[:2000],
'title': title[:500],
'author': plain(entry.get('author') or '')[:300],
'published': when or timezone.now(),
'dated': when is not None, # not an Item field: refresh._apply uses and drops it
'summary': _short(summary),
'content': content,
'words': word_count(content),
'image_url': _image(entry, content_html, base)[:2000],
}
def word_count(content_html):
return len(plain(content_html).split())
def is_teaser(content_html):
return len(plain(content_html)) < TEASER_CHARS
def _drop_title(html_text):
"""trafilatura repeats the headline as the first <h1>; the reader shows it already."""
return re.sub(r'^\s*(<img[^>]*>\s*)?<h1>.*?</h1>', lambda m: m.group(1) or '', html_text, count=1, flags=re.S)
def fetch_article(url, http=None):
"""The article on `url` as (clean HTML, image URL, paywalled), or None.
trafilatura finds the main text and leaves out navigation, ads, comments.
`http`: a logged-in session for a paywalled site (logins.py); paywalled
says the page showed only the free part anyway."""
import trafilatura # heavy import, only where articles are fetched
from .logins import adapter_for
try:
r = (http or requests).get(url, timeout=PAGE_TIMEOUT, headers={'User-Agent': USER_AGENT})
except requests.RequestException as e:
log.info('article %s: %s', url, type(e).__name__)
return None
if r.status_code >= 400 or 'html' not in r.headers.get('Content-Type', 'text/html'):
return None
page = r.text
adapter = adapter_for(r.url)
paywalled = bool(adapter and adapter.paywalled(page))
try:
body = trafilatura.extract(page, url=r.url, output_format='html', include_images=True,
include_links=True, include_tables=True, favor_precision=True)
except Exception:
log.exception('article %s: extraction failed', url)
return None
if not body:
return None
body = re.sub(r'^\s*<html>\s*<body>|</body>\s*</html>\s*$', '', body.strip())
content = clean_html(_drop_title(body), r.url)
if len(plain(content)) < 200 and not paywalled:
return None
image = ''
try:
meta = trafilatura.extract_metadata(page, default_url=r.url)
image = (meta.image if meta else '') or ''
except Exception:
pass
return content, image, paywalled
# ---- finding a site's feed
FEED_TYPES = ('application/rss+xml', 'application/atom+xml', 'application/feed+json', 'application/xml', 'text/xml')
class _FeedLinks(HTMLParser):
def __init__(self):
super().__init__()
self.feeds = []
self.icon = ''
def handle_starttag(self, tag, attrs):
if tag != 'link':
return
a = {k.lower(): (v or '') for k, v in attrs}
rel = a.get('rel', '').lower().split()
if 'alternate' in rel and a.get('type', '').lower() in FEED_TYPES and a.get('href'):
self.feeds.append((a['href'], a.get('title', '')))
if ('icon' in rel or 'apple-touch-icon' in rel) and a.get('href') and not self.icon:
self.icon = a['href']
def discover(page_html, base):
"""[(feed url, title)] a page links to, and its icon URL."""
p = _FeedLinks()
try:
p.feed(page_html)
except Exception:
pass
seen, out = set(), []
for href, title in p.feeds:
u = urljoin(base, href)
if u not in seen:
seen.add(u)
out.append((u, title))
return out, (urljoin(base, p.icon) if p.icon else '')
def short_title(title):
"""A feed's name without the slogan: "tagesschau.de - die erste Adresse für …"
-> "tagesschau.de", "Aktuell - FAZ.NET" -> "FAZ.NET", "DIE ZEIT | Nachrichten, …" -> "DIE ZEIT"."""
parts = [p.strip() for p in re.split(r'\s+[-–—|:]\s+', title or '') if p.strip()]
if len(parts) < 2:
return (title or '').strip()
branded = [p for p in parts if re.search(r'\w\.[a-z]{2,}$', p, re.I)]
return (branded or parts)[0]
def site_icon(site_url):
if not site_url:
return ''
u = urlparse(site_url)
return f'{u.scheme}://{u.netloc}/favicon.ico' if u.scheme and u.netloc else ''

167
feeds/logins.py Normal file
View file

@ -0,0 +1,167 @@
"""Subscriber logins for sites behind a paywall (junge Welt …).
The account holder enters the site's username and password in the feed's
settings; the server logs in with them and fetches the articles as a
subscriber. The password is stored encrypted (Fernet, key derived from
SECRET_KEY — a copy of the database alone doesn't reveal it) and never
handed out again. Each supported site is an adapter: how to log in, how a
paywalled page looks.
"""
import base64
import hashlib
import logging
import threading
import time
from urllib.parse import urlparse
import requests
from cryptography.fernet import Fernet, InvalidToken
from django.conf import settings
from django.utils import timezone
log = logging.getLogger(__name__)
TIMEOUT = 15
SESSION_SECONDS = 6 * 3600 # a logged-in session is reused this long, then logs in again
class LoginFailed(Exception):
pass
class JungeWelt:
domain = 'jungewelt.de'
name = 'junge Welt'
def login(self, http, username, password):
r = http.post('https://www.jungewelt.de/login.php', timeout=TIMEOUT, allow_redirects=False,
data={'username': username, 'password': password, 'ref': '/', 'submit': 'Login'})
where = r.headers.get('Location', '')
if 'loginFailed' in where:
raise LoginFailed('Benutzername oder Passwort falsch')
if r.status_code not in (301, 302, 303) or not http.cookies.get('jW'):
raise LoginFailed(f'Anmeldung fehlgeschlagen ({r.status_code})')
def paywalled(self, page):
return 'mit einem Abonnement und erhalten Sie Zugang' in page
ADAPTERS = {a.domain: a for a in (JungeWelt(),)}
def adapter_for(url):
host = (urlparse(url or '').hostname or '').lower()
for domain, adapter in ADAPTERS.items():
if host == domain or host.endswith('.' + domain):
return adapter
return None
def _fernet():
key = hashlib.sha256(b'diora feeds site login\0' + settings.SECRET_KEY.encode()).digest()
return Fernet(base64.urlsafe_b64encode(key))
def encrypt(text):
return _fernet().encrypt(text.encode()).decode()
def decrypt(token):
"""The plain text, or None when it can't be read (SECRET_KEY changed)."""
try:
return _fernet().decrypt(token.encode()).decode()
except (InvalidToken, ValueError):
return None
def new_http():
from .fetch import USER_AGENT
s = requests.Session()
s.headers['User-Agent'] = USER_AGENT
return s
# Logged-in sessions in this process: (user id, domain) -> (session, logged in at).
_sessions = {}
_lock = threading.Lock()
def forget(user_id, domain):
with _lock:
_sessions.pop((user_id, domain), None)
def check_and_store(user, url, username, password):
"""Log in once; on success store the login (encrypted). Raises LoginFailed."""
from .models import SiteLogin
adapter = adapter_for(url)
if adapter is None:
raise LoginFailed('Für diese Seite gibt es keine Anmeldung.')
http = new_http()
try:
adapter.login(http, username, password)
except requests.RequestException:
raise LoginFailed(f'{adapter.name} ist nicht erreichbar.')
SiteLogin.objects.update_or_create(user=user, domain=adapter.domain, defaults={
'username': username, 'secret': encrypt(password), 'ok': True, 'last_error': '',
'checked_at': timezone.now()})
with _lock:
_sessions[(user.id, adapter.domain)] = (http, time.monotonic())
return adapter
def session_for(user_id, url, fresh=False):
"""A logged-in requests session for the article's site, or None (no login
stored, or it no longer works — then the login is marked as failing)."""
from .models import SiteLogin
adapter = adapter_for(url)
if adapter is None:
return None
key = (user_id, adapter.domain)
with _lock:
cached = _sessions.get(key)
if cached and not fresh and time.monotonic() - cached[1] < SESSION_SECONDS:
return cached[0]
login = SiteLogin.objects.filter(user_id=user_id, domain=adapter.domain).first()
if login is None:
return None
password = decrypt(login.secret)
if password is None:
login.ok, login.last_error = False, 'Bitte die Zugangsdaten neu eingeben.'
login.save(update_fields=['ok', 'last_error'])
return None
http = new_http()
try:
adapter.login(http, login.username, password)
except (LoginFailed, requests.RequestException) as e:
login.ok = False
login.last_error = str(e) if isinstance(e, LoginFailed) else f'{adapter.name} nicht erreichbar'
login.checked_at = timezone.now()
login.save(update_fields=['ok', 'last_error', 'checked_at'])
log.info('site login %s for user %s failed: %s', adapter.domain, user_id, e)
return None
if not login.ok:
login.ok, login.last_error = True, ''
login.save(update_fields=['ok', 'last_error'])
with _lock:
_sessions[key] = (http, time.monotonic())
return http
def has_login(user_id, url):
from .models import SiteLogin
adapter = adapter_for(url)
return bool(adapter) and SiteLogin.objects.filter(user_id=user_id, domain=adapter.domain, ok=True).exists()
def info(user_id, url):
"""What the feed settings show about the login for this source's site."""
from .models import SiteLogin
adapter = adapter_for(url)
if adapter is None:
return None
login = SiteLogin.objects.filter(user_id=user_id, domain=adapter.domain).first()
return {'site': adapter.name, 'domain': adapter.domain,
'username': login.username if login else '', 'ok': bool(login and login.ok),
'error': login.last_error if login else ''}

0
feeds/management/__init__.py Normal file
View file

0
feeds/management/commands/__init__.py Normal file
View file

25
feeds/management/commands/refresh_newsfeeds.py Normal file
View file

@ -0,0 +1,25 @@
from django.core.management.base import BaseCommand
from django.utils import timezone
from feeds.models import Source
from feeds.refresh import refresh
from podcasts.views import _in_backoff_q
class Command(BaseCommand):
help = 'Refresh the feed reader\'s sources (news and blogs)'
def add_arguments(self, parser):
parser.add_argument('--source-id', type=int, help='Refresh only this source (even when backing off)')
def handle(self, *args, **options):
qs = Source.objects.select_related('user').order_by('last_refreshed_at')
if options['source_id']:
qs = qs.filter(pk=options['source_id'])
else:
qs = qs.exclude(_in_backoff_q(timezone.now()))
if not qs.exists():
self.stdout.write('No sources to refresh.')
return
refreshed, failed, new = refresh(qs)
self.stdout.write(f'{refreshed} sources refreshed, {failed} failed, +{new} items')

111
feeds/migrations/0001_initial.py Normal file
View file

@ -0,0 +1,111 @@
# Generated by Django 6.1 on 2026-10-04 17:54
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='FeedSettings',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('news_seen_at', models.DateTimeField(blank=True, null=True)),
('news_keep_days', models.PositiveIntegerField(default=2)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='feed_settings', to=settings.AUTH_USER_MODEL)),
],
),
migrations.CreateModel(
name='Source',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.URLField(max_length=1000)),
('title', models.CharField(max_length=300)),
('site_url', models.URLField(blank=True, max_length=1000)),
('icon_url', models.URLField(blank=True, max_length=1000)),
('kind', models.CharField(choices=[('news', 'Nachrichten'), ('blog', 'Blog')], default='news', max_length=10)),
('category', models.CharField(blank=True, max_length=100)),
('fulltext', models.CharField(choices=[('auto', 'Automatisch'), ('always', 'Immer'), ('never', 'Nie')], default='auto', max_length=10)),
('added_at', models.DateTimeField(auto_now_add=True)),
('last_refreshed_at', models.DateTimeField(blank=True, null=True)),
('http_etag', models.CharField(blank=True, max_length=500)),
('http_last_modified', models.CharField(blank=True, max_length=100)),
('http_body_hash', models.CharField(blank=True, max_length=64)),
('last_error', models.CharField(blank=True, max_length=500)),
('last_error_at', models.DateTimeField(blank=True, null=True)),
('error_count', models.PositiveIntegerField(default=0)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_sources', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['title'],
'unique_together': {('user', 'url')},
},
),
migrations.CreateModel(
name='Item',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('kind', models.CharField(choices=[('news', 'Nachrichten'), ('blog', 'Blog')], max_length=10)),
('guid_hash', models.CharField(max_length=64)),
('url', models.URLField(blank=True, max_length=2000)),
('title', models.CharField(max_length=500)),
('author', models.CharField(blank=True, max_length=300)),
('published', models.DateTimeField(db_index=True)),
('fetched_at', models.DateTimeField(auto_now_add=True)),
('summary', models.TextField(blank=True)),
('content', models.TextField(blank=True)),
('content_full', models.BooleanField(default=False)),
('fulltext_failed', models.BooleanField(default=False)),
('image_url', models.URLField(blank=True, max_length=2000)),
('read_at', models.DateTimeField(blank=True, null=True)),
('starred', models.BooleanField(default=False)),
('gone', models.BooleanField(default=False)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_items', to=settings.AUTH_USER_MODEL)),
('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='feeds.source')),
],
options={
'ordering': ['-published'],
},
),
migrations.CreateModel(
name='Story',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('first_at', models.DateTimeField()),
('last_at', models.DateTimeField(db_index=True)),
('size', models.PositiveIntegerField(default=1)),
('sources', models.PositiveIntegerField(default=1)),
('terms', models.TextField(blank=True)),
('lead', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='feeds.item')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_stories', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['-last_at'],
},
),
migrations.AddField(
model_name='item',
name='story',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='items', to='feeds.story'),
),
migrations.AddIndex(
model_name='item',
index=models.Index(fields=['user', 'kind', 'published'], name='feeds_item_user_id_1a8bce_idx'),
),
migrations.AddIndex(
model_name='item',
index=models.Index(fields=['user', 'kind', 'read_at'], name='feeds_item_user_id_0007dd_idx'),
),
migrations.AlterUniqueTogether(
name='item',
unique_together={('source', 'guid_hash')},
),
]

37
feeds/migrations/0002_item_paywalled_sitelogin.py Normal file
View file

@ -0,0 +1,37 @@
# Generated by Django 6.1 on 2026-10-04 19:22
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feeds', '0001_initial'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddField(
model_name='item',
name='paywalled',
field=models.BooleanField(default=False),
),
migrations.CreateModel(
name='SiteLogin',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('domain', models.CharField(max_length=200)),
('username', models.CharField(max_length=300)),
('secret', models.TextField()),
('ok', models.BooleanField(default=True)),
('last_error', models.CharField(blank=True, max_length=300)),
('checked_at', models.DateTimeField(blank=True, null=True)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_site_logins', to=settings.AUTH_USER_MODEL)),
],
options={
'unique_together': {('user', 'domain')},
},
),
]

42
feeds/migrations/0003_blog_flags_words.py Normal file
View file

@ -0,0 +1,42 @@
# Generated by Django 6.1 on 2026-10-04 19:28
from django.db import migrations, models
def count_words(apps, schema_editor):
"""The reading time of the articles already there."""
from feeds.fetch import word_count
Item = apps.get_model('feeds', 'Item')
for item in Item.objects.exclude(content='').only('id', 'content').iterator():
Item.objects.filter(pk=item.pk).update(words=word_count(item.content))
class Migration(migrations.Migration):
dependencies = [
('feeds', '0002_item_paywalled_sitelogin'),
]
operations = [
migrations.AddField(
model_name='item',
name='words',
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name='source',
name='fav',
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name='source',
name='hidden',
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name='source',
name='notify',
field=models.BooleanField(default=False),
),
migrations.RunPython(count_words, migrations.RunPython.noop),
]

18
feeds/migrations/0004_item_notify.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 6.1 on 2026-10-04 19:33
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feeds', '0003_blog_flags_words'),
]
operations = [
migrations.AddField(
model_name='item',
name='notify',
field=models.BooleanField(db_index=True, default=False),
),
]

46
feeds/migrations/0005_blog_folders_progress.py Normal file
View file

@ -0,0 +1,46 @@
# Generated by Django 6.1 on 2026-10-04 19:50
from django.db import migrations, models
def category_to_folders(apps, schema_editor):
"""A blog's folder was its category: now its first (only) folder."""
Source = apps.get_model('feeds', 'Source')
for s in Source.objects.filter(kind='blog').exclude(category=''):
s.folders = [s.category[:40]]
s.category = ''
s.save(update_fields=['folders', 'category'])
def folders_to_category(apps, schema_editor):
Source = apps.get_model('feeds', 'Source')
for s in Source.objects.filter(kind='blog'):
if s.folders:
s.category = s.folders[0]
s.save(update_fields=['category'])
class Migration(migrations.Migration):
dependencies = [
('feeds', '0004_item_notify'),
]
operations = [
migrations.AddField(
model_name='item',
name='progress',
field=models.FloatField(default=0),
),
migrations.AddField(
model_name='item',
name='progress_at',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name='source',
name='folders',
field=models.JSONField(blank=True, default=list),
),
migrations.RunPython(category_to_folders, folders_to_category),
]

18
feeds/migrations/0006_folder_order.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 6.1 on 2026-10-16 18:21
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feeds', '0005_blog_folders_progress'),
]
operations = [
migrations.AddField(
model_name='feedsettings',
name='folder_order',
field=models.JSONField(blank=True, default=list),
),
]

0
feeds/migrations/__init__.py Normal file
View file

140
feeds/models.py Normal file
View file

@ -0,0 +1,140 @@
from django.contrib.auth.models import User
from django.db import models
class Source(models.Model):
"""A subscribed feed. Every source is either news or a blog, and that shapes
everything after it: news gets bundled into stories, shows up in the
"Seit letztem Mal" briefing and expires after a few days; blog articles stay
until read and are listed per blog."""
NEWS = 'news'
BLOG = 'blog'
KINDS = [(NEWS, 'Nachrichten'), (BLOG, 'Blog')]
# Whether the whole article is fetched from its page: "auto" when the feed
# only carries a teaser.
FULLTEXT_AUTO = 'auto'
FULLTEXT_ALWAYS = 'always'
FULLTEXT_NEVER = 'never'
FULLTEXTS = [(FULLTEXT_AUTO, 'Automatisch'), (FULLTEXT_ALWAYS, 'Immer'), (FULLTEXT_NEVER, 'Nie')]
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_sources')
url = models.URLField(max_length=1000)
title = models.CharField(max_length=300)
site_url = models.URLField(max_length=1000, blank=True)
icon_url = models.URLField(max_length=1000, blank=True)
kind = models.CharField(max_length=10, choices=KINDS, default=NEWS)
# News: the section it belongs to (Politik, Tech …), groups the briefing.
category = models.CharField(max_length=100, blank=True)
# Blogs: the folders it's in (labels, any number; a folder exists while a blog carries it).
folders = models.JSONField(default=list, blank=True)
fulltext = models.CharField(max_length=10, choices=FULLTEXTS, default=FULLTEXT_AUTO)
added_at = models.DateTimeField(auto_now_add=True)
# Blogs: favorites come first; a hidden blog stays out of "Alle" (and its
# counts) but shows in its folder; notify = a notification for new posts.
fav = models.BooleanField(default=False)
hidden = models.BooleanField(default=False)
notify = models.BooleanField(default=False)
# Refresh bookkeeping, as for podcast feeds (see podcasts.views._fetch_feed).
last_refreshed_at = models.DateTimeField(null=True, blank=True)
http_etag = models.CharField(max_length=500, blank=True)
http_last_modified = models.CharField(max_length=100, blank=True)
http_body_hash = models.CharField(max_length=64, blank=True)
last_error = models.CharField(max_length=500, blank=True)
last_error_at = models.DateTimeField(null=True, blank=True)
error_count = models.PositiveIntegerField(default=0)
class Meta:
unique_together = ('user', 'url')
ordering = ['title']
def __str__(self):
return f'{self.title} ({self.kind}, user={self.user_id})'
class Story(models.Model):
"""News items about the same event from one or more sources (see cluster.py)."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_stories')
# The item the story is shown with (the first one, or the longest text).
lead = models.ForeignKey('Item', null=True, blank=True, on_delete=models.SET_NULL, related_name='+')
first_at = models.DateTimeField()
last_at = models.DateTimeField(db_index=True)
size = models.PositiveIntegerField(default=1) # items in it
sources = models.PositiveIntegerField(default=1) # distinct sources among them
# Words that describe it (space-separated, for matching new items).
terms = models.TextField(blank=True)
class Meta:
ordering = ['-last_at']
class Item(models.Model):
source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name='items')
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_items')
kind = models.CharField(max_length=10, choices=Source.KINDS) # the source's, at the time
guid_hash = models.CharField(max_length=64) # sha256 of the entry's id (or link)
url = models.URLField(max_length=2000, blank=True)
title = models.CharField(max_length=500)
author = models.CharField(max_length=300, blank=True)
published = models.DateTimeField(db_index=True)
fetched_at = models.DateTimeField(auto_now_add=True)
summary = models.TextField(blank=True) # plain text, short
content = models.TextField(blank=True) # sanitized HTML
# The content came from the article's page, not the feed.
content_full = models.BooleanField(default=False)
# Fetching the page failed or gave nothing: don't try on every open.
fulltext_failed = models.BooleanField(default=False)
# The page showed only the free part (paywall): fetched again once there's
# a subscriber login for the site (logins.py).
paywalled = models.BooleanField(default=False)
image_url = models.URLField(max_length=2000, blank=True)
words = models.PositiveIntegerField(default=0) # of the content: the reading time
# To be announced (a blog with the bell on): set when the item came in, not on a
# source's first fetch and not for posts older than refresh.NOTIFY_MAX_AGE.
notify = models.BooleanField(default=False, db_index=True)
read_at = models.DateTimeField(null=True, blank=True)
# How far it was read (0-1, the article's scroll position), for every device;
# from READ_AT on it counts as read (views.item_progress).
progress = models.FloatField(default=0)
progress_at = models.DateTimeField(null=True, blank=True)
starred = models.BooleanField(default=False)
story = models.ForeignKey(Story, null=True, blank=True, on_delete=models.SET_NULL, related_name='items')
# Expired news: the row stays a while without its text, so the entry isn't
# taken in again as new while the feed still carries it (refresh.expire).
gone = models.BooleanField(default=False)
class Meta:
unique_together = ('source', 'guid_hash')
indexes = [
models.Index(fields=['user', 'kind', 'published']),
models.Index(fields=['user', 'kind', 'read_at']),
]
ordering = ['-published']
def __str__(self):
return self.title
class SiteLogin(models.Model):
"""A subscriber login for a paywalled site (logins.py); the password encrypted."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_site_logins')
domain = models.CharField(max_length=200)
username = models.CharField(max_length=300)
secret = models.TextField() # the password, Fernet-encrypted
ok = models.BooleanField(default=True)
last_error = models.CharField(max_length=300, blank=True)
checked_at = models.DateTimeField(null=True, blank=True)
class Meta:
unique_together = ('user', 'domain')
class FeedSettings(models.Model):
user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='feed_settings')
# The last time the news briefing was seen; what came after is "Seit letztem Mal".
news_seen_at = models.DateTimeField(null=True, blank=True)
# News older than this goes (unless starred).
news_keep_days = models.PositiveIntegerField(default=2)
# The order of the blog folders (their chips), names (docs/library.md).
folder_order = models.JSONField(default=list, blank=True)

278
feeds/refresh.py Normal file
View file

@ -0,0 +1,278 @@
"""Refreshing sources: new entries become items, teasers get their article,
news is bundled into stories, old news expires."""
import logging
import time
from collections import Counter
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
from datetime import timedelta
from django.db import IntegrityError, transaction
from django.utils import timezone
from podcasts.views import FeedFetchError, _clear_error, _error_text, _fetch_feed
from . import fetch, logins
from .cluster import Clusterer
from .models import FeedSettings, Item, Source, Story
log = logging.getLogger(__name__)
FEED_WORKERS = 12
ARTICLE_WORKERS = 8
ARTICLES_PER_RUN = 80 # articles fetched per refresh at most; the rest when opened
BLOG_KEEP = 1000 # items kept per blog
BLOG_FIRST_UNREAD_DAYS = 7 # a new blog's older articles start out read
UNDATED_FIRST_UNREAD = 3 # of a new feed without dates, only the first few
GONE_KEEP_DAYS = 30 # expired news rows kept (without text) against re-adding
NOTIFY_MAX_AGE = timedelta(days=2) # older posts are never announced (feeds dig up old ones)
def settings_for(user):
s, _ = FeedSettings.objects.get_or_create(user=user)
return s
def _apply(source, result, now, keep_days):
"""Write one download; returns the new items."""
if result.status == 304:
source.last_refreshed_at = now
_clear_error(source)
source.save(update_fields=['last_refreshed_at', 'last_error', 'last_error_at', 'error_count',
'http_etag', 'http_last_modified'])
return []
parsed = result.parsed
feed = parsed.feed
if feed.get('title') and (not source.title or source.title == source.url):
source.title = fetch.short_title(fetch.plain(feed.title))[:300]
if feed.get('link'):
source.site_url = feed.link[:1000]
if not source.icon_url:
icon = (feed.get('image') or {}).get('href') or feed.get('icon') or fetch.site_icon(source.site_url)
source.icon_url = (icon or '')[:1000]
if result.permanent_url and result.permanent_url != source.url and \
not Source.objects.filter(user=source.user, url=result.permanent_url).exists():
source.url = result.permanent_url[:1000]
source.http_etag = result.etag
source.http_last_modified = result.last_modified
source.http_body_hash = result.body_hash
source.last_refreshed_at = now
_clear_error(source)
first = not source.items.exists()
source.save()
known = set(source.items.values_list('guid_hash', flat=True))
oldest_kept = None
if source.kind == Source.BLOG and len(known) >= BLOG_KEEP:
oldest_kept = source.items.order_by('published').values_list('published', flat=True).first()
news_cutoff = now - timedelta(days=keep_days)
new = []
entries = [fetch.parse_entry(e, source.url) for e in parsed.entries]
# Some feeds (Deutschlandfunk) give every entry the time the feed was made:
# then the dates say nothing, and the time an entry first shows up counts.
stamps = Counter(f['published'] for f in entries if f['dated'])
bogus_dates = len(entries) >= 5 and bool(stamps) and stamps.most_common(1)[0][1] >= 0.8 * len(entries)
for n, fields in enumerate(entries):
dated = fields.pop('dated')
undated = bogus_dates or not dated
if undated:
# First time: the feed's order stands for the time (newest first), so the
# list doesn't fill up with "just now"; afterwards, when it showed up.
fields['published'] = now - timedelta(minutes=n) if first else now
fields['undated'] = first and n >= UNDATED_FIRST_UNREAD
if fields['guid_hash'] in known:
continue
known.add(fields['guid_hash'])
if source.kind == Source.NEWS and fields['published'] < news_cutoff:
continue
if oldest_kept and fields['published'] < oldest_kept:
continue
old_undated = fields.pop('undated', False)
item = Item(source=source, user=source.user, kind=source.kind, **fields)
if source.kind == Source.BLOG and first and (
old_undated or fields['published'] < now - timedelta(days=BLOG_FIRST_UNREAD_DAYS)):
item.read_at = now
# The bell: never for what a first fetch brings, never for old posts.
item.notify = source.notify and not first and not undated and fields['published'] >= now - NOTIFY_MAX_AGE
new.append(item)
if new:
try:
with transaction.atomic():
Item.objects.bulk_create(new)
except IntegrityError:
# Two entries of one feed with the same id: one by one, skipping the doubles.
kept = []
for item in new:
try:
with transaction.atomic():
item.save()
kept.append(item)
except IntegrityError:
pass
new = kept
return new
def needs_article(item, source):
if not item.url or item.content_full or item.fulltext_failed:
return False
if source.fulltext == Source.FULLTEXT_NEVER:
return False
if item.paywalled:
# Only the free part came: again once there's a working subscriber login.
return logins.has_login(item.user_id, item.url)
return source.fulltext == Source.FULLTEXT_ALWAYS or fetch.is_teaser(item.content)
def load_article(item):
"""Fetch the article for one item now (when opened); saves it."""
http = logins.session_for(item.user_id, item.url)
got = fetch.fetch_article(item.url, http)
if got and got[2] and http:
# Logged in, still the paywall: the session ran out, log in anew once.
http = logins.session_for(item.user_id, item.url, fresh=True)
if http:
got = fetch.fetch_article(item.url, http)
_store_article(item, got)
return item
def _store_article(item, got):
if got is None:
item.fulltext_failed = True
item.save(update_fields=['fulltext_failed'])
return
content, image, paywalled = got
longer = len(fetch.plain(content)) > len(fetch.plain(item.content))
if paywalled:
# The free part, if it's more than the feed's teaser; the rest waits for a login.
item.paywalled = True
if longer:
item.content = content
item.words = fetch.word_count(content)
if not item.image_url and image:
item.image_url = image[:2000]
item.save(update_fields=['paywalled', 'content', 'words', 'image_url'])
return
if not longer:
item.fulltext_failed = True # the feed had more
item.save(update_fields=['fulltext_failed'])
return
item.content = content
item.words = fetch.word_count(content)
item.content_full = True
item.paywalled = False
if not item.image_url and image:
item.image_url = image[:2000]
item.save(update_fields=['content', 'words', 'content_full', 'paywalled', 'image_url'])
def _articles(items, deadline):
todo = [i for i in items if needs_article(i, i.source)][:ARTICLES_PER_RUN]
if not todo:
return
# Logged-in sessions for paywalled sites, one per account and site (made here, not in the threads).
sessions = {}
for i in todo:
adapter = logins.adapter_for(i.url)
if adapter and (i.user_id, adapter.domain) not in sessions:
sessions[(i.user_id, adapter.domain)] = logins.session_for(i.user_id, i.url)
def http_for(i):
adapter = logins.adapter_for(i.url)
return sessions.get((i.user_id, adapter.domain)) if adapter else None
pool = ThreadPoolExecutor(ARTICLE_WORKERS)
futures = {pool.submit(fetch.fetch_article, i.url, http_for(i)): i for i in todo}
pending = set(futures)
try:
while pending:
left = None if deadline is None else deadline - time.monotonic()
if left is not None and left <= 0:
break
done, pending = wait(pending, timeout=left, return_when=FIRST_COMPLETED)
for f in done:
i = futures[f]
try:
got = f.result()
if got and got[2] and http_for(i) is not None:
# Logged in and still the paywall: next time with a fresh login.
logins.forget(i.user_id, logins.adapter_for(i.url).domain)
_store_article(i, got)
except Exception:
log.exception('article for item %s', i.pk)
finally:
pool.shutdown(wait=False, cancel_futures=True)
def expire(user, now=None):
"""News past the user's keep time loses its text (rows stay GONE_KEEP_DAYS
as markers), stories without visible items go."""
now = now or timezone.now()
keep = settings_for(user).news_keep_days
Item.objects.filter(user=user, kind=Source.NEWS, gone=False, starred=False,
published__lt=now - timedelta(days=keep)).update(
gone=True, content='', summary='', story=None)
Item.objects.filter(user=user, gone=True, published__lt=now - timedelta(days=GONE_KEEP_DAYS)).delete()
Story.objects.filter(user=user).exclude(items__gone=False).delete()
# Blogs: the oldest read articles beyond BLOG_KEEP per blog.
for source in Source.objects.filter(user=user, kind=Source.BLOG):
ids = list(source.items.order_by('-published').values_list('id', flat=True)[BLOG_KEEP:])
if ids:
Item.objects.filter(id__in=ids, starred=False).exclude(read_at=None).delete()
def refresh(sources, deadline=None):
"""Refresh `sources` (downloads in parallel, writes one by one).
Returns (refreshed, failed, new items)."""
sources = list(sources)
now = timezone.now()
keep = {}
refreshed = failed = 0
new_items = []
pool = ThreadPoolExecutor(FEED_WORKERS)
try:
futures = {
pool.submit(_fetch_feed, s.url, s.http_etag, s.http_last_modified, s.http_body_hash,
user_agent=fetch.USER_AGENT): s
for s in sources
}
try:
timeout = None if deadline is None else max(0.0, deadline - time.monotonic())
for f in as_completed(futures, timeout=timeout):
s = futures[f]
try:
result = f.result()
if s.user_id not in keep:
keep[s.user_id] = settings_for(s.user).news_keep_days
new_items += _apply(s, result, now, keep[s.user_id])
refreshed += 1
except Exception as e:
if not isinstance(e, FeedFetchError):
log.exception('feed source %s', s.pk)
s.last_error = _error_text(e)
s.last_error_at = now
s.error_count += 1
s.save(update_fields=['last_error', 'last_error_at', 'error_count'])
failed += 1
except TimeoutError:
pass
finally:
# Past the deadline: don't wait for the slow ones (they stay as they are).
pool.shutdown(wait=False, cancel_futures=True)
for item in new_items:
item.source = next(s for s in sources if s.pk == item.source_id)
_articles(new_items, deadline)
by_user = {}
for item in new_items:
if item.kind == Source.NEWS:
by_user.setdefault(item.user_id, []).append(item)
for user_id, items in by_user.items():
c = Clusterer(items[0].user, now)
for item in sorted(items, key=lambda i: i.published):
c.add(item)
for user_id in {s.user_id for s in sources}:
expire(next(s.user for s in sources if s.user_id == user_id), now)
return refreshed, failed, len(new_items)

541
feeds/tests.py Normal file
View file

@ -0,0 +1,541 @@
import json
from datetime import timedelta
from unittest import mock
import feedparser
from django.contrib.auth.models import User
from django.core.cache import cache
from django.test import TestCase
from django.utils import timezone
from podcasts.views import FeedFetch
from . import cluster, fetch, logins, refresh, textproc
from .models import FeedSettings, Item, SiteLogin, Source, Story
def rss(items, title='Testfeed', link='https://example.org/'):
body = ''.join(
f'<item><guid>{i["guid"]}</guid><title>{i["title"]}</title><link>{i.get("link", "https://example.org/" + i["guid"])}</link>'
f'<pubDate>{i.get("date", timezone.now().strftime("%a, %d %b %Y %H:%M:%S +0000"))}</pubDate>'
f'<description><![CDATA[{i.get("text", "")}]]></description></item>'
for i in items)
return f'<?xml version="1.0"?><rss version="2.0"><channel><title>{title}</title><link>{link}</link>{body}</channel></rss>'.encode()
def fetched(content, url='https://example.org/feed'):
return FeedFetch(200, feedparser.parse(content), 'etag', '', url, url, 'hash')
LONG = '<p>' + 'Ein langer Absatz mit sehr viel Text. ' * 40 + '</p>'
class FetchTest(TestCase):
def test_relative_links_become_absolute_and_scripts_go(self):
out = fetch.clean_html('<p>Hi <a href="/a">x</a><img src="i.jpg"><script>bad()</script>'
'<img data-src="/real.jpg" src="placeholder.gif"></p>', 'https://ex.org/post/1')
self.assertIn('href="https://ex.org/a"', out)
self.assertIn('src="https://ex.org/post/i.jpg"', out)
self.assertIn('src="https://ex.org/real.jpg"', out)
self.assertNotIn('placeholder', out)
self.assertNotIn('bad()', out)
def test_parse_entry(self):
e = feedparser.parse(rss([{'guid': 'g1', 'title': 'A &amp; B', 'text': '<p>Kurz <img src="/p.jpg"></p>'}])).entries[0]
f = fetch.parse_entry(e, 'https://example.org/feed')
self.assertEqual(f['title'], 'A & B')
self.assertEqual(f['image_url'], 'https://example.org/p.jpg')
self.assertTrue(fetch.is_teaser(f['content']))
def test_drop_repeated_headline(self):
self.assertEqual(fetch._drop_title('<img src="x"><h1>Titel</h1><p>Text</p>'), '<img src="x"><p>Text</p>')
def test_short_title(self):
self.assertEqual(fetch.short_title('tagesschau.de - die erste Adresse für Nachrichten'), 'tagesschau.de')
self.assertEqual(fetch.short_title('Aktuell - FAZ.NET'), 'FAZ.NET')
self.assertEqual(fetch.short_title('DIE ZEIT | Nachrichten, News, Hintergründe'), 'DIE ZEIT')
self.assertEqual(fetch.short_title('Golem.de'), 'Golem.de')
self.assertEqual(fetch.short_title('Kotlin-Blog'), 'Kotlin-Blog')
def test_discover(self):
feeds, icon = fetch.discover('<link rel="alternate" type="application/rss+xml" href="/feed" title="F">'
'<link rel="icon" href="/fav.png">', 'https://ex.org/')
self.assertEqual(feeds, [('https://ex.org/feed', 'F')])
self.assertEqual(icon, 'https://ex.org/fav.png')
class ClusterTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('u', password='pw12345678')
self.a = Source.objects.create(user=self.u, url='https://a/', title='A')
self.b = Source.objects.create(user=self.u, url='https://b/', title='B')
self.c = Source.objects.create(user=self.u, url='https://c/', title='C')
def item(self, source, title, summary=''):
return Item.objects.create(source=source, user=self.u, kind='news', guid_hash=title[:60],
title=title, summary=summary, published=timezone.now())
def test_same_event_bundled_other_apart(self):
c = cluster.Clusterer(self.u, timezone.now())
s1 = c.add(self.item(self.a, 'Oktoberfest endet mit neuem Besucherrekord in München'))
s2 = c.add(self.item(self.b, 'Rekord-Wiesn: 7,4 Millionen Besucher auf dem Oktoberfest in München'))
s3 = c.add(self.item(self.c, 'Bahn führt Familienticket dauerhaft ein'))
self.assertEqual(s1, s2)
self.assertNotEqual(s1, s3)
s1.refresh_from_db()
self.assertEqual((s1.size, s1.sources), (2, 2))
def test_same_source_needs_to_be_very_close(self):
c = cluster.Clusterer(self.u, timezone.now())
s1 = c.add(self.item(self.a, 'Nations League: Pressekonferenz vor Irland gegen Israel abgebrochen'))
s2 = c.add(self.item(self.a, 'Nations League: England schlägt Kroatien deutlich'))
self.assertNotEqual(s1, s2)
class ApiTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('tester', password='pw12345678')
self.client.force_login(self.u)
def post(self, url, data=None):
return self.client.post(url, json.dumps(data or {}), content_type='application/json')
def add(self, kind, feed, url='https://example.org/feed'):
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed, url)), \
mock.patch('feeds.fetch.fetch_article', return_value=(LONG, 'https://example.org/img.jpg', False)):
r = self.post('/feeds/sources/add/', {'url': url, 'kind': kind, 'category': 'Politik'})
self.assertEqual(r.status_code, 200)
return r.json()['source']
def test_requires_login(self):
self.client.logout()
self.assertEqual(self.client.get('/feeds/state/').status_code, 401)
def test_add_news_fetches_articles_and_bundles(self):
s = self.add('news', rss([{'guid': 'a', 'title': 'Merz besucht Kiew und sagt Hilfe zu', 'text': 'Kurz.'},
{'guid': 'b', 'title': 'Bahn Familienticket dauerhaft', 'text': 'Kurz.'}]))
self.assertEqual(s['title'], 'Testfeed')
self.assertEqual(s['unread'], 2)
items = Item.objects.filter(user=self.u)
self.assertTrue(all(i.content_full for i in items))
self.assertEqual(Story.objects.filter(user=self.u).count(), 2)
news = self.client.get('/feeds/news/').json()
self.assertEqual(len(news['stories']), 2)
self.assertEqual(news['stories'][0]['category'], 'Politik')
b = self.client.get('/feeds/briefing/').json()
self.assertEqual(b['total'], 2)
self.assertEqual(b['sections'][0]['category'], 'Politik')
self.post('/feeds/briefing/seen/')
self.assertFalse(Item.objects.filter(user=self.u, read_at=None).exists())
self.assertEqual(self.client.get('/feeds/briefing/').json()['total'], 0)
def test_blog_keeps_full_feed_text_and_lists(self):
self.add('blog', rss([{'guid': 'p1', 'title': 'Ein Artikel', 'text': LONG}]))
item = Item.objects.get(user=self.u)
self.assertFalse(item.content_full) # the feed had the whole text
self.assertIsNone(item.story)
blogs = self.client.get('/feeds/blogs/?unread=1').json()
self.assertEqual([i['title'] for i in blogs['items']], ['Ein Artikel'])
detail = self.client.get(f'/feeds/items/{item.id}/').json()['item']
self.assertIn('langer Absatz', detail['content'])
self.post('/feeds/items/read/', {'ids': [item.id]})
self.assertEqual(self.client.get('/feeds/blogs/?unread=1').json()['items'], [])
self.post(f'/feeds/items/{item.id}/star/', {'starred': True})
self.assertEqual(len(self.client.get('/feeds/starred/').json()['items']), 1)
def test_blog_folders(self):
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'x', 'title': 'Essay', 'text': LONG}]))):
self.post('/feeds/sources/add/', {'url': 'https://a.example/feed', 'kind': 'blog', 'category': 'Philosophie'})
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'y', 'title': 'Code', 'text': LONG}]))):
self.post('/feeds/sources/add/', {'url': 'https://b.example/feed', 'kind': 'blog', 'category': ''})
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Philosophie'])
self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=Philosophie').json()['items']], ['Essay'])
self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=').json()['items']], ['Code'])
self.assertIn('text="Blogs – Philosophie"', self.client.get('/feeds/export.opml').content.decode())
def test_old_news_expires_and_is_not_taken_in_again(self):
old = (timezone.now() - timedelta(days=3)).strftime('%a, %d %b %Y %H:%M:%S +0000')
feed = rss([{'guid': 'new', 'title': 'Heute passiert', 'text': LONG},
{'guid': 'old', 'title': 'Vorgestern passiert', 'text': LONG, 'date': old}])
src = self.add('news', feed)
self.assertEqual(Item.objects.filter(user=self.u).count(), 1) # too old to take in
Item.objects.filter(user=self.u).update(published=timezone.now() - timedelta(days=3))
refresh.expire(self.u)
self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0)
self.assertEqual(Story.objects.filter(user=self.u).count(), 0)
source = Source.objects.get(pk=src['id'])
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed)):
refresh.refresh([source])
self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0)
def test_feed_wide_timestamp_means_first_seen(self):
stamp = (timezone.now() - timedelta(hours=5)).strftime('%a, %d %b %Y %H:%M:%S +0000')
self.add('news', rss([{'guid': f'g{n}', 'title': f'Meldung {n}', 'text': LONG, 'date': stamp} for n in range(6)]))
for i in Item.objects.filter(user=self.u):
self.assertLess((timezone.now() - i.published).total_seconds(), 600) # first fetch: feed order, minutes apart
def test_new_blog_without_dates_keeps_order_and_few_unread(self):
feed = (b'<?xml version="1.0"?><rss version="2.0"><channel><title>Undatiert</title>'
+ b''.join(b'<item><guid>u%d</guid><title>Post %d</title><description>x</description></item>' % (n, n)
for n in range(6)) + b'</channel></rss>')
self.add('blog', feed)
items = list(Item.objects.filter(user=self.u).order_by('-published'))
self.assertEqual([i.title for i in items], [f'Post {n}' for n in range(6)])
self.assertEqual(sum(1 for i in items if i.read_at is None), 3)
def test_switching_kind_moves_items(self):
src = self.add('news', rss([{'guid': 'a', 'title': 'Etwas', 'text': LONG}]))
self.post(f'/feeds/sources/{src["id"]}/', {'kind': 'blog'})
item = Item.objects.get(user=self.u)
self.assertEqual((item.kind, item.story), ('blog', None))
self.assertEqual(Story.objects.filter(user=self.u).count(), 0)
def test_other_users_items_invisible(self):
self.add('blog', rss([{'guid': 'p1', 'title': 'Privat', 'text': LONG}]))
item = Item.objects.get(user=self.u)
other = User.objects.create_user('o', password='pw12345678')
self.client.force_login(other)
self.assertEqual(self.client.get(f'/feeds/items/{item.id}/').status_code, 404)
self.assertEqual(self.client.get('/feeds/blogs/').json()['items'], [])
self.post('/feeds/items/read/', {'ids': [item.id]})
item.refresh_from_db()
self.assertIsNone(item.read_at)
def test_opml_export(self):
self.add('blog', rss([]), url='https://blog.example/feed')
r = self.client.get('/feeds/export.opml')
self.assertIn(b'xmlUrl="https://blog.example/feed"', r.content)
self.assertIn('text="Blogs – Politik"', r.content.decode())
def test_settings(self):
self.post('/feeds/settings/', {'news_keep_days': 5})
self.assertEqual(FeedSettings.objects.get(user=self.u).news_keep_days, 5)
class FakeResponse:
def __init__(self, status=302, location='/', text='', url='https://www.jungewelt.de/x', ctype='text/html'):
self.status_code = status
self.headers = {'Location': location, 'Content-Type': ctype}
self.text = text
self.url = url
PAYWALL = ('<html><body><article><h1>T</h1><p>' + 'Freier Anfang des Artikels. ' * 12 + '</p></article>'
'<p>Unterstützen Sie die junge Welt mit einem Abonnement und erhalten Sie Zugang zu allen Analysen</p></body></html>')
FULL = '<html><body><article><h1>T</h1>' + ('<p>' + 'Der ganze Artikel für Abonnenten. ' * 15 + '</p>') * 4 + '</article></body></html>'
class SiteLoginTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('tester', password='pw12345678')
self.client.force_login(self.u)
self.src = Source.objects.create(user=self.u, url='https://www.jungewelt.de/feeds/newsticker.rss',
title='junge Welt', kind='news')
self.item = Item.objects.create(source=self.src, user=self.u, kind='news', guid_hash='a', title='Artikel',
url='https://www.jungewelt.de/artikel/1.html', summary='Anriss',
content='<p>Anriss</p>', published=timezone.now())
logins._sessions.clear()
def post(self, url, data):
return self.client.post(url, json.dumps(data), content_type='application/json')
def test_encryption_round_trip_and_not_plain(self):
token = logins.encrypt('geheim')
self.assertNotIn('geheim', token)
self.assertEqual(logins.decrypt(token), 'geheim')
self.assertIsNone(logins.decrypt('kaputt'))
def test_paywalled_page_keeps_free_part_and_waits(self):
with mock.patch('requests.get', return_value=FakeResponse(200, text=PAYWALL)):
refresh.load_article(self.item)
self.item.refresh_from_db()
self.assertTrue(self.item.paywalled)
self.assertFalse(self.item.content_full)
self.assertIn('Freier Anfang', self.item.content)
self.assertFalse(refresh.needs_article(self.item, self.src)) # no login: don't try again
def test_wrong_login_is_refused_and_not_stored(self):
with mock.patch('requests.Session.post', return_value=FakeResponse(302, 'https://www.jungewelt.de/loginFailed.php?x')):
r = self.post('/feeds/logins/', {'source': self.src.id, 'username': 'u', 'password': 'falsch'})
self.assertEqual(r.status_code, 400)
self.assertIn('falsch', r.json()['error'])
self.assertFalse(SiteLogin.objects.exists())
def test_login_stores_encrypted_and_refetches_full_articles(self):
Item.objects.filter(pk=self.item.pk).update(paywalled=True)
def login_post(session, url, **kw):
session.cookies.set('jW', 'sitzung')
return FakeResponse(302, 'https://www.jungewelt.de/')
with mock.patch('requests.Session.post', autospec=True, side_effect=login_post), \
mock.patch('requests.Session.get', return_value=FakeResponse(200, text=FULL)):
r = self.post('/feeds/logins/', {'source': self.src.id, 'username': 'leser', 'password': 'abo-passwort'})
self.assertEqual(r.status_code, 200)
self.assertEqual(r.json()['refetched'], 1)
login = SiteLogin.objects.get(user=self.u)
self.assertNotIn('abo-passwort', login.secret)
self.item.refresh_from_db()
self.assertTrue(self.item.content_full)
self.assertFalse(self.item.paywalled)
self.assertIn('ganze Artikel', self.item.content)
state = self.client.get('/feeds/state/').json()
info = state['sources'][0]['login']
self.assertEqual((info['username'], info['ok'], info['site']), ('leser', True, 'junge Welt'))
self.assertNotIn('abo-passwort', json.dumps(state))
self.post('/feeds/logins/delete/', {'source': self.src.id})
self.assertFalse(SiteLogin.objects.exists())
def test_sources_of_other_sites_have_no_login(self):
other = Source.objects.create(user=self.u, url='https://blog.example/feed', title='B', kind='blog')
state = self.client.get('/feeds/state/').json()
by_id = {s['id']: s for s in state['sources']}
self.assertIsNone(by_id[other.id]['login'])
self.assertIsNotNone(by_id[self.src.id]['login'])
class BlogFeedTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('tester', password='pw12345678')
self.client.force_login(self.u)
now = timezone.now()
self.busy = Source.objects.create(user=self.u, url='https://busy/', title='Busy', kind='blog', folders=['Politik'])
self.rare = Source.objects.create(user=self.u, url='https://rare/', title='Rare', kind='blog', folders=['Politik'])
for n in range(10): # a post every two hours
Item.objects.create(source=self.busy, user=self.u, kind='blog', guid_hash=f'b{n}', title=f'Busy {n}',
content='<p>' + 'wort ' * 460 + '</p>', words=460, published=now - timedelta(hours=2 * n))
Item.objects.create(source=self.rare, user=self.u, kind='blog', guid_hash='r', title='Rare essay',
published=now - timedelta(hours=5))
def post(self, url, data):
return self.client.post(url, json.dumps(data), content_type='application/json')
def by_id(self):
return {s['id']: s for s in self.client.get('/feeds/state/').json()['sources']}
def test_interval_new_and_words(self):
st = self.by_id()
self.assertLess(st[self.busy.id]['interval_days'], 1)
self.assertEqual(st[self.rare.id]['interval_days'], 30.0) # one post: counts as rare
self.assertEqual(st[self.busy.id]['new'], 10)
items = self.client.get('/feeds/blogs/?unread=1').json()['items']
self.assertEqual(next(i for i in items if i['title'] == 'Busy 0')['words'], 460)
def test_hidden_blog_only_in_its_folder_and_not_counted(self):
self.post(f'/feeds/sources/{self.busy.id}/', {'hidden': True})
state = self.client.get('/feeds/state/').json()
self.assertEqual((state['blogs_unread'], state['blogs_new']), (1, 1))
main = [i['title'] for i in self.client.get('/feeds/blogs/?unread=1').json()['items']]
self.assertEqual(main, ['Rare essay'])
folder = self.client.get('/feeds/blogs/?unread=1&folder=Politik').json()['items']
self.assertEqual(len(folder), 11)
def test_favorites_filter(self):
self.post(f'/feeds/sources/{self.rare.id}/', {'fav': True})
self.assertTrue(self.by_id()[self.rare.id]['fav'])
favs = [i['title'] for i in self.client.get('/feeds/blogs/?fav=1').json()['items']]
self.assertEqual(favs, ['Rare essay'])
class NotifyTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('tester', password='pw12345678')
self.client.force_login(self.u)
self.src = Source.objects.create(user=self.u, url='https://example.org/feed', title='Blog', kind='blog', notify=True)
def fetch(self, items):
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss(items))):
refresh.refresh([self.src])
def test_rules(self):
old = (timezone.now() - timedelta(days=5)).strftime('%a, %d %b %Y %H:%M:%S +0000')
self.fetch([{'guid': 'a', 'title': 'Schon da', 'text': LONG}]) # first fetch: never
start = self.client.get('/feeds/notifications/').json()
self.assertEqual(start['items'], [])
self.fetch([{'guid': 'a', 'title': 'Schon da', 'text': LONG},
{'guid': 'b', 'title': 'Neuer Post', 'text': LONG},
{'guid': 'c', 'title': 'Alter Post', 'text': LONG, 'date': old}])
got = self.client.get(f"/feeds/notifications/?after={start['last']}").json()
self.assertEqual([i['title'] for i in got['items']], ['Neuer Post'])
# Seen up to there: nothing again; read or bell off: nothing either.
self.assertEqual(self.client.get(f"/feeds/notifications/?after={got['last']}").json()['items'], [])
Source.objects.filter(pk=self.src.pk).update(notify=False)
self.assertEqual(self.client.get(f"/feeds/notifications/?after={start['last']}").json()['items'], [])
class FolderAndProgressTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('tester', password='pw12345678')
self.client.force_login(self.u)
self.a = Source.objects.create(user=self.u, url='https://a/', title='A', kind='blog', folders=['Philosophie'])
self.b = Source.objects.create(user=self.u, url='https://b/', title='B', kind='blog', folders=['philosophie', 'Rust'])
self.post_a = Item.objects.create(source=self.a, user=self.u, kind='blog', guid_hash='a', title='Essay',
published=timezone.now())
def post(self, url, data):
return self.client.post(url, json.dumps(data), content_type='application/json')
def test_folders_case_insensitive_rename_dissolve(self):
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Philosophie', 'Rust'])
self.post(f'/feeds/sources/{self.a.id}/', {'folders': [' rust ', 'Neu', 'neu']})
self.a.refresh_from_db()
self.assertEqual(self.a.folders, ['Rust', 'Neu']) # spelled as the existing one, no doubles
self.post('/feeds/folders/rename/', {'name': 'rust', 'new': 'Rust & Linux'})
self.a.refresh_from_db(); self.b.refresh_from_db()
self.assertEqual(self.a.folders, ['Neu', 'Rust & Linux'])
self.assertEqual(self.b.folders, ['philosophie', 'Rust & Linux'])
self.post('/feeds/folders/dissolve/', {'name': 'Rust & Linux'})
self.b.refresh_from_db()
self.assertEqual(self.b.folders, ['philosophie'])
self.assertTrue(Source.objects.filter(pk=self.b.pk).exists())
titles = [i['title'] for i in self.client.get('/feeds/blogs/?folder=Neu').json()['items']]
self.assertEqual(titles, ['Essay'])
def test_progress_marks_read_at_ninety_percent(self):
r = self.post(f'/feeds/items/{self.post_a.id}/progress/', {'progress': 0.4}).json()
self.assertEqual(r, {'progress': 0.4, 'read': False})
self.assertEqual(self.client.get('/feeds/blogs/').json()['items'][0]['progress'], 0.4)
self.assertTrue(self.post(f'/feeds/items/{self.post_a.id}/progress/', {'progress': 0.95}).json()['read'])
class DiscoverKindTest(TestCase):
"""Abos & Quellen: a feed whose entries carry sound is offered as a podcast."""
def setUp(self):
self.u = User.objects.create_user('u', password='pw12345678')
self.client.force_login(self.u)
def discover(self, items):
xml = ('<?xml version="1.0"?><rss version="2.0"><channel><title>X</title>' + ''.join(
f'<item><title>Folge {i}</title><guid>g{i}</guid>{extra}</item>' for i, extra in enumerate(items)) +
'</channel></rss>')
with mock.patch('feeds.views._probe', lambda url: (('https://ex.org/feed', 'X', feedparser.parse(xml)), None)):
r = self.client.post('/feeds/discover/', json.dumps({'url': 'https://ex.org/feed'}),
content_type='application/json')
return r.json()['feeds'][0]
def test_sound_files_make_a_podcast(self):
enc = '<enclosure url="https://ex.org/a.mp3" type="audio/mpeg" length="1"/>'
f = self.discover([enc] * 4)
self.assertEqual(f['suggested_kind'], 'podcast')
self.assertFalse(f['podcast_subscribed'])
def test_text_entries_stay_news_or_a_blog(self):
self.assertIn(self.discover([''] * 4)['suggested_kind'], ('news', 'blog'))
class FolderOrderTest(TestCase):
def test_blog_folders_come_in_the_users_order(self):
u = User.objects.create_user('u', password='pw12345678')
self.client.force_login(u)
Source.objects.create(user=u, url='https://a/', title='A', kind='blog', folders=['Kunst', 'Code'])
Source.objects.create(user=u, url='https://b/', title='B', kind='blog', folders=['Politik'])
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Code', 'Kunst', 'Politik'])
r = self.client.post('/feeds/folders/order/', json.dumps({'order': ['Politik', 'kunst']}),
content_type='application/json')
self.assertEqual(r.json()['folders'], ['Politik', 'Kunst', 'Code'])
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Politik', 'Kunst', 'Code'])
DONATE = 'Unterstützen Sie unabhängigen Journalismus mit einer Spende an unsere Redaktion.'
FRANCE = [
('Proteste in Frankreich: Gewalt bei Demonstrationen gegen Macron', 'Die Proteste in Frankreich gegen die Regierung Macron nehmen zu.'),
('Frankreich: Proteste gegen Sparpläne weiten sich aus', 'In Frankreich wachsen die Proteste gegen die Sparpläne von Macron.'),
('Macron verurteilt Gewalt bei Protesten in Frankreich', 'Präsident Macron verurteilt die Gewalt bei den Protesten in Frankreich.'),
]
class TextprocTest(TestCase):
def test_furniture_is_what_a_source_prints_under_many_articles(self):
texts = [f'Artikel Nummer {n} über etwas ganz anderes als die anderen. {DONATE}' for n in 'abcd']
keys = textproc.furniture(texts)
self.assertEqual(textproc.without_furniture(texts[0], keys), 'Artikel Nummer a über etwas ganz anderes als die anderen.')
html = f'<p>Der eigentliche Text des Artikels steht hier.</p><p>{DONATE}</p>'
self.assertEqual(textproc.html_without_furniture(html, keys), '<p>Der eigentliche Text des Artikels steht hier.</p>')
self.assertEqual(textproc.furniture(texts[:2]), set()) # twice is no pattern
def test_overview_of_two_reports(self):
shared = 'Die Regierung in Paris hat am Montag ein neues Sparpaket von vierzig Milliarden Euro vorgestellt. '
a = shared + 'Gewerkschaften kündigten daraufhin landesweite Streiks für die kommende Woche an und riefen zu Demonstrationen auf.'
b = shared + 'Die Börse in Paris reagierte mit deutlichen Kursgewinnen bei den großen Banken des Landes am Nachmittag.'
docs = [('taz', a + ' Gewerkschaften Gewerkschaften'), ('FAZ', b + ' Börse Börse')]
self.assertTrue(textproc.summarize(docs, k=2))
novel = textproc.novel(docs)
self.assertIn('Gewerkschaften', novel['taz'][0])
self.assertIn('Börse', novel['FAZ'][0])
base = ' Die Regierung stellte das Paket vor und die Opposition kritisierte es scharf.' * 3
words = textproc.word_choice([('taz', base + ' Gewerkschaften streiken.' * 5), ('FAZ', base + ' Börse steigt.' * 5)])
self.assertEqual(words, {'taz': ['Gewerkschaften'], 'FAZ': ['Börse']})
def test_topics_group_one_matter_from_several_sources(self):
stories = [(n, f'{t}. {s}', {src}) for n, ((t, s), src) in enumerate(zip(FRANCE, ['taz', 'nd', 'tagesschau']))]
stories.append((9, 'Bahn erhöht die Preise für das Deutschlandticket. Fahrgäste zahlen mehr.', {'taz'}))
got = textproc.topics(stories)
self.assertEqual(len(got), 1)
self.assertEqual(sorted(got[0]['ids']), [0, 1, 2])
self.assertIn('Frankreich', got[0]['label'])
self.assertEqual(textproc.topics([(n, t, {'taz'}) for n, t, _ in stories[:3]]), []) # one source: no topic
def test_related_story_in_the_other_language(self):
de = 'Der Papst Leo traf in Beirut den libanesischen Präsidenten Aoun und Ministerpräsident Salam.'
en = 'Pope Leo met Lebanese President Aoun and Prime Minister Salam in Beirut on Sunday.'
other = 'Die Bahn erhöht in Berlin die Preise für das Deutschlandticket deutlich.'
sl = [(1, textproc.language(de), textproc.names(de)), (2, textproc.language(en), textproc.names(en)),
(3, textproc.language(other), textproc.names(other))]
self.assertEqual((sl[0][1], sl[1][1]), ('de', 'en'))
got = textproc.related(sl, min_weight=1.0)
self.assertEqual([b for b, _ in got[1]], [2])
self.assertNotIn(3, got)
class NewsTextApiTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('reader', password='pw12345678')
self.client.force_login(self.u)
cache.clear()
now = timezone.now()
self.items = []
for n, ((title, text), name) in enumerate(zip(FRANCE, ['taz', 'nd', 'tagesschau'])):
src = Source.objects.create(user=self.u, url=f'https://{name}/feed', title=name, kind='news', category='Welt')
story = Story.objects.create(user=self.u, first_at=now, last_at=now - timedelta(minutes=n))
for k in range(4): # older articles of the source, each with the donation box
Item.objects.create(source=src, user=self.u, kind='news', guid_hash=f'{name}{k}', title=f'Alt {k}',
published=now - timedelta(days=1, minutes=k),
content=f'<p>Ein älterer Artikel Nummer {k} von {name}.</p><p>{DONATE}</p>')
i = Item.objects.create(source=src, user=self.u, kind='news', guid_hash=f'{name}x', title=title,
summary=text, published=now - timedelta(minutes=n), story=story,
content=f'<p>{text} Mehr dazu im Laufe des Tages von {name}.</p><p>{DONATE}</p>')
story.lead = i
story.save()
self.items.append(i)
def test_first_page_has_topics_and_ids_filter(self):
r = self.client.get('/feeds/news/').json()
self.assertEqual(len(r['topics']), 1)
ids = r['topics'][0]['ids']
self.assertEqual(len(ids), 3)
got = self.client.get('/feeds/news/?ids=' + ','.join(map(str, ids[:2]))).json()
self.assertEqual(sorted(s['id'] for s in got['stories']), sorted(ids[:2]))
self.assertNotIn('topics', got)
def test_unencoded_plus_in_before_is_no_error(self):
when = timezone.now().isoformat() # +00:00, sent unencoded the '+' becomes a space
r = self.client.get('/feeds/news/?before=' + when.replace('+', ' '))
self.assertEqual(r.status_code, 200)
self.assertEqual(self.client.get('/feeds/news/?before=garbled').status_code, 200)
def test_article_comes_without_furniture(self):
got = self.client.get(f'/feeds/items/{self.items[0].id}/').json()['item']
self.assertNotIn('Spende', got['content'])
self.assertIn('Proteste', got['content'])
def test_overview_of_a_story_with_several_sources(self):
a, b = self.items[0], self.items[1]
b.story = a.story
b.save()
got = self.client.get(f'/feeds/stories/{a.story_id}/overview/').json()
self.assertTrue(got['summary'])
self.assertFalse(any('Spende' in x['text'] for x in got['summary']))
self.assertEqual(got['related'], [])
single = self.client.get(f'/feeds/stories/{self.items[2].story_id}/overview/').json()
self.assertNotIn('summary', single)

353
feeds/textproc.py Normal file
View file

@ -0,0 +1,353 @@
"""Reading help for news, with classic text methods (no language model):
- furniture: sentences a source prints under many of its articles (donation boxes,
"Diese Nachricht wurde … gesendet") are no part of an article; they are left out
when an article is shown and when a story is looked at;
- a story's overview: a summary drawn from all its reports (LexRank: the sentences
most like the others, without repeating each other — MMR), what only one report
says (its sentences unlike any other source's), and each source's own words for
the event (log-odds of a word in that report against the story's other reports);
- related stories in the other language (German/English): stories of the same days
that share names (capitalised words that aren't at a sentence's start), weighted by
how rare a name is among the day's stories;
- topics: stories grouped by their words when they're about the same matter, not
only the same event, labelled with the words they share most.
Pure Python: these run on the server with the feeds, for one user's stories at a time.
"""
import math
import re
from collections import Counter, defaultdict
from . import fetch
_SENT_END = re.compile(r'(?<=[.!?…])\s+(?=[„"»A-ZÄÖÜ0-9])')
_TOKEN = re.compile(r"[A-Za-zÄÖÜäöüßÀ-ÿ][A-Za-zÄÖÜäöüßÀ-ÿ0-9'’\-]+")
STOP = set('''
aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher
da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese
diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für
gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer
in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen
nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind
so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter
weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier
heute gestern morgen laut sagt sagte sagten erst mal neu video live ticker meldung bericht hatte hätte
außerdem diejenigen derjenige inzwischen zuvor derzeit dennoch trotzdem deswegen ebenfalls
wäre würde würden zudem sowohl jedoch allerdings bereits rund etwa demnach zufolge angaben unter
the and for with from that this have has are was were will would been into about after over their they
his her its our you your not but all can new says said more than when what who how why off out also had
one two which there been being some other could should would may might them then just only very
'''.split())
# Capitalised words that are no names (German nouns are capitalised too: those don't matter,
# they never match an English text; these would match both).
_NOT_NAMES = set('''
Der Die Das Den Dem Des Ein Eine Einen Einem Einer Er Sie Es Wir Ihr Ich Und Oder Aber Auch Doch Mit Von
Nach Bei Für Auf Aus Im In Am Zum Zur Vor Seit Als Wie Wenn Dass Nicht Noch Nur So Da Dann Hier Jetzt
The A An And Or But With From For After Before In On At By To Of As It He She They We I This That These
Those Mr Mrs Ms Dr Monday Tuesday Wednesday Thursday Friday Saturday Sunday Montag Dienstag Mittwoch
Donnerstag Freitag Samstag Sonntag Sonnabend January February March April May June July August September
October November December Januar Februar März Juni Juli Oktober Dezember Reuters AFP AP dpa BBC CNN
'''.split())
def plain(html):
return fetch.plain(html or '')
def sentences(text):
text = re.sub(r'\s+', ' ', text or '').strip()
return [s.strip() for s in _SENT_END.split(text) if 40 <= len(s.strip()) <= 400]
def _key(s):
"""A sentence, the same however its numbers and spaces differ."""
return re.sub(r'\d+', '0', re.sub(r'\s+', ' ', s.lower())).strip()
def words(text):
return [w.lower() for w in _TOKEN.findall(text or '') if len(w) > 2 and w.lower() not in STOP]
# ---- furniture ------------------------------------------------------------------------------------
FURNITURE_MIN = 3 # in this many of a source's articles: furniture
def furniture(texts):
"""The sentences (keys) that stand in at least FURNITURE_MIN of these texts (one source's)."""
seen = Counter()
for t in texts:
seen.update({_key(s) for s in _units(t)})
return {k for k, n in seen.items() if n >= FURNITURE_MIN}
def _units(text):
return [u.strip() for u in re.split(r'\n|(?<=[.!?])\s+', text or '') if len(u.strip()) > 15]
def without_furniture(text, keys):
if not keys:
return text
return ' '.join(u for u in _units(text) if _key(u) not in keys)
_BLOCK = re.compile(r'<(p|li|h[1-6]|blockquote|figcaption|div)\b[^>]*>(.*?)</\1>', re.S | re.I)
def html_without_furniture(html, keys):
"""The article's HTML without the blocks whose text is (only) furniture."""
if not keys or not html:
return html
def keep(m):
inner = plain(m.group(2))
units = _units(inner)
if units and all(_key(u) in keys for u in units):
return ''
return m.group(0)
return _BLOCK.sub(keep, html)
# ---- tf-idf over sentences -------------------------------------------------------------------------
def _vectors(texts):
toks = [Counter(words(t)) for t in texts]
df = Counter()
for t in toks:
df.update(t.keys())
n = len(texts)
idf = {w: math.log((n + 1) / (d + 1)) + 1 for w, d in df.items()}
vecs = []
for t in toks:
v = {w: (1 + math.log(c)) * idf[w] for w, c in t.items()}
norm = math.sqrt(sum(x * x for x in v.values())) or 1.0
vecs.append({w: x / norm for w, x in v.items()})
return vecs
def _cos(a, b):
if len(a) > len(b):
a, b = b, a
return sum(x * b.get(w, 0.0) for w, x in a.items())
def summarize(docs, k=3, lam=0.7):
"""docs: [(source, text)]. The k sentences that say most of what all say, in their order
of appearance: [(source, sentence)]."""
ss = [(src, s) for src, t in docs for s in sentences(t)[:30]]
if len(ss) <= k:
return ss
vecs = _vectors([s for _, s in ss])
n = len(ss)
sim = [[_cos(vecs[a], vecs[b]) if a != b else 0.0 for b in range(n)] for a in range(n)]
links = [[1.0 if sim[a][b] > 0.1 else 0.0 for b in range(n)] for a in range(n)]
deg = [sum(r) or 1.0 for r in links]
rank = [1.0 / n] * n
for _ in range(40):
rank = [0.15 / n + 0.85 * sum(links[b][a] * rank[b] / deg[b] for b in range(n)) for a in range(n)]
top = max(rank) or 1.0
chosen = []
for _ in range(k):
best, score = None, -1e9
for a in range(n):
if a in chosen:
continue
s = lam * rank[a] / top - (1 - lam) * max((sim[a][c] for c in chosen), default=0.0)
if s > score:
best, score = a, s
chosen.append(best)
return [ss[c] for c in sorted(chosen)]
def novel(docs, per_source=2, limit=0.2):
"""What only one report says: per source, up to per_source of its sentences that no other
source has anything like (cosine < limit), the most central of its own first."""
ss = [(src, s) for src, t in docs for s in sentences(t)[:40]]
if len({src for src, _ in ss}) < 2:
return {}
vecs = _vectors([s for _, s in ss])
out = defaultdict(list)
for a, (src, s) in enumerate(ss):
others = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 != src]
if not others or max(others) >= limit:
continue
own = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 == src and b != a]
out[src].append((sum(own) / len(own) if own else 0.0, s))
return {src: [s for _, s in sorted(v, key=lambda x: -x[0])[:per_source]] for src, v in out.items()}
def word_choice(docs, per_source=6):
"""Each source's own words for the event: log-odds of a word in its report against the
other reports (with a prior from all of them), at least twice in its report."""
forms = defaultdict(Counter)
counts = {}
for src, t in docs:
c = Counter()
for w in _TOKEN.findall(t or ''):
lw = w.lower()
if len(lw) > 3 and lw not in STOP:
c[lw] += 1
forms[lw][w] += 1
counts[src] = counts.get(src, Counter()) + c
if len(counts) < 2:
return {}
total = sum(counts.values(), Counter())
a0 = sum(total.values()) or 1
out = {}
for src, c in counts.items():
n_i = sum(c.values()) or 1
rest = total - c
n_j = sum(rest.values()) or 1
scored = []
for w, f in c.items():
if f < 2:
continue
aw = total[w] / a0 * 100
d = math.log((f + aw) / (n_i + 100 - f - aw)) - math.log((rest[w] + aw) / (n_j + 100 - rest[w] - aw))
scored.append((d / math.sqrt(1 / (f + aw) + 1 / (rest[w] + aw)), w))
# Nouns and names (as written: capitalised), not the verbs and particles around them.
best = [(z, forms[w].most_common(1)[0][0]) for z, w in sorted(scored, reverse=True) if z > 1.0]
out[src] = [w for z, w in best if w[:1].isupper()][:per_source]
return {k: v for k, v in out.items() if v}
# ---- names, languages, related stories -------------------------------------------------------------
_EN = set('the and of to in is that for with was said on as by at from'.split())
_DE = set('der die das und ist nicht mit den von zu im sich auf für dem des eine sagte'.split())
def language(text):
w = [x.lower() for x in _TOKEN.findall(text[:3000])]
en = sum(1 for x in w if x in _EN)
de = sum(1 for x in w if x in _DE)
return 'en' if en > de else 'de'
def names(text):
"""Capitalised words not at a sentence's start (in German: names and nouns; in English:
names), as a set of lowercased forms."""
out = set()
for s in sentences(text) or [text or '']:
toks = _TOKEN.findall(s)
for t in toks[1:]:
if t[0].isupper() and t not in _NOT_NAMES and len(t) > 2:
out.add(t.lower().rstrip('s’\''))
return out
def related(stories, min_weight=17.0, per_story=3):
"""stories: [(id, language, set of names)] of the same days. Pairs of stories in different
languages sharing rare names: {id: [(other id, weight)]}, weight = the shared names' idf."""
df = Counter()
for _, _, ns in stories:
df.update(ns)
n = len(stories)
idf = {w: math.log((n + 1) / (d + 1)) for w, d in df.items()}
out = defaultdict(list)
for a in range(n):
ia, la, na = stories[a]
for b in range(a + 1, n):
ib, lb, nb = stories[b]
if la == lb:
continue
shared = na & nb
if len(shared) < 3:
continue
w = sum(idf[x] for x in shared)
if w >= min_weight:
out[ia].append((ib, round(w, 1)))
out[ib].append((ia, round(w, 1)))
return {k: sorted(v, key=lambda x: -x[1])[:per_story] for k, v in out.items()}
# ---- topics ---------------------------------------------------------------------------------------------
NOT_TOPICS = set('breaking news free live watch video sekunden gebärdensprache podcast newsletter abo online'.split())
TOPIC_MIN = 3
def topics(stories, threshold=0.12, min_size=TOPIC_MIN, max_topics=8, rounds=3):
"""stories: [(id, text, sources)] (titles and summaries, the set of source names). Groups of
stories about one matter, the largest first, each with a label (the words most of its stories
share): [{'label': str, 'ids': [ids]}]. Leader clustering against the groups' centroids, a
few rounds (each story to its nearest centroid, then the centroids anew); a group counts
when it has min_size stories from at least two sources."""
if len(stories) < min_size:
return []
vecs = _vectors([s[1] for s in stories])
order = sorted(range(len(stories)), key=lambda i: -len(vecs[i]))
cents = []
assign = [-1] * len(stories)
for _ in range(rounds):
changed = False
for i in order:
best, sim = -1, threshold
for g, c in enumerate(cents):
x = _cos(vecs[i], c)
if x > sim:
best, sim = g, x
if best < 0:
cents.append(dict(vecs[i]))
best = len(cents) - 1
if assign[i] != best:
assign[i] = best
changed = True
# centroids anew from their members
acc = [Counter() for _ in cents]
for i, g in enumerate(assign):
for w, v in vecs[i].items():
acc[g][w] += v
cents = []
remap = {}
for g, a in enumerate(acc):
if a:
norm = math.sqrt(sum(v * v for v in a.values())) or 1.0
remap[g] = len(cents)
cents.append({w: v / norm for w, v in a.items()})
assign = [remap.get(g, -1) for g in assign]
if not changed:
break
groups = defaultdict(list)
for i, g in enumerate(assign):
groups[g].append(i)
out = []
source_words = {w.lower() for s in stories for src in s[2] for w in _TOKEN.findall(src)}
for g in sorted(groups.values(), key=len, reverse=True):
if len(g) < min_size or len({src for i in g for src in stories[i][2]}) < 2:
continue
# Close enough together: the stories' mean likeness to their group's centre.
acc = Counter()
for i in g:
for w, v in vecs[i].items():
acc[w] += v
norm = math.sqrt(sum(v * v for v in acc.values())) or 1.0
centre = {w: v / norm for w, v in acc.items()}
if sum(_cos(vecs[i], centre) for i in g) / len(g) < 0.3:
continue
df = Counter()
for i in g:
df.update({w for w in words(stories[i][1]) if w not in NOT_TOPICS and w not in source_words})
nice = []
for w, c in df.most_common(8):
if c < max(2, len(g) // 2):
break
forms = Counter(t for i in g for t in _TOKEN.findall(stories[i][1]) if t.lower() == w)
form = forms.most_common(1)[0][0] if forms else w
# Nouns and names make a label, not "rückt" or "näher"; "Linke" once, not "Linken" too.
if form[:1].isupper() and not any(x.lower()[:5] == form.lower()[:5] for x in nice):
nice.append(form)
if len(nice) == 3:
break
if len(nice) < 2: # one shared word is a coincidence, not a topic
continue
out.append({'label': ' · '.join(nice), 'ids': [stories[i][0] for i in g]})
if len(out) >= max_topics:
break
return out

32
feeds/urls.py Normal file
View file

@ -0,0 +1,32 @@
from django.urls import path
from . import views
urlpatterns = [
path('state/', views.state, name='feeds_state'),
path('discover/', views.discover, name='feeds_discover'),
path('sources/add/', views.add_source, name='feeds_add_source'),
path('sources/bulk/', views.add_sources, name='feeds_add_sources'),
path('sources/<int:pk>/', views.edit_source, name='feeds_edit_source'),
path('sources/<int:pk>/delete/', views.delete_source, name='feeds_delete_source'),
path('export.opml', views.export_opml, name='feeds_export_opml'),
path('refresh/', views.refresh_now, name='feeds_refresh'),
path('briefing/', views.briefing, name='feeds_briefing'),
path('briefing/seen/', views.briefing_seen, name='feeds_briefing_seen'),
path('news/', views.news, name='feeds_news'),
path('stories/<int:pk>/', views.story, name='feeds_story'),
path('stories/<int:pk>/overview/', views.story_overview, name='feeds_story_overview'),
path('blogs/', views.blogs, name='feeds_blogs'),
path('starred/', views.starred, name='feeds_starred'),
path('items/read/', views.mark_read, name='feeds_mark_read'),
path('items/<int:pk>/', views.item, name='feeds_item'),
path('items/<int:pk>/star/', views.star, name='feeds_star'),
path('items/<int:pk>/progress/', views.item_progress, name='feeds_item_progress'),
path('folders/rename/', views.rename_folder, name='feeds_rename_folder'),
path('folders/order/', views.folder_order, name='feeds_folder_order'),
path('folders/dissolve/', views.dissolve_folder, name='feeds_dissolve_folder'),
path('settings/', views.settings, name='feeds_settings'),
path('notifications/', views.notifications, name='feeds_notifications'),
path('logins/', views.save_login, name='feeds_save_login'),
path('logins/delete/', views.delete_login, name='feeds_delete_login'),
]

811
feeds/views.py Normal file
View file

@ -0,0 +1,811 @@
"""The feed reader's JSON API (session or Bearer token, like the podcast API).
Sources: GET /feeds/state/ sources, unread counts, settings
POST /feeds/discover/ {url} -> the feeds a page or URL offers
POST /feeds/sources/add/ {url, kind, category, title?}
POST /feeds/sources/bulk/ {sources: [{url, title, kind, category}]} (OPML import)
POST /feeds/sources/<id>/ {title, kind, category (news), folders (blogs), fulltext, fav, hidden, notify}
POST /feeds/folders/rename/ {name, new} (all blogs in it)
POST /feeds/folders/dissolve/ {name} (the blogs stay)
POST /feeds/sources/<id>/delete/
GET /feeds/export.opml
POST /feeds/refresh/ refresh the account's sources now
News: GET /feeds/briefing/ what's new since the briefing was last seen
POST /feeds/briefing/seen/ seen: everything up to now counts as read
GET /feeds/news/?before=&category=&ids= stories, newest first (ids: just these, a topic's);
the first page also has the day's topics
GET /feeds/stories/<id>/ one story with all its items
GET /feeds/stories/<id>/overview/ summary, what only one report says, each source's
words, the story in the other language (feeds/textproc.py)
Blogs: GET /feeds/blogs/?folder=&source=&unread=1&fav=1&before= (folder= alone: blogs without one;
without folder/source the hidden blogs stay out)
Items: GET /feeds/items/<id>/ the article (fetches the page if the feed had a teaser)
POST /feeds/items/read/ {ids, read}
POST /feeds/items/<id>/star/ {starred}
POST /feeds/items/<id>/progress/ {progress} 0-1; from 0.9 on it's read
GET /feeds/starred/
Settings: POST /feeds/settings/ {news_keep_days}
Notices: GET /feeds/notifications/?after=<item id> new posts of blogs with the bell on (devices
remember the last id; without after: only that id)
Logins: POST /feeds/logins/ {source, username, password} subscriber login for its site
POST /feeds/logins/delete/ {source}
"""
import json
import time
from datetime import timedelta
from xml.sax.saxutils import quoteattr
import feedparser
import requests
from django.core.cache import cache
from django.db.models import Count
from django.http import HttpResponse, JsonResponse
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from . import fetch, logins, refresh, textproc
from .models import Item, SiteLogin, Source, Story
PAGE = 40
BLOG_PAGE = 150 # blog posts per page (all unread ones come at once, up to BLOG_UNREAD_MAX)
BLOG_UNREAD_MAX = 600
NEW_HOURS = 24 # a blog post this young and unread is "new"
RARE_POSTS = 10 # how often a blog posts: the mean gap over its last this many posts
FOLDER_MAX = 40 # characters of a folder name
READ_AT = 0.9 # read this far, a post counts as read
BRIEFING_TOP = 8
BRIEFING_PER_CATEGORY = 5
REFRESH_WAIT = 40 # seconds an explicit refresh may take
CATEGORY_NONE = 'Allgemein'
def _auth(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
return None
def _body(request):
try:
return json.loads(request.body or b'{}')
except (ValueError, UnicodeDecodeError):
return None
def _when(value):
"""A datetime parameter; None when there's none or it's garbled (an unencoded '+' arrives as
a space)."""
try:
return parse_datetime((value or '').strip().replace(' ', '+'))
except ValueError:
return None
def _iso(dt):
return dt.isoformat() if dt else None
def _source_data(s, unread=0, new=0, interval=None):
return {
'id': s.id, 'url': s.url, 'title': s.title, 'site_url': s.site_url, 'icon_url': s.icon_url,
'kind': s.kind, 'category': s.category, 'folders': s.folders, 'fulltext': s.fulltext,
'unread': unread, 'new': new,
'fav': s.fav, 'hidden': s.hidden, 'notify': s.notify,
# Days between posts (blogs, for "Seltene zuerst"); a blog with one post counts as rare.
'interval_days': interval,
'last_refreshed_at': _iso(s.last_refreshed_at), 'last_error': s.last_error, 'error_count': s.error_count,
'login': logins.info(s.user_id, s.url),
}
def _item_data(i, content=False):
d = {
'id': i.id, 'source': i.source_id, 'source_title': i.source.title, 'kind': i.kind,
'url': i.url, 'title': i.title, 'author': i.author, 'published': _iso(i.published), 'words': i.words,
'summary': i.summary, 'image_url': i.image_url, 'read': i.read_at is not None, 'progress': i.progress,
'starred': i.starred, 'story': i.story_id,
}
if content:
d['content'] = i.content
d['content_full'] = i.content_full
d['paywalled'] = i.paywalled
return d
def _story_data(s, items):
items = sorted(items, key=lambda i: i.published)
lead = next((i for i in items if i.id == s.lead_id), items[0] if items else None)
return {
'id': s.id, 'first_at': _iso(s.first_at), 'last_at': _iso(s.last_at),
'size': len(items), 'sources': len({i.source_id for i in items}),
'category': (lead.source.category if lead else '') or CATEGORY_NONE,
'unread': sum(1 for i in items if i.read_at is None),
'lead': _item_data(lead) if lead else None,
'items': [_item_data(i) for i in items],
}
def _stories(user, qs):
"""Story dicts for a Story queryset (items fetched in one go)."""
stories = list(qs)
items = {}
for i in Item.objects.filter(story__in=stories, gone=False).select_related('source'):
items.setdefault(i.story_id, []).append(i)
return [_story_data(s, items[s.id]) for s in stories if s.id in items]
# ---- sources
@csrf_exempt
@require_http_methods(['GET'])
def state(request):
if (err := _auth(request)):
return err
unread = dict(Item.objects.filter(user=request.user, gone=False, read_at=None)
.values_list('source').annotate(n=Count('id')))
now = timezone.now()
new = dict(Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False, read_at=None,
published__gte=now - timedelta(hours=NEW_HOURS))
.values_list('source').annotate(n=Count('id')))
sources = list(Source.objects.filter(user=request.user))
intervals = _intervals(request.user, [x for x in sources if x.kind == Source.BLOG], now)
s = refresh.settings_for(request.user)
since = s.news_seen_at or (timezone.now() - timedelta(days=1))
new_news = Story.objects.filter(user=request.user, last_at__gt=since,
items__gone=False, items__read_at=None).distinct().count()
return JsonResponse({
'sources': [_source_data(x, unread.get(x.id, 0), new.get(x.id, 0), intervals.get(x.id)) for x in sources],
'categories': sorted({x.category for x in sources if x.kind == Source.NEWS and x.category}),
'folders': _ordered(_all_folders(sources), s.folder_order),
'news_new': new_news,
'blogs_unread': sum(unread.get(x.id, 0) for x in sources if x.kind == Source.BLOG and not x.hidden),
'blogs_new': sum(new.get(x.id, 0) for x in sources if x.kind == Source.BLOG and not x.hidden),
'settings': {'news_keep_days': s.news_keep_days, 'news_seen_at': _iso(s.news_seen_at)},
})
def _clean_folder(name):
return ' '.join(str(name or '').split())[:FOLDER_MAX]
def _all_folders(sources):
"""The folders the blogs carry, each once (case doesn't matter), sorted."""
seen = {}
for s in sources:
if s.kind == Source.BLOG:
for f in s.folders or []:
seen.setdefault(f.casefold(), f)
return sorted(seen.values(), key=str.casefold)
def _ordered(names, order):
"""The folders in the order the user gave them (docs/library.md), new ones after."""
rank = {n.casefold(): i for i, n in enumerate(order or [])}
return sorted(names, key=lambda n: (rank.get(n.casefold(), len(rank)), n.casefold()))
@csrf_exempt
@require_http_methods(['POST'])
def folder_order(request):
"""{order: [names]}: the blog folders' chip order."""
if (err := _auth(request)):
return err
order = (_body(request) or {}).get('order')
if not isinstance(order, list) or not all(isinstance(n, str) for n in order):
return JsonResponse({'error': 'order: eine Liste von Namen'}, status=400)
s = refresh.settings_for(request.user)
s.folder_order = [_clean_folder(n) for n in order if _clean_folder(n)][:100]
s.save(update_fields=['folder_order'])
sources = Source.objects.filter(user=request.user, kind=Source.BLOG)
return JsonResponse({'folders': _ordered(_all_folders(sources), s.folder_order)})
def _folders_for(user, names):
"""Folder names cleaned, without doubles, spelled as an existing folder is."""
existing = {f.casefold(): f for f in _all_folders(Source.objects.filter(user=user, kind=Source.BLOG))}
out = []
for n in names or []:
n = _clean_folder(n)
if n and n.casefold() not in {o.casefold() for o in out}:
out.append(existing.get(n.casefold(), n))
return out
def _in_folder(source, name):
if name == '':
return not source.folders
return name.casefold() in {f.casefold() for f in source.folders or []}
def _intervals(user, blogs, now):
"""Mean days between the last RARE_POSTS posts of each blog, up to now:
(now - the oldest of them) / their number; one post counts as 30 days."""
out = {}
for s in blogs:
dates = list(s.items.filter(gone=False).order_by('-published').values_list('published', flat=True)[:RARE_POSTS])
if not dates:
continue
days = (now - dates[-1]).total_seconds() / 86400 / len(dates)
out[s.id] = round(max(days, 30.0) if len(dates) == 1 else days, 2)
return out
def _probe(url):
"""(feed url, title, entries) if `url` is a feed, else ([candidate feeds], icon)."""
r = requests.get(url, timeout=fetch.PAGE_TIMEOUT, headers={'User-Agent': fetch.USER_AGENT})
r.raise_for_status()
ctype = r.headers.get('Content-Type', '')
if 'html' in ctype and b'<rss' not in r.content[:2000] and b'<feed' not in r.content[:2000]:
return None, fetch.discover(r.text, r.url)
parsed = feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
if parsed.get('bozo') and not parsed.entries and not parsed.feed.get('title'):
return None, ([], '')
return (r.url, fetch.short_title(fetch.plain(parsed.feed.get('title') or '')) or r.url, parsed), None
@csrf_exempt
@require_http_methods(['POST'])
def discover(request):
"""The feeds behind a URL: itself if it's a feed, else the ones the page links to."""
if (err := _auth(request)):
return err
body = _body(request) or {}
url = (body.get('url') or '').strip()
if not url:
return JsonResponse({'error': 'Adresse fehlt'}, status=400)
if not url.startswith(('http://', 'https://')):
url = 'https://' + url
try:
feed, page = _probe(url)
if feed is None:
candidates, _icon = page
found = []
for u, title in candidates[:6]:
try:
f, _ = _probe(u)
except requests.RequestException:
continue
if f:
found.append(f)
if not found:
return JsonResponse({'error': 'Dort ist kein Feed zu finden.'}, status=404)
else:
found = [feed]
except requests.RequestException:
return JsonResponse({'error': 'Die Adresse ist nicht erreichbar.'}, status=502)
from podcasts.models import PodcastFeed
out = []
for u, title, parsed in found:
entries = parsed.entries[:5]
teaser = sum(1 for e in entries if fetch.is_teaser(fetch.parse_entry(e, u)['content']))
# Most entries with a sound or video file: a podcast (Abos & Quellen offers it as one).
media = sum(1 for e in parsed.entries[:10] if _has_media(e))
podcast = media and media * 2 >= min(len(parsed.entries), 10)
out.append({
'url': u, 'title': title,
'subscribed': Source.objects.filter(user=request.user, url=u).exists(),
'podcast_subscribed': PodcastFeed.objects.filter(user=request.user, rss_url=u).exists(),
'samples': [fetch.plain(e.get('title') or '') for e in entries],
# Many entries a day suggests news, few long ones a blog.
'suggested_kind': 'podcast' if podcast
else Source.NEWS if len(parsed.entries) >= 25 or teaser >= 3 else Source.BLOG,
})
return JsonResponse({'feeds': out})
def _has_media(entry):
for enc in entry.get('enclosures') or []:
if (enc.get('type') or '').lower().startswith(('audio/', 'video/')):
return True
return False
def _place(user, kind, category, folders, **more):
"""Where a new source goes: news into a section (category), a blog into
folders (the category of an OPML folder or the add dialog is its folder)."""
category = (category or '').strip()[:100]
if kind == Source.NEWS:
return dict(kind=kind, category=category, **more)
names = folders if isinstance(folders, list) else ([category] if category else [])
return dict(kind=kind, category='', folders=_folders_for(user, names), **more)
def _kind(v):
return v if v in (Source.NEWS, Source.BLOG) else None
@csrf_exempt
@require_http_methods(['POST'])
def add_source(request):
if (err := _auth(request)):
return err
body = _body(request) or {}
url = (body.get('url') or '').strip()
kind = _kind(body.get('kind'))
if not url.startswith(('http://', 'https://')) or not kind:
return JsonResponse({'error': 'Adresse und Art (Nachrichten/Blog) nötig'}, status=400)
source, created = Source.objects.get_or_create(
user=request.user, url=url[:1000],
defaults=_place(request.user, kind, body.get('category'), body.get('folders'),
title=(body.get('title') or url)[:300]))
if created:
refresh.refresh([source], deadline=time.monotonic() + 25)
source.refresh_from_db()
return JsonResponse({'source': _source_data(source, source.items.filter(read_at=None, gone=False).count()),
'created': created})
@csrf_exempt
@require_http_methods(['POST'])
def add_sources(request):
"""Many at once (an OPML file the page read); refreshed by the next run."""
if (err := _auth(request)):
return err
body = _body(request) or {}
added = 0
for x in body.get('sources') or []:
url = (x.get('url') or '').strip()
kind = _kind(x.get('kind'))
if not url.startswith(('http://', 'https://')) or not kind:
continue
_, created = Source.objects.get_or_create(
user=request.user, url=url[:1000],
defaults=_place(request.user, kind, x.get('category'), x.get('folders'),
title=(x.get('title') or url)[:300]))
added += created
sources = Source.objects.filter(user=request.user, last_refreshed_at=None)
refresh.refresh(sources, deadline=time.monotonic() + REFRESH_WAIT)
return JsonResponse({'added': added})
@csrf_exempt
@require_http_methods(['POST'])
def edit_source(request, pk):
if (err := _auth(request)):
return err
source = Source.objects.filter(user=request.user, pk=pk).first()
body = _body(request)
if source is None or body is None:
return JsonResponse({'error': 'not found'}, status=404)
if 'title' in body and (body['title'] or '').strip():
source.title = body['title'].strip()[:300]
for flag in ('fav', 'hidden', 'notify'):
if isinstance(body.get(flag), bool):
setattr(source, flag, body[flag])
if isinstance(body.get('folders'), list):
source.folders = _folders_for(request.user, body['folders'])
elif 'category' in body and source.kind == Source.BLOG and _kind(body.get('kind')) != Source.NEWS:
# Older clients: one folder in the category field.
cat = _clean_folder(body['category'])
source.folders = _folders_for(request.user, [cat]) if cat else []
if 'category' in body and (source.kind == Source.NEWS or _kind(body.get('kind')) == Source.NEWS):
source.category = (body['category'] or '').strip()[:100]
if body.get('fulltext') in dict(Source.FULLTEXTS):
source.fulltext = body['fulltext']
new_kind = _kind(body.get('kind'))
if new_kind and new_kind != source.kind:
# A section becomes a folder and back.
if new_kind == Source.BLOG and source.category and not source.folders:
source.folders, source.category = _folders_for(request.user, [source.category]), ''
elif new_kind == Source.NEWS and source.folders and not source.category:
source.category = source.folders[0][:100]
source.kind = new_kind
# Its items follow; as blog articles they leave their stories.
source.items.update(kind=new_kind, **({'story': None} if new_kind == Source.BLOG else {}))
Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
source.save()
return JsonResponse({'source': _source_data(source)})
@csrf_exempt
@require_http_methods(['POST'])
def delete_source(request, pk):
if (err := _auth(request)):
return err
Source.objects.filter(user=request.user, pk=pk).delete()
Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
return JsonResponse({'ok': True})
@csrf_exempt
@require_http_methods(['GET'])
def export_opml(request):
if (err := _auth(request)):
return err
groups = {}
for s in Source.objects.filter(user=request.user).order_by('kind', 'category', 'title'):
kind = 'Nachrichten' if s.kind == Source.NEWS else 'Blogs'
place = s.category if s.kind == Source.NEWS else ((s.folders or [''])[0])
label = kind + (f' – {place}' if place else '')
groups.setdefault(label, []).append(s)
lines = ['<?xml version="1.0" encoding="UTF-8"?>', '<opml version="2.0">',
'<head><title>diora Feeds</title></head>', '<body>']
for label, sources in groups.items():
lines.append(f'<outline text={quoteattr(label)}>')
for s in sources:
lines.append(f'<outline type="rss" text={quoteattr(s.title)} xmlUrl={quoteattr(s.url)}'
+ (f' htmlUrl={quoteattr(s.site_url)}' if s.site_url else '') + '/>')
lines.append('</outline>')
lines += ['</body>', '</opml>']
resp = HttpResponse('\n'.join(lines), content_type='text/x-opml; charset=utf-8')
resp['Content-Disposition'] = 'attachment; filename="diora-feeds.opml"'
return resp
@csrf_exempt
@require_http_methods(['POST'])
def refresh_now(request):
if (err := _auth(request)):
return err
refreshed, failed, new = refresh.refresh(Source.objects.filter(user=request.user),
deadline=time.monotonic() + REFRESH_WAIT)
return JsonResponse({'refreshed': refreshed, 'failed': failed, 'new': new})
# ---- news
@csrf_exempt
@require_http_methods(['GET'])
def briefing(request):
"""Since the briefing was last seen (else the last day): the top stories
(reported by several sources), then the rest by section, newest first."""
if (err := _auth(request)):
return err
s = refresh.settings_for(request.user)
since = s.news_seen_at or (timezone.now() - timedelta(days=1))
qs = Story.objects.filter(user=request.user, last_at__gt=since,
items__gone=False, items__read_at=None).distinct()
stories = _stories(request.user, qs)
stories.sort(key=lambda x: (-x['sources'], -x['size'], x['last_at']), reverse=False)
top = [x for x in stories if x['sources'] >= 2][:BRIEFING_TOP]
top_ids = {x['id'] for x in top}
sections = {}
for x in sorted(stories, key=lambda x: x['last_at'], reverse=True):
if x['id'] not in top_ids:
sections.setdefault(x['category'], []).append(x)
return JsonResponse({
'since': _iso(since), 'total': len(stories), 'top': top,
'sections': [{'category': c, 'count': len(v), 'stories': v[:BRIEFING_PER_CATEGORY]}
for c, v in sorted(sections.items(), key=lambda kv: (kv[0] == CATEGORY_NONE, kv[0]))],
})
@csrf_exempt
@require_http_methods(['POST'])
def briefing_seen(request):
if (err := _auth(request)):
return err
now = timezone.now()
s = refresh.settings_for(request.user)
Item.objects.filter(user=request.user, kind=Source.NEWS, read_at=None, fetched_at__lte=now).update(read_at=now)
s.news_seen_at = now
s.save(update_fields=['news_seen_at'])
return JsonResponse({'news_seen_at': _iso(now)})
@csrf_exempt
@require_http_methods(['GET'])
def news(request):
if (err := _auth(request)):
return err
qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at')
ids = [int(x) for x in request.GET.get('ids', '').split(',') if x.strip().isdigit()]
if ids:
stories = _stories(request.user, qs.filter(pk__in=ids[:PAGE]))
return JsonResponse({'stories': stories, 'more': False})
before = _when(request.GET.get('before'))
if before:
qs = qs.filter(last_at__lt=before)
category = request.GET.get('category')
if category:
qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category)
stories = _stories(request.user, qs[:PAGE])
out = {'stories': stories, 'more': len(stories) == PAGE}
if not before:
topics = _news_text(request.user)['topics']
if category:
# The category's share of each topic.
mine = set(qs.values_list('pk', flat=True)[:500])
topics = [t for t in ({**t, 'ids': [i for i in t['ids'] if i in mine]} for t in topics)
if len(t['ids']) >= textproc.TOPIC_MIN]
out['topics'] = topics
return JsonResponse(out)
# ---- reading help (feeds/textproc.py)
FURNITURE_ITEMS = 40 # a source's recent articles that show its furniture
TEXT_DAYS = 2 # topics and related stories: of these days
TEXT_CACHE = 6 * 3600
def _furniture(source):
key = f'feeds:furniture:{source.id}'
keys = cache.get(key)
if keys is None:
texts = [textproc.plain(c) for c in Item.objects.filter(source=source, gone=False)
.order_by('-published').values_list('content', flat=True)[:FURNITURE_ITEMS]]
keys = textproc.furniture(texts)
cache.set(key, keys, TEXT_CACHE)
return keys
def _clean_text(i):
"""An item's text without its source's furniture, plain."""
return textproc.without_furniture(textproc.plain(i.content) or i.summary, _furniture(i.source))
def _news_text(user):
"""The news of the last days, looked at as a whole: topics [{label, ids}] and related
stories in the other language {story id: [ids]}. Cached until a story changes."""
since = timezone.now() - timedelta(days=TEXT_DAYS)
qs = Story.objects.filter(user=user, last_at__gte=since, items__gone=False).distinct()
latest = qs.order_by('-last_at').values_list('last_at', flat=True).first()
key = f'feeds:newstext:{user.id}:{qs.count()}:{latest.timestamp() if latest else 0}'
got = cache.get(key)
if got is not None:
return got
items = {}
for i in Item.objects.filter(story__in=qs, gone=False).select_related('source'):
items.setdefault(i.story_id, []).append(i)
tl, sl = [], []
for sid, its in items.items():
tl.append((sid, ' '.join(f'{i.title}. {i.summary}' for i in its), {i.source.title for i in its}))
text = ' '.join(f'{i.title}. {_clean_text(i)[:4000]}' for i in its)
sl.append((sid, textproc.language(text), textproc.names(text)))
got = {
'topics': textproc.topics(tl),
'related': {sid: [b for b, _ in lst] for sid, lst in textproc.related(sl).items()},
}
cache.set(key, got, TEXT_CACHE)
return got
@csrf_exempt
@require_http_methods(['GET'])
def story(request, pk):
if (err := _auth(request)):
return err
found = _stories(request.user, Story.objects.filter(user=request.user, pk=pk))
if not found:
return JsonResponse({'error': 'not found'}, status=404)
return JsonResponse({'story': found[0]})
@csrf_exempt
@require_http_methods(['GET'])
def story_overview(request, pk):
"""What a story's reports say together: a summary from all of them, what only one says,
each source's own words, and the same story in the other language."""
if (err := _auth(request)):
return err
s = Story.objects.filter(user=request.user, pk=pk).first()
if s is None:
return JsonResponse({'error': 'not found'}, status=404)
its = list(Item.objects.filter(story=s, gone=False).select_related('source').order_by('published'))
key = f'feeds:overview:{s.id}:{len(its)}:{sum(len(i.content) for i in its)}'
out = cache.get(key)
if out is None:
docs = [(i.source.title, _clean_text(i)) for i in its]
out = {}
if len({src for src, _ in docs}) >= 2:
summary = textproc.summarize(docs)
out['summary'] = [{'source': src, 'text': t} for src, t in summary]
said = {t for _, t in summary}
novel = {src: [t for t in ts if t not in said] for src, ts in textproc.novel(docs).items()}
out['novel'] = {src: ts for src, ts in novel.items() if ts}
out['words'] = textproc.word_choice(docs)
cache.set(key, out, TEXT_CACHE)
rel = _news_text(request.user)['related'].get(s.id, [])
related = _stories(request.user, Story.objects.filter(user=request.user, pk__in=rel)) if rel else []
return JsonResponse({**out, 'related': related})
# ---- blogs, items
@csrf_exempt
@require_http_methods(['GET'])
def blogs(request):
if (err := _auth(request)):
return err
qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source')
if request.GET.get('source', '').isdigit():
qs = qs.filter(source_id=int(request.GET['source']))
elif 'folder' in request.GET:
# A folder shows all its blogs, the hidden ones too.
name = request.GET['folder']
ids = [s.id for s in Source.objects.filter(user=request.user, kind=Source.BLOG) if _in_folder(s, name)]
qs = qs.filter(source_id__in=ids)
else:
qs = qs.filter(source__hidden=False)
if request.GET.get('fav') == '1':
qs = qs.filter(source__fav=True)
unread_only = request.GET.get('unread') == '1'
if unread_only:
qs = qs.filter(read_at=None)
before = _when(request.GET.get('before'))
if before:
qs = qs.filter(published__lt=before)
# Unread: all of them at once (the page sorts them into days, favorites, rare blogs).
limit = BLOG_UNREAD_MAX if unread_only else BLOG_PAGE
items = list(qs.order_by('-published')[:limit])
return JsonResponse({'items': [_item_data(i) for i in items], 'more': len(items) == limit})
@csrf_exempt
@require_http_methods(['GET'])
def starred(request):
if (err := _auth(request)):
return err
items = Item.objects.filter(user=request.user, starred=True).select_related('source').order_by('-published')
return JsonResponse({'items': [_item_data(i) for i in items[:200]]})
@csrf_exempt
@require_http_methods(['GET'])
def item(request, pk):
if (err := _auth(request)):
return err
i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source').first()
if i is None:
return JsonResponse({'error': 'not found'}, status=404)
if refresh.needs_article(i, i.source):
refresh.load_article(i)
d = _item_data(i, content=True)
d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source))
return JsonResponse({'item': d})
@csrf_exempt
@require_http_methods(['POST'])
def mark_read(request):
if (err := _auth(request)):
return err
body = _body(request) or {}
ids = [int(x) for x in body.get('ids') or [] if str(x).isdigit()]
qs = Item.objects.filter(user=request.user, id__in=ids)
n = qs.update(read_at=timezone.now() if body.get('read', True) else None)
return JsonResponse({'updated': n})
@csrf_exempt
@require_http_methods(['POST'])
def item_progress(request, pk):
"""How far the article was read (from any device); from READ_AT on it's read."""
if (err := _auth(request)):
return err
body = _body(request) or {}
try:
progress = min(1.0, max(0.0, float(body.get('progress'))))
except (TypeError, ValueError):
return JsonResponse({'error': 'progress 0-1'}, status=400)
i = Item.objects.filter(user=request.user, pk=pk).first()
if i is None:
return JsonResponse({'error': 'not found'}, status=404)
i.progress = progress
i.progress_at = timezone.now()
if progress >= READ_AT and i.read_at is None:
i.read_at = i.progress_at
i.save(update_fields=['progress', 'progress_at', 'read_at'])
return JsonResponse({'progress': i.progress, 'read': i.read_at is not None})
@csrf_exempt
@require_http_methods(['POST'])
def rename_folder(request):
"""A folder's new name, on every blog that carries it."""
if (err := _auth(request)):
return err
body = _body(request) or {}
old, new = _clean_folder(body.get('name')), _clean_folder(body.get('new'))
if not old or not new:
return JsonResponse({'error': 'Name fehlt'}, status=400)
n = 0
for s in Source.objects.filter(user=request.user, kind=Source.BLOG):
if _in_folder(s, old):
kept = [f for f in s.folders if f.casefold() != old.casefold() and f.casefold() != new.casefold()]
s.folders = kept + [new]
s.save(update_fields=['folders'])
n += 1
return JsonResponse({'blogs': n})
@csrf_exempt
@require_http_methods(['POST'])
def dissolve_folder(request):
"""The folder goes from every blog; the blogs stay subscribed."""
if (err := _auth(request)):
return err
name = _clean_folder((_body(request) or {}).get('name'))
n = 0
for s in Source.objects.filter(user=request.user, kind=Source.BLOG):
if name and _in_folder(s, name):
s.folders = [f for f in s.folders if f.casefold() != name.casefold()]
s.save(update_fields=['folders'])
n += 1
return JsonResponse({'blogs': n})
@csrf_exempt
@require_http_methods(['POST'])
def star(request, pk):
if (err := _auth(request)):
return err
body = _body(request) or {}
n = Item.objects.filter(user=request.user, pk=pk).update(starred=bool(body.get('starred', True)))
return JsonResponse({'updated': n})
@csrf_exempt
@require_http_methods(['GET'])
def notifications(request):
"""What a device should announce: posts marked when they came in, still
unread, of blogs that still have the bell on, after the id the device saw
last. Without `after` (a device's first time) nothing, only where to start."""
if (err := _auth(request)):
return err
qs = Item.objects.filter(user=request.user, notify=True)
last = qs.order_by('-id').values_list('id', flat=True).first() or 0
after = request.GET.get('after', '')
if not after.isdigit():
return JsonResponse({'items': [], 'last': last})
items = (qs.filter(id__gt=int(after), read_at=None, gone=False, source__notify=True)
.select_related('source').order_by('-published')[:50])
return JsonResponse({'items': [_item_data(i) for i in items], 'last': last})
@csrf_exempt
@require_http_methods(['POST'])
def save_login(request):
"""Check a subscriber login for the source's site and keep it; then the
articles that came only in part are fetched again."""
if (err := _auth(request)):
return err
body = _body(request) or {}
source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
username = (body.get('username') or '').strip()
password = body.get('password') or ''
if source is None or not username or not password:
return JsonResponse({'error': 'Benutzername und Passwort nötig'}, status=400)
try:
adapter = logins.check_and_store(request.user, source.url, username, password)
except logins.LoginFailed as e:
return JsonResponse({'error': str(e)}, status=400)
# Whatever came in part (or not at all) from this site: fetch again, the newest now.
items = Item.objects.filter(user=request.user, gone=False, content_full=False, url__icontains=adapter.domain)
items.update(fulltext_failed=False)
todo = list(items.select_related('source').order_by('-published')[:40])
refresh._articles(todo, deadline=time.monotonic() + 30)
done = Item.objects.filter(pk__in=[i.pk for i in todo], content_full=True).count()
return JsonResponse({'ok': True, 'login': logins.info(request.user.id, source.url), 'refetched': done})
@csrf_exempt
@require_http_methods(['POST'])
def delete_login(request):
if (err := _auth(request)):
return err
body = _body(request) or {}
source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
adapter = logins.adapter_for(source.url) if source else None
if adapter:
SiteLogin.objects.filter(user=request.user, domain=adapter.domain).delete()
logins.forget(request.user.id, adapter.domain)
return JsonResponse({'ok': True})
@csrf_exempt
@require_http_methods(['POST'])
def settings(request):
if (err := _auth(request)):
return err
body = _body(request) or {}
s = refresh.settings_for(request.user)
days = body.get('news_keep_days')
if isinstance(days, int) and 1 <= days <= 30:
s.news_keep_days = days
s.save(update_fields=['news_keep_days'])
return JsonResponse({'news_keep_days': s.news_keep_days})

36
gpodder/tests.py Normal file
View file

@ -0,0 +1,36 @@
import base64
import json
from django.contrib.auth.models import User
from django.test import TestCase
from podcasts.models import EpisodeProgress, PodcastEpisode, PodcastFeed
class EpisodeActionTests(TestCase):
"""Play positions from a gPodder client (AntennaPod sync) land on the user's own episode."""
def setUp(self):
self.alice = User.objects.create_user(username='alice', password='pw12345678')
bob = User.objects.create_user(username='bob', password='pw12345678')
url = 'https://example.org/feed'
audio = 'https://example.org/e1.mp3'
# Bob subscribed first: his copy of the episode has the lower id.
bobs_feed = PodcastFeed.objects.create(user=bob, rss_url=url, title='Feed')
self.bobs_ep = PodcastEpisode.objects.create(feed=bobs_feed, guid='e1', title='E1', audio_url=audio,
duration_seconds=1000)
alices_feed = PodcastFeed.objects.create(user=self.alice, rss_url=url, title='Feed')
self.alices_ep = PodcastEpisode.objects.create(feed=alices_feed, guid='e1', title='E1', audio_url=audio,
duration_seconds=1000)
self.auth = 'Basic ' + base64.b64encode(b'alice:pw12345678').decode()
def test_position_goes_to_own_episode(self):
actions = [{'podcast': 'https://example.org/feed', 'episode': 'https://example.org/e1.mp3',
'action': 'play', 'position': 300, 'total': 1000}]
r = self.client.post('/api/2/episodes/alice.json', json.dumps(actions), content_type='application/json',
HTTP_AUTHORIZATION=self.auth)
self.assertEqual(r.status_code, 200, r.content)
progress = EpisodeProgress.objects.get(user=self.alice)
self.assertEqual(progress.episode, self.alices_ep)
self.assertEqual(progress.position_seconds, 300)
self.assertFalse(EpisodeProgress.objects.filter(episode=self.bobs_ep).exists())

10
gpodder/views.py
View file

@ -8,7 +8,7 @@ from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from podcasts.models import PodcastFeed, PodcastEpisode, EpisodeProgress
from podcasts.models import PodcastFeed, PodcastEpisode, EpisodeProgress, PodcastSettings, counts_as_played
from podcasts.views import _refresh_feed
from .models import GpodderDevice, GpodderSubscriptionChange, GpodderEpisodeAction
@ -151,12 +151,13 @@ def subscriptions_by_device(request, username, deviceid):
except (json.JSONDecodeError, ValueError):
return JsonResponse({'error': 'invalid JSON'}, status=400)
new_feed = PodcastSettings.new_feed_defaults(request.user)
for url in body.get('add', []):
if not url:
continue
PodcastFeed.objects.get_or_create(
user=request.user, rss_url=url,
defaults={'title': url}
defaults={'title': url, **new_feed}
)
GpodderSubscriptionChange.objects.create(user=request.user, url=url, action='add')
@ -230,11 +231,12 @@ def episode_actions(request, username):
# Sync playback position to EpisodeProgress
if action == 'play' and a.get('position') is not None:
try:
ep = PodcastEpisode.objects.filter(audio_url=episode_url).first()
# Only this user's subscriptions: several users can have the same feed.
ep = PodcastEpisode.objects.filter(audio_url=episode_url, feed__user=request.user).first()
if ep:
pos = int(a['position'])
dur = ep.duration_seconds or 0
played = dur > 0 and pos >= dur * 0.9
played = counts_as_played(pos, dur)
EpisodeProgress.objects.update_or_create(
user=request.user, episode=ep,
defaults={'position_seconds': pos, 'played': played}

10
playwright.config.js
View file

@ -6,7 +6,7 @@
// migrated and seeded with a test user.
const path = require('path');
const { defineConfig, devices } = require('@playwright/test');
const { BASE_URL, SERVER_ENV } = require('./e2e/env');
const { BASE_URL, PORT, PYTHON, SERVER_ENV } = require('./e2e/env');
module.exports = defineConfig({
testDir: './e2e',
@ -18,6 +18,9 @@ module.exports = defineConfig({
baseURL: BASE_URL,
trace: 'retain-on-failure',
screenshot: 'only-on-failure',
// A new service worker taking over reloads the page (28-serviceworker.js),
// mid-test. It isn't what these tests are about.
serviceWorkers: 'block',
},
projects: [
{
@ -27,9 +30,10 @@ module.exports = defineConfig({
],
globalSetup: require.resolve('./e2e/global-setup.js'),
webServer: {
command: 'python3 manage.py runserver 127.0.0.1:8000 --noreload',
command: `${PYTHON} manage.py runserver 127.0.0.1:${PORT} --noreload`,
url: BASE_URL + '/accounts/login/',
reuseExistingServer: !process.env.CI,
// Never reuse: a server already on the port has someone's real database behind it.
reuseExistingServer: false,
timeout: 30_000,
cwd: path.join(__dirname),
env: SERVER_ENV,

878
podcasts/antennapod.py Normal file
View file

@ -0,0 +1,878 @@
"""Import an AntennaPod database backup into one user's podcasts.
AntennaPod's "Datenbank exportieren" writes its whole SQLite database. From
it come the subscriptions with their settings, the state of every episode
that has one (played, new, started, queued, favourite), the queue and which
episodes were on the phone (the app offers to download those again). See
the import_antennapod command for how to run it.
The backup is only read. Episodes without any state (and not on the phone)
are not imported: the feed's current episodes come with its next refresh
anyway. Those that diora already has, though, are taken out of the inbox
(AntennaPod: not new).
"""
import re
import sqlite3
import time
from contextlib import contextmanager
from dataclasses import dataclass, field
from datetime import datetime, timezone as dt_timezone
from itertools import groupby
from pathlib import Path
from urllib.parse import urlsplit
from django.db import transaction
from django.db.models import Q
from .models import MAX_SPEED, MIN_SPEED, EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings
from .views import MAX_DESCRIPTION, _keep_sorted, _leave_inbox, _queue_order, _write_queue_order
CHUNK = 500 # ids per IN (...) query and rows per bulk write
# Feeds.feed_volume_adaption -> diora volume factor (0 = off = 1.0)
VOLUME_CODES = {1: 0.5, 2: 0.2, 3: 1.5, 4: 2.0, 5: 2.5}
# Feeds.new_episodes_action; 0 = the global setting (not in the backup)
NEW_EPISODE_ACTIONS = {1: PodcastFeed.NEW_INBOX, 2: PodcastFeed.NEW_IGNORE, 3: PodcastFeed.NEW_QUEUE}
# Feeds.auto_download; 1 = the global setting (not in the backup)
AUTO_DOWNLOAD_OFF, AUTO_DOWNLOAD_GLOBAL, AUTO_DOWNLOAD_ON = 0, 1, 2
# FeedItems.read
NEW, UNPLAYED, PLAYED = -1, 0, 1
SUBSCRIBED = 0 # Feeds.state
# FeedMedia.filesize above this is garbage (seen: 10^13); taken as unknown.
MAX_FILESIZE = 4_000_000_000
SETTING_FIELDS = ('playback_speed', 'skip_intro', 'skip_outro', 'volume', 'include_words',
'exclude_words', 'min_duration', 'new_episodes', 'auto_download', 'notify',
'episode_sort', 'episode_filter', 'skip_silence')
# Feeds.sort_order (SortOrder codes) -> episode_sort; 11 = the global order
# (not in the backup). The others (title Z-A, file name, size) diora can't show.
SORT_ORDERS = {1: 'old', 2: 'new', 3: 'title', 5: 'short', 6: 'long'}
GLOBAL_SORT = 11
# Feeds.hide (FeedItemFilter, comma-separated) -> episode_filter
FILTERS = {'unplayed': 'unplayed', 'played': 'played', 'paused': 'in_progress', 'is_favorite': 'favorites',
'queued': 'queued', 'downloaded': 'downloaded'}
# Feeds.feed_skip_silence (SkipSilence codes); 1 = the global setting
SKIP_SILENCE = {0: False, 2: True}
# Columns read from the backup, with the value to assume if an older
# AntennaPod didn't have the column yet.
FEED_COLUMNS = {
'id': None, 'title': '', 'custom_title': None, 'download_url': '', 'link': '', 'author': '',
'image_url': '', 'keep_updated': 1, 'include_filter': '', 'exclude_filter': '',
'minimal_duration_filter': -1, 'new_episodes_action': 0, 'feed_playback_speed': -1,
'feed_skip_intro': 0, 'feed_skip_ending': 0, 'feed_volume_adaption': 0, 'auto_download': 1,
'episode_notification': 0, 'state': SUBSCRIBED, 'sort_order': None, 'hide': '', 'feed_skip_silence': 1,
}
class AntennaPodImportError(Exception):
"""The file isn't an AntennaPod backup (or the user doesn't exist)."""
# ---------------------------------------------------------------------------
# Small helpers
# ---------------------------------------------------------------------------
def normalize_url(url):
"""A URL reduced to what identifies it: no scheme (http = https), host in
lower case without a leading www., no default port, no trailing slash, no
fragment. None for anything but http(s)."""
url = (url or '').strip()
try:
parts = urlsplit(url)
port = parts.port
except ValueError:
return None
if parts.scheme.lower() not in ('http', 'https') or not parts.hostname:
return None
host = parts.hostname.lower()
if host.startswith('www.'):
host = host[4:]
if port and port not in (80, 443):
host = f'{host}:{port}'
path = parts.path.rstrip('/')
return host + path + (f'?{parts.query}' if parts.query else '')
def _web_url(url, max_length=1000):
"""url if it is a usable http(s) URL for a URLField, else ''."""
url = (url or '').strip()
if normalize_url(url) is None or len(url) > max_length:
return ''
return url
# AntennaPod's filter syntax: words separated by spaces, "quoted phrases".
_TERM = re.compile(r'([^"]\S*|".+?")\s*')
def parse_filter_terms(raw):
"""AntennaPod filter text -> its terms, e.g. 'a "b c"' -> ['a', 'b c']."""
terms = []
for m in _TERM.finditer(raw or ''):
# diora separates its words by commas, so a comma can't be in one.
term = ' '.join(m.group(1).replace('"', '').replace(',', ' ').split())
if term and term.casefold() not in {t.casefold() for t in terms}:
terms.append(term)
return terms
def filter_words(raw, max_length=500):
"""AntennaPod filter text -> diora's comma-separated words ('' if none);
as many whole terms as fit into the field."""
out = ''
for term in parse_filter_terms(raw):
candidate = f'{out}, {term}' if out else term
if len(candidate) > max_length:
break
out = candidate
return out
def _ms(value):
"""A millisecond timestamp -> aware datetime; None for 0/empty/absurd."""
try:
value = int(value or 0)
except (TypeError, ValueError):
return None
if value <= 0:
return None
try:
return datetime.fromtimestamp(value / 1000, tz=dt_timezone.utc)
except (OverflowError, OSError, ValueError):
return None
def _chunks(seq, size=CHUNK):
seq = list(seq)
for i in range(0, len(seq), size):
yield seq[i:i + size]
@contextmanager
def _given_updated_at():
"""Let bulk_create keep the EpisodeProgress.updated_at we set (auto_now
would put now everywhere, and the history and the statistics go by it)."""
f = EpisodeProgress._meta.get_field('updated_at')
f.auto_now = False
try:
yield
finally:
f.auto_now = True
# ---------------------------------------------------------------------------
# Feed settings
# ---------------------------------------------------------------------------
def episode_sort(code):
"""Feeds.sort_order -> (episode_sort or None, representable). None and
the global order are no setting of the feed (representable, nothing to set)."""
try:
code = int(code) if code not in (None, '') else None
except (TypeError, ValueError):
return None, False
if code is None or code == GLOBAL_SORT:
return None, True
return SORT_ORDERS.get(code), code in SORT_ORDERS
def episode_filter(raw):
"""Feeds.hide -> (episode_filter or None, representable): the filter only
if diora can show all of its terms (a part of it would be another list)."""
terms = [t.strip() for t in (raw or '').split(',') if t.strip()]
if not terms:
return None, True
if any(t not in FILTERS for t in terms):
return None, False
return list(dict.fromkeys(FILTERS[t] for t in terms)), True
def feed_settings(ap, global_auto_download=False):
"""diora settings for one AntennaPod feed row: only those AntennaPod sets
to something other than its default. Returns (settings, not_kept_updated)."""
s = {}
speed = ap['feed_playback_speed']
if speed is not None and speed > 0: # -1 = the global speed
s['playback_speed'] = min(max(round(float(speed), 2), MIN_SPEED), MAX_SPEED)
for src, dst in (('feed_skip_intro', 'skip_intro'), ('feed_skip_ending', 'skip_outro')):
seconds = int(ap[src] or 0)
if seconds > 0:
s[dst] = min(seconds, 600)
volume = VOLUME_CODES.get(ap['feed_volume_adaption'] or 0)
if volume:
s['volume'] = volume
for src, dst in (('include_filter', 'include_words'), ('exclude_filter', 'exclude_words')):
words = filter_words(ap[src])
if words:
s[dst] = words
minutes = int(ap['minimal_duration_filter'] or 0) // 60 # seconds, -1 = none
if minutes > 0:
s['min_duration'] = min(minutes, 600)
action = NEW_EPISODE_ACTIONS.get(ap['new_episodes_action'] or 0)
# "Don't update this podcast": AntennaPod never shows anything new of it.
not_kept_updated = not ap['keep_updated'] and action != PodcastFeed.NEW_QUEUE
if not_kept_updated:
action = PodcastFeed.NEW_IGNORE
if action:
s['new_episodes'] = action
code = ap['auto_download']
if code == AUTO_DOWNLOAD_ON:
s['auto_download'] = True
elif code == AUTO_DOWNLOAD_GLOBAL:
s['auto_download'] = bool(global_auto_download)
elif code == AUTO_DOWNLOAD_OFF:
s['auto_download'] = False
# Feeds.episode_notification: 1 = a notification for each new episode
# (0, AntennaPod's default, leaves diora's setting alone).
if ap['episode_notification']:
s['notify'] = True
# The podcast's episode list: its order and filter, where diora has them.
sort, _ = episode_sort(ap['sort_order'])
if sort:
s['episode_sort'] = sort
filters, _ = episode_filter(ap['hide'])
if filters:
s['episode_filter'] = filters
# Skip silence: off or on for this podcast (the global setting: diora's stays).
if ap['feed_skip_silence'] in SKIP_SILENCE:
s['skip_silence'] = SKIP_SILENCE[ap['feed_skip_silence']]
return s, not_kept_updated
# ---------------------------------------------------------------------------
# The import
# ---------------------------------------------------------------------------
@dataclass
class ItemState:
"""What one AntennaPod episode says about its diora episode."""
played: bool
dismissed: bool
position: int # seconds
favorite: bool
activity: datetime | None # last played or finished, if known
when: datetime # what updated_at becomes: activity, else pub date, else the backup date
def merge(self, other):
"""Two AntennaPod episodes (duplicate feeds) for one diora episode:
the one listened to last, a star from either."""
a = self if self.when >= other.when else other
return ItemState(a.played, a.dismissed, a.position, self.favorite or other.favorite, a.activity, a.when)
@dataclass
class Stats:
backup_feeds: int = 0
not_subscribed: int = 0
matched: int = 0
created: int = 0
duplicates: int = 0
created_titles: list = field(default_factory=list)
skipped_feeds: list = field(default_factory=list) # (title, url)
settings: dict = field(default_factory=lambda: {name: 0 for name in SETTING_FIELDS})
not_kept_updated: int = 0
sort_unshown: int = 0 # per-feed orders / filters diora has no equivalent of
filter_unshown: int = 0
auto_download_on: int = 0
auto_download_explicit: int = 0
auto_download_global: int = 0
auto_download_default: bool = False
auto_download_default_changed: bool = False
stateful: int = 0
by_guid: int = 0
by_url: int = 0
episodes_created: int = 0
no_audio: int = 0
in_skipped_feeds: int = 0
stateless_matched: int = 0
progress_new: int = 0
progress_updated: int = 0
progress_unchanged: int = 0
progress_kept: int = 0
left_inbox: int = 0
favorites_backup: int = 0
favorites_before: int = 0
favorites_after: int = 0
queue_backup: int = 0
queue_missing: int = 0
queue_before: int = 0
queue_after: int = 0
queue_appended: int = 0
keep_sorted: str = ''
inbox_before: int = 0
inbox_after: int = 0
downloads_backup: int = 0
downloads: int = 0
downloads_bytes: int = 0
downloads_created: int = 0
downloads_missing: int = 0
downloads_offered: bool = True
ignored_left_inbox: int = 0
queue_removed: int = 0
removed_titles: list = field(default_factory=list)
diora_only_left_inbox: int = 0
backup_date: datetime | None = None
seconds: float = 0.0
def _inbox_count(user):
hidden = EpisodeProgress.objects.filter(user=user).filter(Q(played=True) | Q(dismissed=True))
return PodcastEpisode.objects.filter(feed__user=user).exclude(id__in=hidden.values('episode_id')).count()
def _open_backup(path):
uri = Path(path).resolve().as_uri() + '?mode=ro'
db = None
try:
db = sqlite3.connect(uri, uri=True)
tables = {r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
except sqlite3.DatabaseError as e:
if db is not None:
db.close()
raise AntennaPodImportError(f'{path} is not an SQLite database: {e}')
missing = {'Feeds', 'FeedItems', 'FeedMedia', 'Queue', 'Favorites'} - tables
if missing:
db.close()
raise AntennaPodImportError(
f'{path} is not an AntennaPod backup (missing tables: {", ".join(sorted(missing))})')
return db
def _columns(db, table):
return {r[1] for r in db.execute(f'PRAGMA table_info({table})')}
class AntennaPodImport:
def __init__(self, user, path, backup_date, global_auto_download=False, offer_downloads=True,
quiet_diora_only=False, replace=False):
self.user = user
self.path = path
self.backup_date = backup_date
self.global_auto_download = global_auto_download
# False: the app doesn't offer AntennaPod's downloads (they stay in AntennaPod).
self.offer_downloads = offer_downloads
# True: subscriptions only diora has keep their episodes, but the
# ones out before the backup leave the inbox.
self.quiet_diora_only = quiet_diora_only
# A clean slate: AntennaPod's state is the new start. Podcasts only
# diora has are unsubscribed, the queue is AntennaPod's alone, and
# AntennaPod wins also over later changes in diora.
self.replace = replace
self.stats = Stats(backup_date=backup_date)
def run(self, dry_run=False):
started = time.monotonic()
db = _open_backup(self.path)
try:
with transaction.atomic():
self._run(db)
if dry_run:
transaction.set_rollback(True)
finally:
db.close()
self.stats.seconds = time.monotonic() - started
return self.stats
def _run(self, db):
st = self.stats
user = self.user
st.queue_before = len(_queue_order(user))
st.inbox_before = _inbox_count(user)
st.favorites_before = EpisodeProgress.objects.filter(user=user, favorite=True).count()
# AntennaPod downloads new episodes of every podcast by default: so
# do diora's future subscriptions.
settings = PodcastSettings.for_user(user)
if self.global_auto_download and not settings.auto_download_default:
settings.auto_download_default = True
settings.save(update_fields=['auto_download_default'])
st.auto_download_default_changed = True
st.auto_download_default = settings.auto_download_default
self.new_feed = PodcastSettings.new_feed_defaults(user)
feed_map = self._import_feeds(db) # AntennaPod feed id -> diora feed id
if self.replace:
self._remove_diora_only(feed_map)
queue_items = [r[0] for r in db.execute('SELECT feeditem FROM Queue ORDER BY id')]
favorite_items = {r[0] for r in db.execute('SELECT feeditem FROM Favorites')}
st.queue_backup = len(queue_items)
st.favorites_backup = len(favorite_items)
states, not_new, item_episode, downloads = self._import_episodes(
db, feed_map, set(queue_items), favorite_items)
self._import_progress(states, not_new)
self._quiet_inbox(feed_map, states)
self._import_queue(queue_items, item_episode)
self._import_downloads(settings, downloads, item_episode)
st.queue_after = len(_queue_order(user))
st.inbox_after = _inbox_count(user)
st.favorites_after = EpisodeProgress.objects.filter(user=user, favorite=True).count()
# -- feeds --------------------------------------------------------------
def _import_feeds(self, db):
st = self.stats
have = _columns(db, 'Feeds')
select = ', '.join(c if c in have else f'{default!r} AS {c}' if default is not None else f'NULL AS {c}'
for c, default in FEED_COLUMNS.items())
db.row_factory = sqlite3.Row
try:
rows = db.execute(f'SELECT {select} FROM Feeds ORDER BY id').fetchall()
finally:
db.row_factory = None
st.backup_feeds = len(rows)
mine = {}
for f in PodcastFeed.objects.filter(user=self.user).order_by('id'):
mine.setdefault(normalize_url(f.rss_url) or f.rss_url, f)
feed_map = {} # AntennaPod id -> PodcastFeed (saved or still to create)
to_create = {} # normalised URL -> new PodcastFeed
changed = {} # diora id -> PodcastFeed with changed settings
for ap in rows:
if ap['state'] != SUBSCRIBED:
st.not_subscribed += 1
continue
url = (ap['download_url'] or '').strip()
title = (ap['custom_title'] or ap['title'] or url).strip()
key = normalize_url(url)
if key is None or len(url) > 1000:
st.skipped_feeds.append((title, url))
continue
feed = mine.get(key) or to_create.get(key)
if feed is None:
feed = PodcastFeed(
user=self.user, rss_url=url, title=title[:300], author=(ap['author'] or '')[:300],
link=_web_url(ap['link']), artwork_url=_web_url(ap['image_url']), **self.new_feed,
)
to_create[key] = feed
st.created += 1
st.created_titles.append(feed.title)
elif any(f is feed for f in feed_map.values()):
st.duplicates += 1
elif feed.pk:
st.matched += 1
feed_map[ap['id']] = feed
settings, not_kept_updated = feed_settings(ap, self.global_auto_download)
st.not_kept_updated += not_kept_updated
st.sort_unshown += not episode_sort(ap['sort_order'])[1]
st.filter_unshown += not episode_filter(ap['hide'])[1]
st.auto_download_explicit += ap['auto_download'] == AUTO_DOWNLOAD_ON
st.auto_download_global += ap['auto_download'] == AUTO_DOWNLOAD_GLOBAL
for name, value in settings.items():
if getattr(feed, name) != value:
setattr(feed, name, value)
st.settings[name] += 1
if feed.pk:
changed[feed.pk] = feed
feed.auto_queue = feed.new_episodes == PodcastFeed.NEW_QUEUE
if changed:
PodcastFeed.objects.bulk_update(list(changed.values()), [*SETTING_FIELDS, 'auto_queue'],
batch_size=CHUNK)
if to_create:
# last_refreshed_at stays empty: the next refresh fetches them,
# with the first-fetch rule (only the newest episode is new).
PodcastFeed.objects.bulk_create(list(to_create.values()), batch_size=CHUNK)
ids = {}
for urls in _chunks(f.rss_url for f in to_create.values()):
ids.update(PodcastFeed.objects.filter(user=self.user, rss_url__in=urls).values_list('rss_url', 'id'))
for f in to_create.values():
f.pk = ids[f.rss_url]
st.auto_download_on = sum(1 for f in {id(f): f for f in feed_map.values()}.values() if f.auto_download)
return {ap_id: f.pk for ap_id, f in feed_map.items()}
# -- episodes -----------------------------------------------------------
def _import_episodes(self, db, feed_map, queued, favorites):
"""Match AntennaPod's episodes to diora's, create the missing ones
that have a state (or were downloaded). Returns ({episode id: ItemState}
for those with a state, {episode ids} of matched ones without (not new),
{AntennaPod item id: episode id}, [(pub date ms, AntennaPod item id,
size)] of the downloaded ones)."""
st = self.stats
backup = self.backup_date
states = {}
not_new = set()
item_episode = {}
downloads = []
to_create = {} # (diora feed id, guid) -> [ItemState, AntennaPod item id, audio url, duration, pub_date]
media = _columns(db, 'FeedMedia')
cur = db.execute(
'SELECT i.feed, i.id, i.item_identifier, i.read, i.pubDate, m.download_url, m.duration, '
'm.position, m.last_played_time, m.playback_completion_date, '
+ ('m.downloaded, ' if 'downloaded' in media else '0, ')
+ ('m.filesize ' if 'filesize' in media else '0 ')
+ 'FROM FeedItems i LEFT JOIN FeedMedia m ON m.feeditem = i.id ORDER BY i.feed, i.id'
)
def rows():
while True:
batch = cur.fetchmany(5000)
if not batch:
return
yield from batch
for ap_feed, items in groupby(rows(), key=lambda r: r[0]):
feed_id = feed_map.get(ap_feed)
by_guid = by_url = None
for (_, item_id, ident, read, pub_ms, audio, duration, position, last_played, completed,
downloaded, size) in items:
position = max(int(position or 0), 0) // 1000
activity = max(filter(None, (_ms(last_played), _ms(completed))), default=None)
in_queue = item_id in queued
favorite = item_id in favorites
stateful = read in (NEW, PLAYED) or position > 0 or activity is not None or in_queue or favorite
# FeedMedia.downloaded: when the file was downloaded (ms; 1 from
# older versions), 0 = not on the phone.
downloaded = (downloaded or 0) > 0
if downloaded:
size = int(size or 0)
downloads.append((int(pub_ms or 0), item_id, size if 0 < size <= MAX_FILESIZE else 0))
if feed_id is None:
st.in_skipped_feeds += stateful
continue
if by_guid is None:
by_guid, by_url = {}, {}
for ep_id, guid, url in (PodcastEpisode.objects.filter(feed_id=feed_id)
.order_by('id').values_list('id', 'guid', 'audio_url')):
by_guid.setdefault(guid, ep_id)
by_url.setdefault(normalize_url(url), ep_id)
ident = (ident or '').strip()
audio = (audio or '').strip()
guid = (ident or audio)[:1000]
ep_id = by_guid.get(guid) if guid else None
how = 'guid'
if ep_id is None and audio:
ep_id = by_url.get(normalize_url(audio))
how = 'url'
if not stateful:
if ep_id is not None:
not_new.add(ep_id)
item_episode[item_id] = ep_id
continue
if not downloaded:
continue
# On the phone in AntennaPod but not in diora yet (a feed
# not fetched so far): created as not new, so the app can
# offer to download it again.
else:
st.stateful += 1
played = read == PLAYED
pub_date = _ms(pub_ms)
state = ItemState(
played=played,
# Only AntennaPod's "new" stays in the inbox; queued is never new.
dismissed=read != NEW or in_queue,
position=0 if played else position,
favorite=favorite,
activity=activity,
when=activity or (min(pub_date, backup) if pub_date else backup),
)
if ep_id is not None:
st.by_guid += how == 'guid'
st.by_url += how == 'url'
states[ep_id] = state.merge(states[ep_id]) if ep_id in states else state
item_episode[item_id] = ep_id
continue
if normalize_url(audio) is None or len(audio) > 1000:
st.no_audio += stateful
continue
key = (feed_id, guid)
if key in to_create:
pending = to_create[key]
pending[0] = state.merge(pending[0])
pending[1].append(item_id)
else:
to_create[key] = [state, [item_id], audio, max(int(duration or 0), 0) // 1000, pub_date]
st.downloads_created += not stateful
if to_create:
self._create_episodes(db, to_create, states, item_episode)
not_new -= states.keys()
st.stateless_matched = len(not_new)
return states, not_new, item_episode, downloads
def _create_episodes(self, db, to_create, states, item_episode):
st = self.stats
# A few at a time: shownotes can be large.
for keys in _chunks(to_create, 200):
first_item = {to_create[key][1][0]: key for key in keys}
marks = ','.join('?' * len(first_item))
details = {
first_item[item_id]: (title, description, image)
for item_id, title, description, image in db.execute(
f'SELECT id, title, description, image_url FROM FeedItems WHERE id IN ({marks})',
list(first_item))
}
batch = []
for key in keys:
_, _, audio, duration, pub_date = to_create[key]
title, description, image = details.get(key, ('', '', ''))
batch.append(PodcastEpisode(
feed_id=key[0], guid=key[1], title=(title or 'Untitled')[:500],
description=(description or '')[:MAX_DESCRIPTION], audio_url=audio,
duration_seconds=duration, pub_date=pub_date, artwork_url=_web_url(image),
))
PodcastEpisode.objects.bulk_create(batch)
st.episodes_created = len(to_create) - st.downloads_created
by_feed = {}
for feed_id, guid in to_create:
by_feed.setdefault(feed_id, []).append(guid)
for feed_id, guids in by_feed.items():
for chunk in _chunks(guids):
created = PodcastEpisode.objects.filter(feed_id=feed_id, guid__in=chunk).values_list('guid', 'id')
for guid, ep_id in created:
state, item_ids = to_create[(feed_id, guid)][:2]
states[ep_id] = state
for item_id in item_ids:
item_episode[item_id] = ep_id
# -- progress -----------------------------------------------------------
def _import_progress(self, states, not_new):
st = self.stats
user = self.user
backup = self.backup_date
existing = {}
for ids in _chunks([*states, *not_new]):
for p in EpisodeProgress.objects.filter(user=user, episode_id__in=ids):
existing[p.episode_id] = p
def diora_newer(p, activity):
# Changed in diora after the backup was made (and after the
# episode was last played in AntennaPod): diora's state stays.
return not self.replace and p.updated_at > backup and (activity is None or p.updated_at > activity)
new_rows, changed, favorite_only, dismiss_only = [], [], [], []
for ep_id, s in states.items():
p = existing.get(ep_id)
if p is None:
new_rows.append(EpisodeProgress(
user=user, episode_id=ep_id, position_seconds=s.position, played=s.played,
dismissed=s.dismissed, favorite=s.favorite, updated_at=s.when,
))
st.progress_new += 1
continue
favorite = p.favorite or s.favorite # stars from both
if diora_newer(p, s.activity):
st.progress_kept += 1
if favorite != p.favorite:
favorite_only.append(p.pk)
continue
wanted = (s.position, s.played, s.dismissed, favorite, s.when)
if (p.position_seconds, p.played, p.dismissed, p.favorite, p.updated_at) == wanted:
st.progress_unchanged += 1
continue
p.position_seconds, p.played, p.dismissed, p.favorite, p.updated_at = wanted
changed.append(p)
st.progress_updated += 1
# Known to AntennaPod, but not new there: out of the inbox.
for ep_id in not_new:
p = existing.get(ep_id)
if p is None:
new_rows.append(EpisodeProgress(user=user, episode_id=ep_id, dismissed=True, updated_at=backup))
st.left_inbox += 1
elif not p.played and not p.dismissed and not diora_newer(p, None):
dismiss_only.append(p.pk)
st.left_inbox += 1
with _given_updated_at():
EpisodeProgress.objects.bulk_create(new_rows, batch_size=CHUNK)
# bulk_update writes updated_at as given (no auto_now there).
EpisodeProgress.objects.bulk_update(
changed, ['position_seconds', 'played', 'dismissed', 'favorite', 'updated_at'], batch_size=CHUNK)
# A star or leaving the inbox isn't listening: updated_at stays.
for ids in _chunks(favorite_only):
EpisodeProgress.objects.filter(pk__in=ids).update(favorite=True)
for ids in _chunks(dismiss_only):
EpisodeProgress.objects.filter(pk__in=ids).update(dismissed=True)
def _quiet_inbox(self, feed_map, states):
"""Episodes AntennaPod never showed as new, though diora has them in
the inbox: those of podcasts whose new episodes are ignored (AntennaPod
stopped fetching archived ones, so it never saw them), and with
quiet_diora_only those of podcasts AntennaPod doesn't have. Only ones
out before the backup; AntennaPod's own new ones stay."""
st = self.stats
user = self.user
backup = self.backup_date
imported = set(feed_map.values())
feeds = set(PodcastFeed.objects.filter(pk__in=imported, new_episodes=PodcastFeed.NEW_IGNORE)
.values_list('pk', flat=True))
diora_only = set()
if self.quiet_diora_only:
diora_only = set(PodcastFeed.objects.filter(user=user).exclude(pk__in=imported).values_list('pk', flat=True))
if not feeds and not diora_only:
return
keep = {ep_id for ep_id, s in states.items() if not s.dismissed}
hidden = EpisodeProgress.objects.filter(user=user).filter(Q(played=True) | Q(dismissed=True))
rows = (PodcastEpisode.objects.filter(feed_id__in=feeds | diora_only)
.filter(Q(pub_date__lte=backup) | Q(pub_date__isnull=True))
.exclude(id__in=hidden.values('episode_id'))
.values_list('id', 'feed_id'))
ids = []
for ep_id, feed_id in rows:
if ep_id in keep:
continue
ids.append(ep_id)
if feed_id in feeds:
st.ignored_left_inbox += 1
else:
st.diora_only_left_inbox += 1
# Changed in diora after the backup: diora's state stays.
newer = set()
for chunk in ([] if self.replace else _chunks(ids)):
newer.update(EpisodeProgress.objects.filter(user=user, episode_id__in=chunk, updated_at__gt=backup)
.values_list('episode_id', flat=True))
ids = [i for i in ids if i not in newer]
for chunk in _chunks(ids):
EpisodeProgress.objects.filter(user=user, episode_id__in=chunk, dismissed=False).update(dismissed=True)
have = set(EpisodeProgress.objects.filter(user=user, episode_id__in=chunk)
.values_list('episode_id', flat=True))
with _given_updated_at():
EpisodeProgress.objects.bulk_create(
[EpisodeProgress(user=user, episode_id=i, dismissed=True, updated_at=backup)
for i in chunk if i not in have],
ignore_conflicts=True,
)
def _remove_diora_only(self, feed_map):
"""Unsubscribe the podcasts the backup doesn't have (their episodes,
states and queue entries go with them)."""
st = self.stats
gone = PodcastFeed.objects.filter(user=self.user).exclude(pk__in=set(feed_map.values()))
st.removed_titles = sorted(gone.values_list('title', flat=True))
gone.delete()
# -- queue --------------------------------------------------------------
def _import_queue(self, queue_items, item_episode):
"""AntennaPod's queue first, then what else was in diora's."""
st = self.stats
user = self.user
before = _queue_order(user)
front = []
for item_id in queue_items:
ep_id = item_episode.get(item_id)
if ep_id is None:
st.queue_missing += 1
elif ep_id not in front:
front.append(ep_id)
if not front:
return
in_front = set(front)
rest = [ep_id for ep_id in before if ep_id not in in_front]
if self.replace and rest:
for chunk in _chunks(rest):
PodcastQueue.objects.filter(user=user, episode_id__in=chunk).delete()
st.queue_removed = len(rest)
before = [ep_id for ep_id in before if ep_id in in_front]
rest = []
st.queue_appended = len(rest)
present = set(before)
PodcastQueue.objects.bulk_create(
[PodcastQueue(user=user, episode_id=ep_id, position=len(before) + i)
for i, ep_id in enumerate(e for e in front if e not in present)],
batch_size=CHUNK,
)
_write_queue_order(user, front + rest)
_leave_inbox(user, front) # queued = not new
if _keep_sorted(user):
st.keep_sorted = PodcastSettings.objects.get(user=user).keep_sorted
# -- downloads ----------------------------------------------------------
def _import_downloads(self, settings, downloads, item_episode):
"""The episodes AntennaPod had on the phone, newest first: the app
can't take its files over, so it offers to download them again.
Replaces what an earlier import left."""
st = self.stats
st.downloads_backup = len(downloads)
entries, seen = [], set()
for _, item_id, size in sorted(downloads, key=lambda d: (-d[0], -d[1])):
ep_id = item_episode.get(item_id)
if ep_id is None:
st.downloads_missing += 1
continue
if ep_id in seen:
continue
seen.add(ep_id)
entries.append({'episode_id': ep_id, 'size': size})
st.downloads = len(entries)
st.downloads_bytes = sum(e['size'] for e in entries)
st.downloads_offered = self.offer_downloads
settings.import_downloads = entries if self.offer_downloads else []
settings.save(update_fields=['import_downloads'])
# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------
SETTING_LABELS = {
'playback_speed': 'speed', 'skip_intro': 'skip intro', 'skip_outro': 'skip outro', 'volume': 'volume',
'include_words': 'include words', 'exclude_words': 'exclude words', 'min_duration': 'min duration',
'new_episodes': 'new episodes', 'auto_download': 'auto-download', 'notify': 'notifications',
'episode_sort': 'episode order', 'episode_filter': 'episode filter', 'skip_silence': 'skip silence',
}
def _on(flag):
return 'on' if flag else 'off'
def summary_lines(st, username, path, dry_run, global_auto_download):
when = st.backup_date.strftime('%Y-%m-%d %H:%M UTC') if st.backup_date else '?'
subscribed = st.backup_feeds - st.not_subscribed
settings = ', '.join(f'{SETTING_LABELS[k]} {v}' for k, v in st.settings.items())
lines = [
f"AntennaPod import for user '{username}' from {Path(path).name} (backup date {when})"
+ (' -- DRY RUN, nothing was written' if dry_run else ''),
f'Feeds: {st.backup_feeds} in backup, {subscribed} subscribed ({st.not_subscribed} not subscribed '
f'ignored); {st.matched} matched, {st.created} created, {len(st.skipped_feeds)} skipped (not http/https), '
f'{st.duplicates} duplicates'
+ (f', {len(st.removed_titles)} only in diora unsubscribed (--replace)' if st.removed_titles else ''),
f'Settings: changed: {settings}',
f' {st.not_kept_updated} feeds not kept updated in AntennaPod -> new episodes "ignore"',
f' episode lists diora can\'t show that way (left as they are): {st.sort_unshown} orders, '
f'{st.filter_unshown} filters',
f'Auto-download: on for {st.auto_download_on} feeds ({st.auto_download_explicit} on in AntennaPod, '
f"{st.auto_download_global} use AntennaPod's global setting -> {_on(global_auto_download)} "
f'by --global-auto-download); default for new subscriptions: {_on(st.auto_download_default)}'
+ (' (set by this import)' if st.auto_download_default_changed else ''),
f'Episodes: {st.stateful} with a state: {st.by_guid} matched by guid, {st.by_url} by URL, '
f'{st.episodes_created} created, {st.no_audio} skipped (no audio URL), {st.in_skipped_feeds} in skipped feeds',
f' {st.stateless_matched} without a state matched to diora episodes (not new in AntennaPod)',
f'Progress: {st.progress_new} new, {st.progress_updated} updated, '
f'{st.progress_unchanged} unchanged, {st.progress_kept} kept (diora newer); '
f'{st.left_inbox} taken out of the inbox (not new in AntennaPod)',
f'Favourites: {st.favorites_backup} in backup; {st.favorites_before} before, {st.favorites_after} after',
f'Queue: {st.queue_backup} in backup ({st.queue_missing} not found); {st.queue_before} before, '
f'{st.queue_after} after ({st.queue_appended} diora entries after AntennaPod\'s'
+ (f', {st.queue_removed} removed (--replace)' if st.queue_removed else '') + ')'
+ (f'; kept sorted by "{st.keep_sorted}"' if st.keep_sorted else ''),
f'Inbox: {st.inbox_before} before, {st.inbox_after} after; out of it: {st.ignored_left_inbox} of podcasts '
f'whose new episodes are ignored (AntennaPod never showed them)'
+ (f', {st.diora_only_left_inbox} of podcasts only diora has' if st.diora_only_left_inbox else ''),
f'Downloads: {st.downloads} on the phone in AntennaPod ({st.downloads_bytes / 1_000_000:.0f} MB) '
+ ('-> offered to the app for download' if st.downloads_offered else '-> not offered (--no-downloads)')
+ (f' ({st.downloads_created} episodes created for them)' if st.downloads_created else '')
+ (f'; {st.downloads_missing} not found (feeds not imported, no audio URL)' if st.downloads_missing else ''),
f'Created feeds ({len(st.created_titles)}):',
*(f' - {t}' for t in st.created_titles),
*([f'Unsubscribed feeds ({len(st.removed_titles)}):'] if st.removed_titles else []),
*(f' - {t}' for t in st.removed_titles),
f'Skipped feeds ({len(st.skipped_feeds)}):',
*(f' - {t} ({u})' for t, u in st.skipped_feeds),
f'Done in {st.seconds:.1f} s.',
]
return lines

213
podcasts/audiobooks.py Normal file
View file

@ -0,0 +1,213 @@
"""Audiobooks: podcast subscriptions marked as one (PodcastFeed.kind), listened to like
a book in the library — chapters in order, one progress, bookmarks (docs/audiobooks.md)."""
import json
import re
from datetime import datetime, timezone as dt_timezone
from django.db.models import Max, Q
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from .models import EpisodeBookmark, EpisodeProgress, PodcastEpisode, PodcastFeed
MAX_NOTE = 1000
_FAR = datetime.max.replace(tzinfo=dt_timezone.utc)
_DIGITS = re.compile(r'(\d+)')
CHAPTER_FIELDS = ('id', 'title', 'audio_url', 'duration_seconds', 'artwork_url', 'pub_date', 'episode_number')
def _natural(title):
"""'Kapitel 2' before 'Kapitel 10'."""
return [int(part) if part.isdigit() else part.casefold() for part in _DIGITS.split(title or '')]
def chapter_key(ep):
number = ep['episode_number']
return (number is None, number or 0, ep['pub_date'] or _FAR, _natural(ep['title']), ep['id'])
def _chapters(user, feed):
"""The feed's episodes in chapter order, with the user's place and played mark."""
eps = sorted(PodcastEpisode.objects.filter(feed=feed).values(*CHAPTER_FIELDS), key=chapter_key)
progress = {
p['episode_id']: p for p in EpisodeProgress.objects.filter(user=user, episode__feed=feed)
.values('episode_id', 'position_seconds', 'played', 'updated_at')
}
out = []
for i, ep in enumerate(eps):
p = progress.get(ep['id']) or {}
out.append({
'episode__id': ep['id'],
'episode__title': ep['title'],
'episode__audio_url': ep['audio_url'],
'episode__duration_seconds': ep['duration_seconds'],
'episode__artwork_url': ep['artwork_url'],
'episode__feed__id': feed.id,
'episode__feed__title': feed.title,
'episode__feed__artwork_url': feed.artwork_url,
'episode__pub_date': ep['pub_date'].isoformat() if ep['pub_date'] else None,
'position_seconds': p.get('position_seconds') or 0,
'played': bool(p.get('played')),
'index': i,
# When it was last listened to (a place or played), for where the book goes on.
'_heard_at': p.get('updated_at') if (p.get('played') or p.get('position_seconds')) else None,
})
return out
def _next(chapters):
"""Where the book goes on: in the chapter listened to last if it isn't finished, else
in the first unfinished one after it (or before it, when the end was reached). Never
listened: the first unfinished one. None: all finished."""
open_ = [ch for ch in chapters if not ch['played']]
if not open_:
return None
heard = [ch for ch in chapters if ch['_heard_at']]
if heard:
last = max(heard, key=lambda ch: ch['_heard_at'])
after = [ch for ch in open_ if ch['index'] >= last['index']]
ch = after[0] if after else open_[0]
else:
ch = open_[0]
return {'episode_id': ch['episode__id'], 'index': ch['index'], 'position_seconds': ch['position_seconds']}
def _book(feed, chapters, last_played_at):
total = heard = done = 0
for ch in chapters:
dur = max(ch['episode__duration_seconds'] or 0, 0)
total += dur
if ch['played']:
done += 1
heard += dur
elif dur:
heard += min(ch['position_seconds'], dur)
nxt = _next(chapters)
return {
'feed_id': feed.id, 'title': feed.title, 'author': feed.author, 'artwork_url': feed.artwork_url,
'folder': feed.folder, 'folders': feed.folders or [],
'chapters': len(chapters), 'done': done, 'total_seconds': total, 'heard_seconds': heard,
'next': nxt, 'last_played_at': last_played_at.isoformat() if last_played_at else None,
}
def _bookmark_json(bm):
return {'id': bm.id, 'episode_id': bm.episode_id, 'position_ms': bm.position_ms, 'note': bm.note,
'created_at': bm.created_at.isoformat()}
def _last_played(user, feed_ids):
"""When each book was last listened to: a place or a played mark, not the rows that
only took its chapters out of "Neu" when it was marked."""
return dict(
EpisodeProgress.objects.filter(user=user, episode__feed_id__in=feed_ids)
.filter(Q(position_seconds__gt=0) | Q(played=True))
.values('episode__feed_id').annotate(at=Max('updated_at')).values_list('episode__feed_id', 'at')
)
@csrf_exempt
@require_http_methods(['GET'])
def audiobook_list(request):
"""The user's audiobooks with their progress, the last listened first."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
feeds = list(PodcastFeed.objects.filter(user=request.user, kind=PodcastFeed.AUDIOBOOK))
last = _last_played(request.user, [f.id for f in feeds])
books = [_book(f, _chapters(request.user, f), last.get(f.id)) for f in feeds]
books.sort(key=lambda b: b['title'].casefold())
books.sort(key=lambda b: b['last_played_at'] or '', reverse=True) # never listened: last
return JsonResponse({'books': books})
@csrf_exempt
@require_http_methods(['GET'])
def audiobook_detail(request, feed_id):
"""One audiobook: its progress, its chapters (shaped like queue entries, for the
player) and its bookmarks in chapter order."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
feed = PodcastFeed.objects.filter(pk=feed_id, user=request.user, kind=PodcastFeed.AUDIOBOOK).first()
if feed is None:
return JsonResponse({'error': 'not found'}, status=404)
chapters = _chapters(request.user, feed)
order = {ch['episode__id']: ch['index'] for ch in chapters}
marks = sorted(EpisodeBookmark.objects.filter(user=request.user, episode__feed=feed),
key=lambda bm: (order.get(bm.episode_id, 0), bm.position_ms, bm.id))
book = _book(feed, chapters, _last_played(request.user, [feed.id]).get(feed.id))
for ch in chapters:
del ch['_heard_at']
return JsonResponse({
'book': book,
'chapters': chapters,
'bookmarks': [_bookmark_json(bm) for bm in marks],
})
def _body(request):
try:
body = json.loads(request.body or b'{}')
except (json.JSONDecodeError, ValueError):
return None
return body if isinstance(body, dict) else None
def _note(value):
if value is None:
return ''
if not isinstance(value, str):
raise ValueError
return value.strip()[:MAX_NOTE]
@csrf_exempt
@require_http_methods(['POST'])
def bookmark_add(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
body = _body(request)
if body is None:
return JsonResponse({'error': 'invalid JSON'}, status=400)
episode = PodcastEpisode.objects.filter(pk=body.get('episode_id'), feed__user=request.user).first() \
if isinstance(body.get('episode_id'), int) else None
if episode is None:
return JsonResponse({'error': 'not found'}, status=404)
pos = body.get('position_ms')
if not isinstance(pos, (int, float)) or isinstance(pos, bool) or not 0 <= pos < 2 ** 31:
return JsonResponse({'error': 'invalid position_ms'}, status=400)
try:
note = _note(body.get('note'))
except ValueError:
return JsonResponse({'error': 'invalid note'}, status=400)
bm = EpisodeBookmark.objects.create(user=request.user, episode=episode, position_ms=int(pos), note=note)
return JsonResponse({'ok': True, 'bookmark': _bookmark_json(bm)})
@csrf_exempt
@require_http_methods(['POST'])
def bookmark_update(request, pk):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
bm = EpisodeBookmark.objects.filter(pk=pk, user=request.user).first()
body = _body(request)
if bm is None:
return JsonResponse({'error': 'not found'}, status=404)
if body is None:
return JsonResponse({'error': 'invalid JSON'}, status=400)
try:
bm.note = _note(body.get('note'))
except ValueError:
return JsonResponse({'error': 'invalid note'}, status=400)
bm.save(update_fields=['note'])
return JsonResponse({'ok': True, 'bookmark': _bookmark_json(bm)})
@csrf_exempt
@require_http_methods(['POST'])
def bookmark_delete(request, pk):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
EpisodeBookmark.objects.filter(pk=pk, user=request.user).delete()
return JsonResponse({'ok': True})

244
podcasts/chapters.py Normal file
View file

@ -0,0 +1,244 @@
"""Podcast chapters: Podlove Simple Chapters from the feed, Podcasting 2.0
JSON chapters fetched from the URL a feed points to, and the CHAP frames of
the ID3v2 tag at the start of the audio file.
Both end up in one shape, a list sorted by start time:
[{"start": 62.5, "title": "…", "url": "https://…", "img": "https://…"}]
url and img only when present.
The JSON file and the audio file live on servers the feed's author chose, so
fetching them is an SSRF surface like the WebDAV import, and it reuses that
module's guards: public addresses only (checked before connecting and again
on the connected socket), redirects followed by hand with each hop
re-checked, and a size cap. Transcripts (transcripts.py) are fetched the same
way, through `fetch_bytes`.
"""
import json
import re
import time
from urllib.parse import urljoin
import requests
from accounts.webdav import WebDAVError, WebDAVInputError, _assert_peer_is_safe, _read_capped, assert_safe_url
from . import id3
MAX_CHAPTERS = 500
MAX_TITLE = 500
MAX_URL = 1000
CHAPTERS_TIMEOUT = 10 # seconds
CHAPTERS_MAX_BYTES = 1024 * 1024
CHAPTERS_MAX_REDIRECTS = 3
ID3_MAX_BYTES = 4 * 1024 * 1024 # of the audio file's start; a larger tag is read this far
ID3_DEADLINE = 30 # seconds for reading it, however slowly it trickles in
ID3_CHUNK = 64 * 1024
_SCHEMES = ('http://', 'https://')
def _clean_url(value):
value = (value or '').strip() if isinstance(value, str) else ''
if value.lower().startswith(_SCHEMES) and len(value) <= MAX_URL:
return value
return ''
def _chapter(start, title, url='', img=''):
"""One normalised chapter, or None if the start isn't a usable time."""
try:
start = float(start)
except (TypeError, ValueError):
return None
if start != start or start < 0 or start == float('inf'): # NaN, negative, inf
return None
ch = {'start': round(start, 3), 'title': ' '.join(str(title or '').split())[:MAX_TITLE]}
url, img = _clean_url(url), _clean_url(img)
if url:
ch['url'] = url
if img:
ch['img'] = img
return ch
def _finish(chapters):
chapters = [c for c in chapters if c is not None]
chapters.sort(key=lambda c: c['start'])
return chapters[:MAX_CHAPTERS]
# ---------------------------------------------------------------------------
# Podlove Simple Chapters (inline in the feed)
# ---------------------------------------------------------------------------
# "1:02:03.500", "01:02:03", "02:03", "123.5" (normal play time, as PSC uses)
_NPT = re.compile(r'^\s*(?:(?:(\d+):)?(\d{1,2}):)?(\d+(?:\.\d+)?)\s*$')
def parse_npt(value):
"""Seconds (float) from a PSC start time, or None."""
m = _NPT.match(str(value or ''))
if not m:
return None
hours, minutes, seconds = m.groups()
return int(hours or 0) * 3600 + int(minutes or 0) * 60 + float(seconds)
def from_psc(entry):
"""Chapters from feedparser's entry["psc_chapters"], [] if there are none."""
raw = (entry.get('psc_chapters') or {}).get('chapters') or []
return _finish(
_chapter(parse_npt(c.get('start')), c.get('title'), c.get('href'), c.get('image'))
for c in raw
)
# ---------------------------------------------------------------------------
# Podcasting 2.0 JSON chapters (fetched)
# ---------------------------------------------------------------------------
def from_json(data):
"""Normalise a JSON chapters document (https://github.com/Podcastindex-org/
podcast-namespace/blob/main/chapters/jsonChapters.md)."""
raw = data.get('chapters') if isinstance(data, dict) else None
if not isinstance(raw, list):
raise ValueError('no chapters list')
chapters = []
for c in raw:
if not isinstance(c, dict):
continue
# toc: false = "don't show this one to the user" (e.g. art changes).
if c.get('toc') is False:
continue
chapters.append(_chapter(c.get('startTime'), c.get('title'), c.get('url'), c.get('img')))
return _finish(chapters)
class ChaptersUnavailable(Exception):
"""The chapters can't be loaded right now; worth trying again later."""
class ChaptersInvalid(Exception):
"""The chapters URL is unusable for good (blocked, missing, not chapters)."""
# Generic names for the two kinds of failure, for other fetches (transcripts).
FetchUnavailable = ChaptersUnavailable
FetchInvalid = ChaptersInvalid
def _open(url, headers):
"""GET `url` with every redirect hop checked; the streamed response of
the final hop, which the caller has to close.
Raises ChaptersInvalid for permanent problems and ChaptersUnavailable for
ones that may go away (network trouble, server errors).
"""
for _ in range(CHAPTERS_MAX_REDIRECTS + 1):
try:
assert_safe_url(url)
except WebDAVInputError as e:
raise ChaptersInvalid(str(e))
try:
r = requests.get(url, timeout=CHAPTERS_TIMEOUT, allow_redirects=False, stream=True, headers=headers)
except requests.RequestException as e:
raise ChaptersUnavailable(e.__class__.__name__)
try:
_assert_peer_is_safe(r)
if r.status_code in (301, 302, 303, 307, 308):
location = r.headers.get('Location')
if not location:
raise ChaptersInvalid('redirect without a location')
url = urljoin(url, location)
r.close()
continue
if r.status_code >= 500 or r.status_code == 429:
raise ChaptersUnavailable(f'HTTP {r.status_code}')
if r.status_code >= 400:
raise ChaptersInvalid(f'HTTP {r.status_code}')
except WebDAVInputError as e:
r.close()
raise ChaptersInvalid(str(e))
except Exception:
r.close()
raise
return r
raise ChaptersInvalid('too many redirects')
def fetch_bytes(url, max_bytes, accept='*/*', agent='diora podcast chapters'):
"""The body at `url`, at most `max_bytes` of it (more is ChaptersInvalid)."""
r = _open(url, {'User-Agent': agent, 'Accept': accept})
try:
return _read_capped(r, max_bytes, lambda: ChaptersInvalid('too large'))
except WebDAVError as e: # transfer broke off in _read_capped
raise ChaptersUnavailable(str(e))
finally:
r.close()
def fetch_json(url):
"""Download and parse the JSON at `url`."""
body = fetch_bytes(url, CHAPTERS_MAX_BYTES, accept='application/json')
try:
return json.loads(body)
except ValueError:
raise ChaptersInvalid('not JSON')
def fetch(url):
"""Normalised chapters from a Podcasting 2.0 chapters URL."""
try:
return from_json(fetch_json(url))
except ValueError:
raise ChaptersInvalid('not a chapters document')
# ---------------------------------------------------------------------------
# ID3v2 chapters (CHAP frames at the start of the audio file)
# ---------------------------------------------------------------------------
def read_id3_tag(url):
"""The ID3v2 tag at the start of the file at `url` (at most ID3_MAX_BYTES
of it), or b'' if the file doesn't start with one.
Asks for just that part with a Range request. Servers that ignore Range
and send the whole file are read only as far as needed, then dropped.
"""
r = _open(url, {
'User-Agent': 'diora podcast chapters',
'Range': f'bytes=0-{ID3_MAX_BYTES - 1}',
'Accept-Encoding': 'identity',
})
data = bytearray()
need = None # bytes of the tag, known once its header is in
deadline = time.monotonic() + ID3_DEADLINE
try:
for chunk in r.iter_content(chunk_size=ID3_CHUNK):
data += chunk
if need is None and len(data) >= id3.HEADER_SIZE:
size = id3.tag_size(bytes(data[:id3.HEADER_SIZE]))
if size is None:
return b''
need = min(size, ID3_MAX_BYTES)
if need is not None and len(data) >= need:
break
if time.monotonic() > deadline:
raise ChaptersUnavailable('too slow')
except requests.RequestException as e:
raise ChaptersUnavailable(e.__class__.__name__)
finally:
r.close()
if need is None:
return b'' # shorter than a tag header
return bytes(data[:need]) # may be less if the file ended early
def fetch_id3(url):
"""Normalised chapters from the ID3v2 tag of the audio file at `url`;
[] if it has none."""
return _finish(
_chapter(start_ms / 1000, title, link)
for start_ms, _end_ms, title, link in id3.chapters(read_id3_tag(url))
)

530
podcasts/discover.py Normal file
View file

@ -0,0 +1,530 @@
"""Stöbern (docs/podcasts.md): finding episodes in the user's own podcasts — search in titles
and shownotes, similar episodes, people, topics, forgotten ones — and one controlled step out:
where people from them are guests elsewhere (Apple's episode search, fyyd), and podcasts like
the most listened ones (fyyd's recommendations). An episode from outside goes onto the
Merkliste; its podcast becomes an "Auf Abruf" subscription.
The full-text index is SQLite FTS5, contentless (only the index, no second copy of 100 MB of
shownotes): podcasts_episode_fts, rowid = the episode's id, filled by index_new() after every
refresh (new episodes have higher ids). Episodes deleted since stay in the index; every query
joins the episodes, so they never show.
"""
import html
import logging
import math
import random
import re
import unicodedata
import urllib.parse
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
import requests
from django.core.cache import cache
from django.db import connection
from django.db.models import Q
from django.http import JsonResponse
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from . import people as people_mod
from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastSettings
log = logging.getLogger(__name__)
FTS = 'podcasts_episode_fts'
VOCAB = 'podcasts_episode_fts_vocab'
BODY_CHARS = 6000 # of the shownotes, per episode
INDEX_BATCH = 2000
SIMILAR_TERMS = 10
HTTP_TIMEOUT = 6
USER_AGENT = 'diora podcast discovery'
CREATE_SQL = [
f"CREATE VIRTUAL TABLE IF NOT EXISTS {FTS} USING fts5(title, body, content='', contentless_delete=1, "
f"tokenize='unicode61 remove_diacritics 2')",
f"CREATE VIRTUAL TABLE IF NOT EXISTS {VOCAB} USING fts5vocab({FTS}, 'row')",
]
DROP_SQL = [f'DROP TABLE IF EXISTS {VOCAB}', f'DROP TABLE IF EXISTS {FTS}']
def _plain(text):
text = re.sub(r'<[^>]+>', ' ', text or '')
return re.sub(r'\s+', ' ', html.unescape(text)).strip()
def _fold(word):
"""A word as the index keeps it: lower case, without diacritics."""
w = unicodedata.normalize('NFKD', word.lower())
return ''.join(c for c in w if not unicodedata.combining(c))
_WORD = re.compile(r'\w+', re.UNICODE)
def _words(text):
return [_fold(w) for w in _WORD.findall(text or '') if not w.isdigit()]
def _quoted(term):
return '"' + term.replace('"', '""') + '"'
# ---- the index ------------------------------------------------------------------------------------
def index_new(limit=None):
"""Index the episodes newer than the newest one indexed. Returns how many."""
with connection.cursor() as cur:
cur.execute(f'SELECT max(rowid) FROM {FTS}')
last = cur.fetchone()[0] or 0
done = 0
while True:
rows = list(PodcastEpisode.objects.filter(id__gt=last).order_by('id')
.values_list('id', 'title', 'description')[:INDEX_BATCH])
if not rows:
return done
with connection.cursor() as cur:
cur.executemany(f'INSERT OR REPLACE INTO {FTS}(rowid, title, body) VALUES (%s, %s, %s)',
[(i, t or '', _plain(d)[:BODY_CHARS]) for i, t, d in rows])
done += len(rows)
last = rows[-1][0]
if limit and done >= limit:
return done
def _user_episode_sql(user, stream=False, other_than_feed=None):
"""SQL and params restricting fts rowids to the user's podcast episodes."""
sql = (f' JOIN podcasts_podcastepisode e ON e.id = {FTS}.rowid'
f' JOIN podcasts_podcastfeed pf ON pf.id = e.feed_id'
f' WHERE pf.user_id = %s AND pf.kind = %s')
params = [user.id, PodcastFeed.PODCAST]
if stream:
sql += ' AND pf.new_episodes != %s'
params.append(PodcastFeed.NEW_IGNORE)
if other_than_feed is not None:
sql += ' AND e.feed_id != %s'
params.append(other_than_feed)
return sql, params
def search_ids(user, q, stream=False, limit=50, offset=0):
"""Episode ids of the user's podcasts matching all words of q (the last one as a prefix,
while typing), titles weighing three times the shownotes, best first."""
words = _words(q)[:12]
if not words:
return []
match = ' '.join(_quoted(w) for w in words[:-1]) + (' ' if len(words) > 1 else '') + _quoted(words[-1]) + '*'
where, params = _user_episode_sql(user, stream)
with connection.cursor() as cur:
cur.execute(f'SELECT {FTS}.rowid FROM {FTS}{where} AND {FTS} MATCH %s '
f'ORDER BY bm25({FTS}, 3.0, 1.0) LIMIT %s OFFSET %s', params + [match, limit, offset])
return [r[0] for r in cur.fetchall()]
def similar_ids(user, episode, limit=20):
"""Episodes of the user's other podcasts most like `episode`: its rarest telling words
(tf·idf over the index) asked for together, best match first."""
tf = Counter()
for w in _words(episode.title):
tf[w] += 3
for w in _words(_plain(episode.description)[:BODY_CHARS]):
tf[w] += 1
terms = [w for w in tf if len(w) > 2]
if not terms:
return []
with connection.cursor() as cur:
cur.execute(f'SELECT count(*) FROM podcasts_podcastepisode')
n = cur.fetchone()[0] or 1
df = {}
for i in range(0, len(terms), 500):
chunk = terms[i:i + 500]
cur.execute(f'SELECT term, doc FROM {VOCAB} WHERE term IN ({",".join(["%s"] * len(chunk))})', chunk)
df.update(dict(cur.fetchall()))
scored = []
for w, c in tf.items():
d = df.get(w, 0)
if d < 2 or d > max(n * 0.05, 10): # only here, or everywhere: says nothing
continue
scored.append((c * math.log(n / d), w))
scored.sort(reverse=True)
best = [w for _, w in scored[:SIMILAR_TERMS]]
if not best:
return []
where, params = _user_episode_sql(user, other_than_feed=episode.feed_id)
with connection.cursor() as cur:
cur.execute(f'SELECT {FTS}.rowid FROM {FTS}{where} AND {FTS} MATCH %s '
f'ORDER BY bm25({FTS}, 3.0, 1.0) LIMIT %s',
params + [' OR '.join(_quoted(w) for w in best), limit])
return [r[0] for r in cur.fetchall()]
# ---- helpers ---------------------------------------------------------------------------------------
def _rows(user, ids):
"""The episodes as the episode lists give them, in the order of ids."""
from .views import _episode_list_data
if not ids:
return []
eps, _ = _episode_list_data(user, ids=ids, limit=len(ids))
by_id = {e['id']: e for e in eps}
return [by_id[i] for i in ids if i in by_id]
def _heard_by_feed(user):
"""{feed_id: episodes heard in the last 12 months} (as heard_365 in state/)."""
from .views import HEARD_MIN_SECONDS
since = timezone.now() - timedelta(days=365)
out = Counter()
for fid in (EpisodeProgress.objects.filter(user=user, updated_at__gte=since)
.filter(Q(played=True) | Q(position_seconds__gt=HEARD_MIN_SECONDS))
.values_list('episode__feed_id', flat=True)):
out[fid] += 1
return out
def _norm_url(url):
u = (url or '').strip().lower()
u = re.sub(r'^https?://', '', u)
return u.rstrip('/')
def _get_json(url, **params):
try:
r = requests.get(url, params=params, timeout=HTTP_TIMEOUT, headers={'User-Agent': USER_AGENT})
r.raise_for_status()
return r.json()
except Exception as e: # noqa: BLE001 - a service away is no error for the user
log.info('discover: %s: %s', url, e)
return None
# ---- people ----------------------------------------------------------------------------------------
PEOPLE_TTL = 12 * 3600
def people_of(user):
key = f'podcasts:people:{user.id}'
got = cache.get(key)
if got is None:
rows = list(PodcastEpisode.objects.filter(feed__user=user, feed__kind=PodcastFeed.PODCAST)
.values_list('id', 'feed_id', 'title'))
got = people_mod.people(rows, _heard_by_feed(user))[:200]
cache.set(key, got, PEOPLE_TTL)
return got
def person_ids(user, name, limit=200):
return list(PodcastEpisode.objects.filter(feed__user=user, feed__kind=PodcastFeed.PODCAST, title__icontains=name)
.order_by('-pub_date', '-id').values_list('id', flat=True)[:limit])
def elsewhere(user, name):
"""Episodes with `name` in podcasts the user doesn't have: Apple's episode search and fyyd's,
those that name the person in their title. [{source, podcast, feed_url, title, audio_url,
guid, pub_date, duration_seconds, artwork_url, link}]"""
key = f'podcasts:elsewhere:{user.id}:' + re.sub(r'\W+', '_', _fold(name))
got = cache.get(key)
if got is not None:
return got
mine = {_norm_url(u) for u in PodcastFeed.objects.filter(user=user).values_list('rss_url', flat=True)}
mine_titles = {t.casefold() for t in PodcastFeed.objects.filter(user=user).values_list('title', flat=True)}
folded = _fold(name)
out, seen = [], set()
def take(item):
if folded not in _fold(item['title']) or not item['feed_url'] or not item['audio_url']:
return
if _norm_url(item['feed_url']) in mine or item['podcast'].casefold() in mine_titles:
return
k = (_fold(item['title']), _fold(item['podcast']))
if k in seen:
return
seen.add(k)
out.append(item)
apple = _get_json('https://itunes.apple.com/search', term=name, media='podcast', entity='podcastEpisode',
limit=50, country='DE') or {}
for r in apple.get('results', []):
take({'source': 'apple', 'podcast': r.get('collectionName', ''), 'feed_url': r.get('feedUrl', ''),
'title': r.get('trackName', ''), 'audio_url': r.get('episodeUrl', ''), 'guid': r.get('episodeGuid', ''),
'pub_date': r.get('releaseDate'), 'duration_seconds': int((r.get('trackTimeMillis') or 0) / 1000),
'artwork_url': r.get('artworkUrl600') or r.get('artworkUrl160') or '',
'link': r.get('trackViewUrl', '')})
fy = _get_json('https://api.fyyd.de/0.2/search/episode', term=f'"{name}"', count=50) or {}
eps = [e for e in (fy.get('data') or []) if isinstance(e, dict) and folded in _fold(e.get('title', ''))]
feeds = {}
for pid in list(dict.fromkeys(e.get('podcast_id') for e in eps))[:12]:
p = _get_json('https://api.fyyd.de/0.2/podcast', podcast_id=pid) or {}
if isinstance(p.get('data'), dict):
feeds[pid] = p['data']
for e in eps:
p = feeds.get(e.get('podcast_id'))
if not p:
continue
take({'source': 'fyyd', 'podcast': p.get('title', ''), 'feed_url': p.get('xmlURL', ''),
'title': html.unescape(e.get('title', '')), 'audio_url': e.get('enclosure', ''), 'guid': e.get('guid', ''),
'pub_date': e.get('pubdate'), 'duration_seconds': int(e.get('duration') or 0),
'artwork_url': e.get('imgURL') or p.get('imgURL') or '', 'link': e.get('url_fyyd', '')})
out.sort(key=lambda i: i['pub_date'] or '', reverse=True)
cache.set(key, out, 6 * 3600)
return out
# ---- topics, forgotten, recommendations ------------------------------------------------------------
def topics_of(user, days=7):
"""Topics of the Strom's last days (feeds.textproc.topics, as for the news):
[{'label', 'ids'}], the largest first."""
from feeds import textproc
since = timezone.now() - timedelta(days=days)
# Titles only: the shownotes bring every podcast's furniture ("Link", "Abo", "Mediathek").
eps = list(PodcastEpisode.objects.filter(feed__user=user, feed__kind=PodcastFeed.PODCAST, pub_date__gte=since)
.exclude(feed__new_episodes=PodcastFeed.NEW_IGNORE)
.values_list('id', 'title', 'feed__title'))
stories = [(i, t, {ft}) for i, t, ft in eps]
return textproc.topics(stories, max_topics=10)
FORGOTTEN_DAYS = 60
def forgotten(user):
"""Unheard older episodes of the podcasts the user listens to most, and of the rare ones —
a handful each, the same all day."""
from .views import _intervals
rnd = random.Random(f'{user.id}:{timezone.localdate()}')
heard = _heard_by_feed(user)
started = set(EpisodeProgress.objects.filter(user=user).filter(Q(played=True) | Q(position_seconds__gt=0))
.values_list('episode_id', flat=True))
before = timezone.now() - timedelta(days=FORGOTTEN_DAYS)
def pick(feed_ids, n):
ids = [i for i in PodcastEpisode.objects.filter(feed_id__in=feed_ids, pub_date__lt=before)
.values_list('id', flat=True) if i not in started]
return rnd.sample(ids, min(n, len(ids)))
mine = list(PodcastFeed.objects.filter(user=user, kind=PodcastFeed.PODCAST)
.exclude(new_episodes=PodcastFeed.NEW_IGNORE).values_list('id', flat=True))
loved = [f for f in mine if heard.get(f, 0) >= 3]
intervals = _intervals(user)
rare = [f for f in mine if (intervals.get(f) or 0) >= 30]
return {'loved': _rows(user, pick(loved, 15)), 'rare': _rows(user, pick(rare, 10))}
RECOMMEND_FROM = 10 # the most listened podcasts asked about
RECOMMEND_TTL = 24 * 3600
def _fyyd_id(feed):
got = _get_json('https://api.fyyd.de/0.2/search/podcast', title=feed['title'], count=5) or {}
want = _norm_url(feed['rss_url'])
for p in got.get('data') or []:
if isinstance(p, dict) and (_norm_url(p.get('xmlURL')) == want or (p.get('title') or '').casefold() == feed['title'].casefold()):
return p.get('id')
return None
def recommend(user):
"""Podcasts like the user's most listened (fyyd's recommendations), the ones recommended for
several of them first; none the user has. [{title, feed_url, artwork_url, description, link,
because: [titles], count}]"""
key = f'podcasts:recommend:{user.id}'
got = cache.get(key)
if got is not None:
return got
heard = _heard_by_feed(user)
feeds = list(PodcastFeed.objects.filter(user=user, kind=PodcastFeed.PODCAST).values('id', 'title', 'rss_url'))
top = sorted((f for f in feeds if heard.get(f['id'])), key=lambda f: -heard[f['id']])[:RECOMMEND_FROM]
mine = {_norm_url(f['rss_url']) for f in feeds}
mine_titles = {f['title'].casefold() for f in feeds}
def ask(feed):
pid = _fyyd_id(feed)
if not pid:
return feed, []
r = _get_json('https://api.fyyd.de/0.2/podcast/recommend', podcast_id=pid, count=12) or {}
return feed, [p for p in r.get('data') or [] if isinstance(p, dict)]
found = {}
with ThreadPoolExecutor(max_workers=5) as pool:
for feed, recs in pool.map(ask, top):
for p in recs:
url = p.get('xmlURL') or ''
if not url or _norm_url(url) in mine or (p.get('title') or '').casefold() in mine_titles:
continue
k = _norm_url(url)
item = found.setdefault(k, {'title': p.get('title', ''), 'feed_url': url, 'artwork_url': p.get('imgURL') or '',
'description': _plain(p.get('description'))[:300],
'link': p.get('htmlURL') or '', 'because': [], 'count': 0})
if feed['title'] not in item['because']:
item['because'].append(feed['title'])
item['count'] += 1
out = sorted(found.values(), key=lambda i: (-i['count'], i['title'].casefold()))[:30]
cache.set(key, out, RECOMMEND_TTL)
return out
# ---- an episode from outside -------------------------------------------------------------------------
def take(user, item):
"""An episode from elsewhere onto the Merkliste: its podcast becomes an "Auf Abruf"
subscription (or is one already), the episode is found in it — or, when the feed no longer
lists it, made from what the search said. Returns (episode, created_feed)."""
from .views import _later_add, _refresh_feed
url = (item.get('feed_url') or '').strip()
feed = PodcastFeed.objects.filter(user=user, rss_url=url).first()
created = False
if feed is None:
feed = PodcastFeed.objects.create(
user=user, rss_url=url, title=(item.get('podcast') or url)[:300], artwork_url=(item.get('artwork_url') or '')[:1000],
**{**PodcastSettings.new_feed_defaults(user), 'new_episodes': PodcastFeed.NEW_IGNORE})
created = True
try:
_refresh_feed(feed)
except Exception as e: # noqa: BLE001 - the episode is made from the search below
log.info('discover: %s: %s', url, e)
eps = feed.episodes.all()
ep = None
if item.get('guid'):
ep = eps.filter(guid=item['guid']).first()
if ep is None and item.get('audio_url'):
ep = eps.filter(audio_url=item['audio_url']).first()
if ep is None and item.get('title'):
ep = eps.filter(title__iexact=item['title']).first()
if ep is None:
when = parse_datetime(item.get('pub_date') or '') if item.get('pub_date') else None
ep = PodcastEpisode.objects.create(
feed=feed, guid=(item.get('guid') or item.get('audio_url') or item.get('title'))[:1000],
title=(item.get('title') or '')[:500], audio_url=item.get('audio_url', '')[:1000],
duration_seconds=int(item.get('duration_seconds') or 0), pub_date=when,
artwork_url=(item.get('artwork_url') or '')[:1000])
_later_add(user, [ep.id])
try:
index_new()
except Exception: # noqa: BLE001
pass
return ep, created
# ---- views ---------------------------------------------------------------------------------------------
def _auth(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
return None
def _int(request, name, default, low, high):
try:
return min(max(int(request.GET.get(name, default)), low), high)
except ValueError:
return default
@require_http_methods(['GET'])
def search_view(request):
"""?q=…&stream=1&offset=&limit=: episodes whose title or shownotes have every word."""
err = _auth(request)
if err:
return err
ids = search_ids(request.user, request.GET.get('q', ''), stream=request.GET.get('stream') == '1',
limit=_int(request, 'limit', 50, 1, 200), offset=_int(request, 'offset', 0, 0, 10 ** 6))
return JsonResponse({'episodes': _rows(request.user, ids)})
@require_http_methods(['GET'])
def similar_view(request, pk):
err = _auth(request)
if err:
return err
ep = PodcastEpisode.objects.filter(pk=pk, feed__user=request.user).first()
if ep is None:
return JsonResponse({'error': 'not found'}, status=404)
return JsonResponse({'episodes': _rows(request.user, similar_ids(request.user, ep, _int(request, 'limit', 20, 1, 50)))})
@require_http_methods(['GET'])
def people_view(request):
"""The people in the user's podcasts: [{name, episodes, feeds, score}], the most relevant first."""
err = _auth(request)
if err:
return err
return JsonResponse({'people': people_of(request.user)[:_int(request, 'limit', 80, 1, 200)]})
@require_http_methods(['GET'])
def person_view(request):
"""?name=: the person's episodes in the user's podcasts, newest first."""
err = _auth(request)
if err:
return err
name = request.GET.get('name', '').strip()
if len(name) < 3:
return JsonResponse({'error': 'name required'}, status=400)
return JsonResponse({'name': name, 'episodes': _rows(request.user, person_ids(request.user, name))})
@require_http_methods(['GET'])
def elsewhere_view(request):
"""?name=: episodes with the person in podcasts the user doesn't have."""
err = _auth(request)
if err:
return err
name = request.GET.get('name', '').strip()
if len(name) < 3:
return JsonResponse({'error': 'name required'}, status=400)
return JsonResponse({'name': name, 'episodes': elsewhere(request.user, name)})
@require_http_methods(['GET'])
def topics_view(request):
"""?days=7: topics of the Strom's last days: [{label, ids}], plus their episodes."""
err = _auth(request)
if err:
return err
days = _int(request, 'days', 7, 1, 30)
key = f'podcasts:topics:{request.user.id}:{days}'
got = cache.get(key)
if got is None:
got = topics_of(request.user, days)
cache.set(key, got, 3600)
return JsonResponse({'topics': got})
@require_http_methods(['GET'])
def forgotten_view(request):
err = _auth(request)
if err:
return err
return JsonResponse(forgotten(request.user))
@require_http_methods(['GET'])
def recommend_view(request):
err = _auth(request)
if err:
return err
return JsonResponse({'podcasts': recommend(request.user)})
@csrf_exempt
@require_http_methods(['POST'])
def take_view(request):
"""{podcast, feed_url, title, audio_url, guid, pub_date, duration_seconds, artwork_url} (an
item of elsewhere/): onto the Merkliste; returns {episode_id, feed_id, subscribed}."""
import json
err = _auth(request)
if err:
return err
try:
item = json.loads(request.body or b'{}')
except (json.JSONDecodeError, ValueError):
return JsonResponse({'error': 'invalid JSON'}, status=400)
if not isinstance(item, dict) or not re.match(r'^https?://', item.get('feed_url') or '') or not item.get('audio_url'):
return JsonResponse({'error': 'feed_url and audio_url required'}, status=400)
ep, created = take(request.user, item)
return JsonResponse({'ok': True, 'episode_id': ep.id, 'feed_id': ep.feed_id, 'subscribed': created})

43
podcasts/feedext.py Normal file
View file

@ -0,0 +1,43 @@
"""Teach feedparser the Podcasting 2.0 tags we read.
feedparser 6 doesn't know the podcast namespace. Unknown tags still come
through, but only under whatever prefix the feed happened to declare, and an
unknown tag with attributes is stored as a single dict, so a second
<podcast:transcript> overwrites the first. Registering the namespace fixes the
prefix ("podcast_…" for every feed); the handler below keeps every
transcript in a list, entry["podcast_transcripts"].
podcast:chapters is left to feedparser's default handling, which gives
entry["podcast_chapters"] = {"url": …, "type": …} (one per item, as the spec
says).
"""
from feedparser.mixin import _FeedParserMixin as FeedParserMixin
from feedparser.util import FeedParserDict
PODCAST_NAMESPACES = (
'https://podcastindex.org/namespace/1.0',
# The URL the namespace was announced under; some early feeds still use it.
'https://github.com/podcastindex-org/podcast-namespace/blob/main/docs/1.0.md',
)
def _start_podcast_transcript(self, attrs_d):
context = self._get_context()
context.setdefault('podcast_transcripts', []).append(FeedParserDict(attrs_d))
def _end_podcast_transcript(self):
pass
def install():
"""Idempotent; called when podcasts.views is imported."""
for uri in PODCAST_NAMESPACES:
FeedParserMixin.namespaces.setdefault(uri, 'podcast')
# Filled lazily from `namespaces` by the first parser; once it has
# been, later additions have to go in here directly.
if FeedParserMixin._matchnamespaces:
FeedParserMixin._matchnamespaces.setdefault(uri.lower(), 'podcast')
FeedParserMixin._start_podcast_transcript = _start_podcast_transcript
FeedParserMixin._end_podcast_transcript = _end_podcast_transcript

116
podcasts/folders.py Normal file
View file

@ -0,0 +1,116 @@
"""Folders of podcasts and audiobooks (docs/library.md): like tags, any number per
subscription (PodcastFeed.folders), shown as filter chips in their order
(PodcastSettings.folder_order). Renaming onto an existing name merges; deleting
a folder leaves its subscriptions where they are."""
import json
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from .models import PodcastFeed, PodcastSettings
MAX_FOLDERS = 20
MAX_NAME = 100
def clean_name(value):
return ' '.join(str(value or '').split())[:MAX_NAME]
def clean_list(value):
"""A list of folder names: trimmed, without empties and doubles (first spelling wins)."""
if not isinstance(value, list):
raise ValueError
out, seen = [], set()
for v in value:
if not isinstance(v, str):
raise ValueError
name = clean_name(v)
if name and name.casefold() not in seen:
seen.add(name.casefold())
out.append(name)
return out[:MAX_FOLDERS]
def set_folders(feed, names):
"""The feed's folders, and `folder` (the first) for older clients."""
feed.folders = names
feed.folder = names[0] if names else ''
def _body(request):
try:
body = json.loads(request.body or b'{}')
except (json.JSONDecodeError, ValueError):
return None
return body if isinstance(body, dict) else None
def _data(user):
counts = {}
for names in PodcastFeed.objects.filter(user=user).values_list('folders', flat=True):
for n in names or []:
counts[n] = counts.get(n, 0) + 1
order = [n for n in PodcastSettings.for_user(user).folder_order if n in counts]
rest = sorted((n for n in counts if n not in order), key=str.casefold)
return {'order': order + rest, 'folders': [{'name': n, 'count': counts[n]} for n in order + rest]}
@csrf_exempt
@require_http_methods(['GET', 'POST'])
def folders(request):
"""GET {order, folders: [{name, count}]}; POST {order: [names]} sets the chip order."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
if request.method == 'POST':
body = _body(request)
try:
order = clean_list((body or {}).get('order'))
except ValueError:
return JsonResponse({'error': 'order: a list of names'}, status=400)
s = PodcastSettings.for_user(request.user)
s.folder_order = order
s.save(update_fields=['folder_order'])
return JsonResponse(_data(request.user))
def _rewrite(user, fn):
for feed in PodcastFeed.objects.filter(user=user).only('id', 'folders', 'folder'):
names = fn(list(feed.folders or []))
if names != (feed.folders or []):
set_folders(feed, clean_list(names))
feed.save(update_fields=['folders', 'folder'])
@csrf_exempt
@require_http_methods(['POST'])
def rename_folder(request):
"""{old, new}: every subscription in `old` is in `new` instead (merged if it exists)."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
body = _body(request) or {}
old, new = clean_name(body.get('old')), clean_name(body.get('new'))
if not old or not new:
return JsonResponse({'error': 'old and new required'}, status=400)
_rewrite(request.user, lambda names: [new if n == old else n for n in names])
s = PodcastSettings.for_user(request.user)
s.folder_order = clean_list([new if n == old else n for n in s.folder_order])
s.save(update_fields=['folder_order'])
return JsonResponse(_data(request.user))
@csrf_exempt
@require_http_methods(['POST'])
def delete_folder(request):
"""{name}: the folder goes, its subscriptions stay."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
name = clean_name((_body(request) or {}).get('name'))
if not name:
return JsonResponse({'error': 'name required'}, status=400)
_rewrite(request.user, lambda names: [n for n in names if n != name])
s = PodcastSettings.for_user(request.user)
s.folder_order = [n for n in s.folder_order if n != name]
s.save(update_fields=['folder_order'])
return JsonResponse(_data(request.user))

281
podcasts/id3.py Normal file
View file

@ -0,0 +1,281 @@
"""Chapters embedded in the ID3v2 tag at the start of an MP3 file.
Written from the ID3v2.3.0 and ID3v2.4.0 specifications and the ID3v2
Chapter Frame Addendum (CHAP and CTOC frames). Only what chapters need is
read: the tag header, an optional extended header (skipped), and the frames.
Of those, CHAP frames give the chapters, with the title from an embedded TIT2
frame and a link from WXXX or another W*** frame; CTOC frames, when there
are any, give the order (which matters only for chapters starting at the
same time, as callers sort by start). Pictures (APIC) and everything else are
skipped.
Pure parsing, no I/O: the caller hands over the first bytes of the file
(`tag_size` says how many are needed).
"""
import re
import struct
import zlib
HEADER_SIZE = 10
MAX_FRAME_BYTES = 4 * 1024 * 1024 # a decompressed frame can't grow past this
_FRAME_ID = re.compile(rb'[A-Z0-9]{4}\Z')
# Header flags
_UNSYNC = 0x80
_EXTENDED = 0x40
_FOOTER = 0x10
def _synchsafe(raw):
"""Integer from bytes that carry 7 bits each (the high bit is always 0)."""
value = 0
for b in raw:
value = (value << 7) | (b & 0x7F)
return value
def tag_size(header):
"""Bytes the ID3v2 tag at the start of a file takes up, header and footer
included, from the file's first 10 bytes; None if there's no such tag."""
if len(header) < HEADER_SIZE or header[:3] != b'ID3':
return None
major, revision, flags = header[3], header[4], header[5]
if major == 0xFF or revision == 0xFF or any(b & 0x80 for b in header[6:10]):
return None
size = HEADER_SIZE + _synchsafe(header[6:10])
if major >= 4 and flags & _FOOTER:
size += HEADER_SIZE
return size
def _deunsync(data):
"""Undo unsynchronisation: every $FF $00 was an $FF before."""
return data.replace(b'\xff\x00', b'\xff')
# ---------------------------------------------------------------------------
# Frames
# ---------------------------------------------------------------------------
def _plausible_frame_start(buf, at):
"""True if a frame (or padding, or the end of `buf`) can begin at `at`."""
if at == len(buf):
return True
if at > len(buf):
return False
head = buf[at:at + 4]
return bool(_FRAME_ID.match(head)) or not head.strip(b'\x00')
def _v24_frame_size(buf, pos, raw):
"""Size of a v2.4 frame. The spec says synchsafe; some writers (older
iTunes among them) store a plain integer, so where the two differ, take
the one after which the next frame starts."""
plain = int.from_bytes(raw, 'big')
if any(b & 0x80 for b in raw):
return plain # can't be synchsafe
safe = _synchsafe(raw)
if safe == plain or _plausible_frame_start(buf, pos + HEADER_SIZE + safe):
return safe
if _plausible_frame_start(buf, pos + HEADER_SIZE + plain):
return plain
return safe
def _frame_payload(data, fmt, major, unsync_all):
"""The frame's content after its format flags are dealt with, or None if
it can't be read (encrypted, broken compression)."""
if major == 3:
compressed, encrypted = fmt & 0x80, fmt & 0x40
# Added after the frame header, in flag order: decompressed size (4),
# encryption method (1), group id (1).
skip = (4 if compressed else 0) + (1 if encrypted else 0) + (1 if fmt & 0x20 else 0)
unsync = False # v2.3 unsynchronises the whole tag, undone already
else:
compressed, encrypted = fmt & 0x08, fmt & 0x04
# Group id (1), encryption method (1), data length indicator (4).
skip = (1 if fmt & 0x40 else 0) + (1 if encrypted else 0) + (4 if fmt & 0x01 else 0)
unsync = unsync_all or fmt & 0x02
if encrypted:
return None
data = data[skip:]
if unsync:
data = _deunsync(data)
if compressed:
try:
data = zlib.decompressobj().decompress(data, MAX_FRAME_BYTES)
except zlib.error:
return None
return data
def _frames(buf, pos, major, unsync_all=False):
"""(frame id, content) for each complete frame in buf[pos:]."""
while pos + HEADER_SIZE <= len(buf):
frame_id = buf[pos:pos + 4]
if not _FRAME_ID.match(frame_id):
return # padding, or something we can't make sense of
raw_size = buf[pos + 4:pos + 8]
size = _v24_frame_size(buf, pos, raw_size) if major >= 4 else int.from_bytes(raw_size, 'big')
fmt = buf[pos + 9]
start = pos + HEADER_SIZE
if start + size > len(buf):
return # cut off (the caller read only part of a large tag)
payload = _frame_payload(buf[start:start + size], fmt, major, unsync_all)
if payload is not None:
yield frame_id.decode('ascii'), payload
pos = start + size
# ---------------------------------------------------------------------------
# Frame contents
# ---------------------------------------------------------------------------
_CODECS = {0: 'latin-1', 1: 'utf-16', 2: 'utf-16-be', 3: 'utf-8'}
def _decode(raw, encoding):
codec = _CODECS.get(encoding)
if codec is None:
return ''
if encoding in (1, 2) and len(raw) % 2:
raw = raw[:-1]
return raw.decode(codec, errors='replace')
def _text_frame(data):
"""The (first) string of a T*** frame."""
if not data:
return ''
text = _decode(data[1:], data[0])
# v2.4 allows several strings, separated by NUL; the first is the title.
return text.split('\x00')[0].replace('', '').strip()
def _wxxx_frame(data):
"""The URL of a WXXX frame: encoding, description, NUL, URL (latin-1)."""
if not data:
return ''
encoding, rest = data[0], data[1:]
if encoding in (1, 2): # the description ends with a two-byte NUL
i = 0
while i + 1 < len(rest) and not (rest[i] == 0 and rest[i + 1] == 0):
i += 2
url = rest[i + 2:]
else:
i = rest.find(b'\x00')
url = rest[i + 1:] if i >= 0 else b''
return url.split(b'\x00')[0].decode('latin-1').strip()
def _url_frame(data):
"""The URL of any other W*** frame, which is nothing but the URL."""
return data.split(b'\x00')[0].decode('latin-1').strip()
def _chap(data, major):
"""(element id, start ms, end ms, title, url) of a CHAP frame, or None."""
nul = data.find(b'\x00')
if nul < 0 or len(data) < nul + 17:
return None
element_id = data[:nul]
start_ms, end_ms, _start_offset, _end_offset = struct.unpack('>IIII', data[nul + 1:nul + 17])
title = url = ''
# Embedded frames, in the same format as the tag's own. Unsynchronisation
# of the whole tag has already been undone on `data`.
for frame_id, payload in _frames(data, nul + 17, major):
if frame_id == 'TIT2' and not title:
title = _text_frame(payload)
elif frame_id == 'WXXX' and not url:
url = _wxxx_frame(payload)
elif frame_id.startswith('W') and not url:
url = _url_frame(payload)
return element_id, start_ms, end_ms, title, url
def _ctoc(data):
"""(element id, top level?, child element ids) of a CTOC frame, or None."""
nul = data.find(b'\x00')
if nul < 0 or len(data) < nul + 3:
return None
element_id, flags, count = data[:nul], data[nul + 1], data[nul + 2]
children, pos = [], nul + 3
for _ in range(count):
end = data.find(b'\x00', pos)
if end < 0:
break
children.append(data[pos:end])
pos = end + 1
return element_id, bool(flags & 0x02), children
def _toc_order(chaps, tocs):
"""Element ids of the chapters the table of contents lists, in its order
(top-level CTOCs first, nested ones walked depth first); [] if there's no
usable table of contents."""
if not tocs:
return []
nested = {child for _, _, children in tocs for child in children}
roots = [t for t in tocs if t[1]] or [t for t in tocs if t[0] not in nested]
by_id = {t[0]: t for t in tocs}
order, seen = [], set()
def walk(toc):
if toc[0] in seen:
return # a loop
seen.add(toc[0])
for child in toc[2]:
if child in chaps:
if child not in order:
order.append(child)
elif child in by_id:
walk(by_id[child])
for root in roots:
walk(root)
return order
def chapters(data):
"""[(start ms, end ms, title, url)] of the chapters in the ID3v2 tag at
the start of `data`, in table-of-contents order where there is one, else
in tag order.
[] if there is no tag or it has no chapters; a tag cut short yields the
chapters in the part that's there."""
size = tag_size(data)
if size is None:
return []
major, flags = data[3], data[5]
if major not in (3, 4):
return [] # v2.2 has no chapter frames
body = data[HEADER_SIZE:min(len(data), size)]
unsync = bool(flags & _UNSYNC)
if major == 3 and unsync:
body = _deunsync(body)
pos = 0
if flags & _EXTENDED and len(body) >= 4:
if major == 3: # size of what follows the size field
pos = 4 + int.from_bytes(body[:4], 'big')
else: # synchsafe size of the whole extended header
pos = _synchsafe(body[:4])
chaps, order, tocs = {}, [], []
for frame_id, payload in _frames(body, pos, major, unsync_all=(major == 4 and unsync)):
if frame_id == 'CHAP':
chap = _chap(payload, major)
if chap and chap[0] not in chaps:
chaps[chap[0]] = chap
order.append(chap[0])
elif frame_id == 'CTOC':
toc = _ctoc(payload)
if toc:
tocs.append(toc)
# The table of contents' order first; chapters it doesn't list are kept
# (after it) rather than dropped, since writers don't always list all.
toc = _toc_order(chaps, tocs)
listed = set(toc)
order = toc + [e for e in order if e not in listed]
return [chaps[e][1:] for e in order]

68
podcasts/management/commands/import_antennapod.py Normal file
View file

@ -0,0 +1,68 @@
import os
from datetime import datetime, timezone as dt_timezone
from django.contrib.auth.models import User
from django.core.management.base import BaseCommand, CommandError
from django.utils.dateparse import parse_datetime
from podcasts.antennapod import AntennaPodImport, AntennaPodImportError, summary_lines
class Command(BaseCommand):
help = (
"Import an AntennaPod database backup (Settings > Database > Export) into one user's podcasts: "
'subscriptions with their settings, played/new/started episodes, favourites and the queue; '
'the episodes downloaded in AntennaPod are offered to the app for download. '
'Feeds it creates are fetched by the next refresh; running the import once more after that '
'also takes their older episodes that were not new in AntennaPod out of the inbox. '
'Running it twice changes nothing more.'
)
def add_arguments(self, parser):
parser.add_argument('path', help='AntennaPod backup (.db)')
parser.add_argument('--user', required=True, help='diora username')
parser.add_argument('--dry-run', action='store_true',
help='Do everything inside a transaction that is rolled back, print the summary')
parser.add_argument('--global-auto-download', choices=('on', 'off'), default='off',
help="Auto-download for feeds that use AntennaPod's global setting (default off)")
parser.add_argument('--replace', action='store_true',
help="A clean slate: unsubscribe podcasts the backup doesn't have, keep only "
"AntennaPod's queue, AntennaPod wins over later changes in diora")
parser.add_argument('--quiet-diora-only', action='store_true',
help='Podcasts only diora has (not in the backup): their episodes out before the backup '
'leave the inbox (the subscriptions stay)')
parser.add_argument('--no-downloads', action='store_true',
help="Don't offer AntennaPod's downloaded episodes to the app (they stay in AntennaPod)")
parser.add_argument('--backup-date',
help='When the backup was made (ISO date/time, UTC if no zone); default: the file date. '
'diora changes after it win over the backup.')
def handle(self, *args, **options):
path = options['path']
if not os.path.isfile(path):
raise CommandError(f'No such file: {path}')
try:
user = User.objects.get(username=options['user'])
except User.DoesNotExist:
raise CommandError(f"No such user: {options['user']}")
if options['backup_date']:
raw = options['backup_date']
backup_date = parse_datetime(raw) or parse_datetime(f'{raw}T00:00')
if backup_date is None:
raise CommandError(f'Invalid --backup-date: {raw}')
if backup_date.tzinfo is None:
backup_date = backup_date.replace(tzinfo=dt_timezone.utc)
else:
backup_date = datetime.fromtimestamp(os.path.getmtime(path), tz=dt_timezone.utc)
global_auto_download = options['global_auto_download'] == 'on'
try:
stats = AntennaPodImport(user, path, backup_date, global_auto_download,
offer_downloads=not options['no_downloads'],
quiet_diora_only=options['quiet_diora_only'],
replace=options['replace']).run(dry_run=options['dry_run'])
except AntennaPodImportError as e:
raise CommandError(str(e))
for line in summary_lines(stats, user.username, path, options['dry_run'], global_auto_download):
self.stdout.write(line)

10
podcasts/management/commands/index_episodes.py Normal file
View file

@ -0,0 +1,10 @@
from django.core.management.base import BaseCommand
from podcasts.discover import index_new
class Command(BaseCommand):
help = 'Put the episodes not yet in the full-text index into it (podcasts/discover.py)'
def handle(self, *args, **options):
self.stdout.write(f'{index_new()} episodes indexed')

32
podcasts/management/commands/refresh_feeds.py
View file

@ -1,23 +1,36 @@
import sys
from django.core.management.base import BaseCommand
from django.utils import timezone
from podcasts.models import PodcastFeed
from podcasts.views import _refresh_feed
from podcasts.discover import index_new
from podcasts.views import _in_backoff_q, _refresh_feeds, refresh_due
class Command(BaseCommand):
help = 'Refresh podcast feeds (fetch new episodes from RSS)'
def add_arguments(self, parser):
parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID')
parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID (even when backing off)')
parser.add_argument('--limit', type=int, default=0, help='Max number of feeds to refresh')
parser.add_argument(
'--full', action='store_true',
help='Download every feed in full (ignore ETag/Last-Modified) and take in episodes skipped before '
'(e.g. video ones); those out before the last refresh go into the list, not into "Neu"',
)
def handle(self, *args, **options):
qs = PodcastFeed.objects.all().order_by('last_refreshed_at')
if options['feed_id']:
qs = qs.filter(pk=options['feed_id'])
else:
# A feed failing 3+ times in a row is tried every 2, 4, 8, 16,
# at most 24 hours instead of every hour.
qs = qs.exclude(_in_backoff_q(timezone.now()))
# Each feed as often as its interval says (docs/podcasts.md): by its rhythm,
# or hourly, daily, weekly. --feed-id and --full take every one.
if not options['full']:
qs = qs.filter(pk__in=refresh_due(list(qs.select_related('user'))))
if options['limit']:
qs = qs[: options['limit']]
@ -26,9 +39,8 @@ class Command(BaseCommand):
self.stdout.write('No feeds to refresh.')
return
for feed in qs:
try:
new_ep = _refresh_feed(feed)
self.stdout.write(f'{feed.title}: +{new_ep} episodes')
except Exception as e:
self.stderr.write(f'ERROR refreshing "{feed.title}" ({feed.rss_url}): {e}')
# Parallel downloads, one per distinct feed URL (see _refresh_feeds).
refreshed, failed, new = _refresh_feeds(qs, full=options['full'])
# New episodes into the full-text index (Stöbern, podcasts/discover.py).
indexed = index_new()
self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes, {indexed} indexed')

27
podcasts/migrations/0004_podcastsettings.py Normal file
View file

@ -0,0 +1,27 @@
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0003_episodeprogress_dismissed'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='PodcastSettings',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('enqueue_position', models.CharField(
choices=[('back', 'Ans Ende'), ('front', 'An den Anfang'), ('after_current', 'Nach der laufenden Folge')],
default='back', max_length=20)),
('auto_queue_position', models.CharField(
choices=[('back', 'Ans Ende'), ('front', 'An den Anfang'), ('after_current', 'Nach der laufenden Folge')],
default='front', max_length=20)),
('user', models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE, related_name='podcast_settings',
to=settings.AUTH_USER_MODEL)),
],
),
]

33
podcasts/migrations/0005_episode_chapters_transcripts.py Normal file
View file

@ -0,0 +1,33 @@
# Generated by Django 6.1 on 2026-10-02 05:53
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0004_podcastsettings'),
]
operations = [
migrations.AddField(
model_name='podcastepisode',
name='chapters',
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name='podcastepisode',
name='chapters_fetched_at',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name='podcastepisode',
name='chapters_url',
field=models.URLField(blank=True, max_length=1000),
),
migrations.AddField(
model_name='podcastepisode',
name='transcripts',
field=models.JSONField(blank=True, default=list),
),
]

18
podcasts/migrations/0006_episodeprogress_favorite.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 6.1 on 2026-10-02 12:00
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0005_episode_chapters_transcripts'),
]
operations = [
migrations.AddField(
model_name='episodeprogress',
name='favorite',
field=models.BooleanField(default=False),
),
]

74
podcasts/migrations/0007_feed_settings.py Normal file
View file

@ -0,0 +1,74 @@
from django.db import migrations, models
def auto_queue_to_new_episodes(apps, schema_editor):
"""Feeds that queued their new episodes keep doing so."""
PodcastFeed = apps.get_model('podcasts', 'PodcastFeed')
PodcastFeed.objects.filter(auto_queue=True).update(new_episodes='queue')
def new_episodes_to_auto_queue(apps, schema_editor):
PodcastFeed = apps.get_model('podcasts', 'PodcastFeed')
PodcastFeed.objects.filter(new_episodes='queue').update(auto_queue=True)
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0006_episodeprogress_favorite'),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='exclude_words',
field=models.CharField(blank=True, max_length=500),
),
migrations.AddField(
model_name='podcastfeed',
name='include_words',
field=models.CharField(blank=True, max_length=500),
),
migrations.AddField(
model_name='podcastfeed',
name='min_duration',
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name='podcastfeed',
name='new_episodes',
field=models.CharField(
choices=[('inbox', 'In „Neu“'), ('queue', 'In die Warteschlange'), ('ignore', 'Nicht anzeigen')],
default='inbox', max_length=10),
),
migrations.AddField(
model_name='podcastfeed',
name='playback_speed',
field=models.FloatField(blank=True, null=True),
),
migrations.AddField(
model_name='podcastfeed',
name='skip_intro',
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name='podcastfeed',
name='skip_outro',
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name='podcastfeed',
name='volume',
field=models.FloatField(default=1.0),
),
migrations.AddField(
model_name='podcastsettings',
name='keep_sorted',
field=models.CharField(
blank=True,
choices=[('', 'Aus'), ('new', 'Neueste zuerst'), ('old', 'Älteste zuerst'),
('short', 'Kürzeste zuerst'), ('long', 'Längste zuerst'), ('feed', 'Nach Podcast')],
default='', max_length=10),
),
migrations.RunPython(auto_queue_to_new_episodes, new_episodes_to_auto_queue),
]

38
podcasts/migrations/0008_feed_http_cache.py Normal file
View file

@ -0,0 +1,38 @@
# Generated by Django 6.1 on 2026-10-02 12:00
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0007_feed_settings'),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='http_etag',
field=models.CharField(blank=True, max_length=500),
),
migrations.AddField(
model_name='podcastfeed',
name='http_last_modified',
field=models.CharField(blank=True, max_length=100),
),
migrations.AddField(
model_name='podcastfeed',
name='last_error',
field=models.CharField(blank=True, max_length=500),
),
migrations.AddField(
model_name='podcastfeed',
name='last_error_at',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name='podcastfeed',
name='error_count',
field=models.PositiveIntegerField(default=0),
),
]

25
podcasts/migrations/0009_episode_transcript_cache.py Normal file
View file

@ -0,0 +1,25 @@
# Generated by Django 6.1 on 2026-10-02 07:44
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0008_feed_http_cache'),
]
operations = [
migrations.CreateModel(
name='EpisodeTranscript',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('sources', models.JSONField(default=list)),
('data', models.JSONField(blank=True, null=True)),
('error', models.CharField(blank=True, max_length=20)),
('fetched_at', models.DateTimeField(auto_now=True)),
('episode', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='transcript_cache', to='podcasts.podcastepisode')),
],
),
]

21
podcasts/migrations/0010_feed_auto_download.py Normal file
View file

@ -0,0 +1,21 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0009_episode_transcript_cache'),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='auto_download',
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name='podcastsettings',
name='auto_download_default',
field=models.BooleanField(default=False),
),
]

18
podcasts/migrations/0011_settings_import_downloads.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 6.1 on 2026-10-02 09:51
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0010_feed_auto_download'),
]
operations = [
migrations.AddField(
model_name='podcastsettings',
name='import_downloads',
field=models.JSONField(blank=True, default=list),
),
]

51
podcasts/migrations/0012_player_state.py Normal file
View file

@ -0,0 +1,51 @@
# Generated by Django 6.1 on 2026-10-02 12:50
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0011_settings_import_downloads'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='PlayerState',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('kind', models.CharField(blank=True, choices=[('', 'Nichts'), ('podcast', 'Podcast'), ('radio', 'Radio')], default='', max_length=10)),
('station_url', models.CharField(blank=True, max_length=1000)),
('station_name', models.CharField(blank=True, max_length=300)),
('station_id', models.IntegerField(blank=True, null=True)),
('position_ms', models.PositiveBigIntegerField(default=0)),
('playing', models.BooleanField(default=False)),
('speed', models.FloatField(default=1.0)),
('device_id', models.CharField(blank=True, max_length=64)),
('device_name', models.CharField(blank=True, max_length=100)),
('updated_at', models.DateTimeField(blank=True, null=True)),
('seq', models.BigIntegerField(default=0)),
('command', models.JSONField(blank=True, null=True)),
('online_devices', models.JSONField(blank=True, default=list)),
('episode', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='podcasts.podcastepisode')),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='player_state', to=settings.AUTH_USER_MODEL)),
],
),
migrations.CreateModel(
name='PlayerDevice',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('device_id', models.CharField(max_length=64)),
('device_name', models.CharField(blank=True, max_length=100)),
('kind', models.CharField(blank=True, choices=[('', 'Unbekannt'), ('app', 'App'), ('web', 'Browser')], default='', max_length=10)),
('last_seen', models.DateTimeField()),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='player_devices', to=settings.AUTH_USER_MODEL)),
],
options={
'unique_together': {('user', 'device_id')},
},
),
]

16
podcasts/migrations/0013_feed_notify.py Normal file
View file

@ -0,0 +1,16 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0012_player_state'),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='notify',
field=models.BooleanField(default=False),
),
]

72
podcasts/migrations/0014_playback_routine.py Normal file
View file

@ -0,0 +1,72 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0013_feed_notify'),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='episode_sort',
field=models.CharField(blank=True, max_length=10),
),
migrations.AddField(
model_name='podcastfeed',
name='episode_filter',
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name='podcastfeed',
name='skip_silence',
field=models.BooleanField(blank=True, null=True),
),
migrations.AddField(
model_name='podcastsettings',
name='skip_back',
field=models.PositiveSmallIntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='podcastsettings',
name='skip_forward',
field=models.PositiveSmallIntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='podcastsettings',
name='smart_played',
field=models.PositiveIntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='podcastsettings',
name='speed_presets',
field=models.JSONField(blank=True, null=True),
),
migrations.AddField(
model_name='podcastsettings',
name='keep_skipped',
field=models.BooleanField(default=True),
),
migrations.AddField(
model_name='podcastsettings',
name='delete_dequeue',
field=models.BooleanField(default=True),
),
migrations.AddField(
model_name='podcastsettings',
name='mobile_downloads',
field=models.BooleanField(default=True),
),
migrations.AddField(
model_name='podcastsettings',
name='mobile_streaming',
field=models.CharField(choices=[('allow', 'Erlauben'), ('ask', 'Nachfragen'), ('never', 'Nie')],
default='allow', max_length=10),
),
migrations.AddField(
model_name='podcastsettings',
name='times_by_speed',
field=models.BooleanField(default=False),
),
]

16
podcasts/migrations/0015_episodeprogress_discarded_at.py Normal file
View file

@ -0,0 +1,16 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0014_playback_routine'),
]
operations = [
migrations.AddField(
model_name='episodeprogress',
name='discarded_at',
field=models.DateTimeField(blank=True, null=True),
),
]

23
podcasts/migrations/0016_playerstate_sleep.py Normal file
View file

@ -0,0 +1,23 @@
# Generated by Django 6.1 on 2026-10-02 22:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0015_episodeprogress_discarded_at'),
]
operations = [
migrations.AddField(
model_name='playerstate',
name='sleep_episodes',
field=models.PositiveSmallIntegerField(default=0),
),
migrations.AddField(
model_name='playerstate',
name='sleep_left_ms',
field=models.PositiveBigIntegerField(default=0),
),
]

18
podcasts/migrations/0017_feed_body_hash.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 6.1 on 2026-10-04 08:13
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0016_playerstate_sleep'),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='http_body_hash',
field=models.CharField(blank=True, max_length=64),
),
]

18
podcasts/migrations/0018_episode_silences.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 6.1 on 2026-10-04 10:28
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0017_feed_body_hash'),
]
operations = [
migrations.AddField(
model_name='podcastepisode',
name='silences',
field=models.JSONField(blank=True, null=True),
),
]

35
podcasts/migrations/0019_audiobooks.py Normal file
View file

@ -0,0 +1,35 @@
# Generated by Django 6.1 on 2026-10-16 10:38
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0018_episode_silences'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='kind',
field=models.CharField(choices=[('podcast', 'Podcast'), ('audiobook', 'Hörbuch')], default='podcast', max_length=10),
),
migrations.CreateModel(
name='EpisodeBookmark',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('position_ms', models.PositiveIntegerField()),
('note', models.CharField(blank=True, max_length=1000)),
('created_at', models.DateTimeField(auto_now_add=True)),
('episode', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bookmarks', to='podcasts.podcastepisode')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='episode_bookmarks', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['position_ms', 'id'],
},
),
]

18
podcasts/migrations/0020_audiobook_folder.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 6.1 on 2026-10-16 13:29
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0019_audiobooks'),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='folder',
field=models.CharField(blank=True, max_length=100),
),
]

32
podcasts/migrations/0021_folders_list.py Normal file
View file

@ -0,0 +1,32 @@
# Generated by Django 6.1 on 2026-10-16 18:21
from django.db import migrations, models
def folder_to_folders(apps, schema_editor):
"""An audiobook's one folder becomes its first (and only) folder in the list."""
PodcastFeed = apps.get_model('podcasts', 'PodcastFeed')
for feed in PodcastFeed.objects.exclude(folder=''):
feed.folders = [feed.folder]
feed.save(update_fields=['folders'])
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0020_audiobook_folder'),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='folders',
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name='podcastsettings',
name='folder_order',
field=models.JSONField(blank=True, default=list),
),
migrations.RunPython(folder_to_folders, migrations.RunPython.noop),
]

49
podcasts/migrations/0022_lists_and_levels.py Normal file
View file

@ -0,0 +1,49 @@
# Generated by Django 6.1 on 2026-10-19 15:53
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0021_folders_list'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='refresh_interval',
field=models.CharField(blank=True, choices=[('', 'Automatisch'), ('hourly', 'Stündlich'), ('daily', 'Täglich'), ('weekly', 'Wöchentlich')], default='', max_length=10),
),
migrations.AddField(
model_name='podcastsettings',
name='lists_v2',
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name='podcastsettings',
name='stream_seen_at',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AlterField(
model_name='podcastfeed',
name='new_episodes',
field=models.CharField(choices=[('inbox', 'Im Strom'), ('queue', 'Stamm: in die Merkliste'), ('play', 'Stamm: in die Wiedergabeliste'), ('ignore', 'Auf Abruf')], default='inbox', max_length=10),
),
migrations.CreateModel(
name='PodcastLater',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('added_at', models.DateTimeField(auto_now_add=True)),
('auto', models.BooleanField(default=False)),
('episode', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='later_by', to='podcasts.podcastepisode')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='podcast_later', to=settings.AUTH_USER_MODEL)),
],
options={
'unique_together': {('user', 'episode')},
},
),
]

Some files were not shown because too many files have changed in this diff Show more