mrwnslz/diora-web
1
0
Fork 0
Code Issues 4 Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: mrwnslz:master
Branches Tags
mrwnslz:master
mrwnslz:wt-tui
mrwnslz:testing
mrwnslz:worktree-book-folders
mrwnslz:worktree-tui-reader
..
compare: mrwnslz:testing
Branches Tags
mrwnslz:master
mrwnslz:wt-tui
mrwnslz:testing
mrwnslz:worktree-book-folders
mrwnslz:worktree-tui-reader

No commits in common. "master" and "testing" have entirely different histories.
master ... testing

226 changed files with 6796 additions and 41508 deletions
Show stats Expand all files Collapse all files

28
.forgejo/workflows/test.yml
View file

@ -38,31 +38,3 @@ jobs:
- name: Run tests
run: /tmp/venv/bin/python manage.py test
e2e:
# The browser tests (e2e/, docs/refactoring.md): a real runserver on its own
# throwaway database, a book uploaded, read, highlighted, the theme switched.
runs-on: ubuntu-latest
container:
image: catthehacker/ubuntu:act-24.04
steps:
- uses: actions/checkout@v4
- name: Install Python dependencies
run: |
python -m venv /tmp/venv
/tmp/venv/bin/pip install --no-cache-dir -r requirements.txt
- name: Install Playwright and Chromium
run: |
npm ci
npx playwright install --with-deps chromium
- name: Browser tests
env:
CI: '1'
E2E_PYTHON: /tmp/venv/bin/python
run: |
mkdir -p data
npx playwright test

1
.gitattributes vendored
View file

@ -1 +0,0 @@
*.pdf binary

134
accounts/keyshare.py
View file

@ -1,134 +0,0 @@
"""Passing the book key from an unlocked device to a new one (models.KeyShare).
The book key never reaches the server in the clear: the new device posts an ECDH P-256
public key, an unlocked device of the same account answers with its own public key and
the book key AES-GCM-encrypted under the shared secret. Both devices show a code made
from the request's public key (static/js/app/33-unlock.js), so the user approves the
right device. Requests run out after a few minutes; an answer is handed out once.
POST /accounts/keyshare/ {pub, device} -> {token}
GET /accounts/keyshare/<token>/ -> {state, pub, iv, ct}
GET /accounts/keyshare/pending/ -> {requests: [...]}
POST /accounts/keyshare/<token>/answer/ {pub, iv, ct} | {decline: true}
"""
import base64
import binascii
import json
from datetime import timedelta
from django.http import JsonResponse
from django.utils import timezone
from django.views.decorators.http import require_http_methods
from .models import KeyShare
LIFETIME = timedelta(minutes=10)
# Only requests whose device asked lately are offered: a closed tab drops out quickly.
ALIVE = timedelta(seconds=20)
MAX_OPEN = 5
def _auth(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
return None
def _body(request):
try:
body = json.loads(request.body or b'{}')
except (json.JSONDecodeError, ValueError):
return None
return body if isinstance(body, dict) else None
def _b64(value, low, high):
"""A base64 string whose decoded length lies in [low, high], else None."""
if not isinstance(value, str) or len(value) > 4 * high:
return None
try:
raw = base64.b64decode(value, validate=True)
except (binascii.Error, ValueError):
return None
return value if low <= len(raw) <= high else None
def _purge(user):
KeyShare.objects.filter(user=user, created_at__lt=timezone.now() - LIFETIME).delete()
@require_http_methods(['POST'])
def create(request):
if (r := _auth(request)):
return r
body = _body(request)
pub = _b64((body or {}).get('pub'), 65, 65) # uncompressed P-256 point
if pub is None:
return JsonResponse({'error': 'pub required'}, status=400)
_purge(request.user)
# A device that reloads asks anew: the oldest requests make room instead of blocking it.
stale = KeyShare.objects.filter(user=request.user).order_by('-created_at').values_list('pk', flat=True)[MAX_OPEN - 1:]
KeyShare.objects.filter(pk__in=list(stale)).delete()
device = str(body.get('device') or '')[:100]
share = KeyShare.objects.create(user=request.user, request_pub=pub, device=device,
polled_at=timezone.now())
return JsonResponse({'token': share.token})
@require_http_methods(['GET'])
def status(request, token):
if (r := _auth(request)):
return r
_purge(request.user)
share = KeyShare.objects.filter(user=request.user, token=token).first()
if share is None:
return JsonResponse({'state': 'expired'})
if share.declined:
share.delete()
return JsonResponse({'state': 'declined'})
if not share.answer_ct:
share.polled_at = timezone.now()
share.save(update_fields=['polled_at'])
return JsonResponse({'state': 'pending'})
data = {'state': 'answered', 'pub': share.answer_pub, 'iv': share.answer_iv, 'ct': share.answer_ct}
share.delete() # handed out once
return JsonResponse(data)
@require_http_methods(['GET'])
def pending(request):
if (r := _auth(request)):
return r
_purge(request.user)
open_ = KeyShare.objects.filter(user=request.user, declined=False, answer_ct='',
polled_at__gte=timezone.now() - ALIVE).order_by('-created_at')
return JsonResponse({'requests': [
{'token': s.token, 'pub': s.request_pub, 'device': s.device,
'created_at': s.created_at.isoformat()} for s in open_]})
@require_http_methods(['POST'])
def answer(request, token):
if (r := _auth(request)):
return r
_purge(request.user)
share = KeyShare.objects.filter(user=request.user, token=token).first()
if share is None:
return JsonResponse({'error': 'expired'}, status=404)
if share.answer_ct or share.declined:
return JsonResponse({'error': 'already answered'}, status=409)
body = _body(request) or {}
if body.get('decline'):
share.declined = True
share.answered_at = timezone.now()
share.save(update_fields=['declined', 'answered_at'])
return JsonResponse({'ok': True})
pub = _b64(body.get('pub'), 65, 65)
iv = _b64(body.get('iv'), 12, 12)
ct = _b64(body.get('ct'), 32 + 16, 32 + 16) # 256-bit key + GCM tag
if not (pub and iv and ct):
return JsonResponse({'error': 'pub, iv and ct required'}, status=400)
share.answer_pub, share.answer_iv, share.answer_ct = pub, iv, ct
share.answered_at = timezone.now()
share.save(update_fields=['answer_pub', 'answer_iv', 'answer_ct', 'answered_at'])
return JsonResponse({'ok': True})

33
accounts/migrations/0006_webdav_inbox.py
View file

@ -1,33 +0,0 @@
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('accounts', '0005_webdavsource'),
]
operations = [
migrations.AddField(
model_name='webdavsource',
name='inbox',
field=models.BooleanField(default=False),
),
migrations.CreateModel(
name='WebDAVSeen',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('path', models.CharField(max_length=1000)),
('size', models.BigIntegerField(default=0)),
('modified', models.CharField(blank=True, max_length=100)),
('claimed_at', models.DateTimeField(blank=True, null=True)),
('done_at', models.DateTimeField(blank=True, null=True)),
('result', models.CharField(blank=True, max_length=20)),
('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='seen', to='accounts.webdavsource')),
],
options={
'unique_together': {('source', 'path')},
},
),
]

28
accounts/migrations/0007_uservault.py
View file

@ -1,28 +0,0 @@
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('accounts', '0006_webdav_inbox'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='UserVault',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=40)),
('ct', models.TextField()),
('iv', models.CharField(max_length=64)),
('updated_at', models.DateTimeField(auto_now=True)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='vault', to=settings.AUTH_USER_MODEL)),
],
options={
'unique_together': {('user', 'name')},
},
),
]

50
accounts/migrations/0008_oidc_keyshare.py
View file

@ -1,50 +0,0 @@
# Generated by Django 6.1 on 2026-11-01 13:35
import accounts.models
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('accounts', '0007_uservault'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='KeyShare',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('token', models.CharField(default=accounts.models._generate_token, max_length=64, unique=True)),
('device', models.CharField(blank=True, max_length=100)),
('request_pub', models.TextField()),
('answer_pub', models.TextField(blank=True)),
('answer_iv', models.CharField(blank=True, max_length=64)),
('answer_ct', models.TextField(blank=True)),
('declined', models.BooleanField(default=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('polled_at', models.DateTimeField(blank=True, null=True)),
('answered_at', models.DateTimeField(blank=True, null=True)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='key_shares', to=settings.AUTH_USER_MODEL)),
],
),
migrations.CreateModel(
name='OidcIdentity',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('issuer', models.CharField(max_length=200)),
('sub', models.CharField(max_length=255)),
('username', models.CharField(blank=True, max_length=150)),
('email', models.CharField(blank=True, max_length=254)),
('created_at', models.DateTimeField(auto_now_add=True)),
('last_login_at', models.DateTimeField(blank=True, null=True)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='oidc_identities', to=settings.AUTH_USER_MODEL)),
],
options={
'unique_together': {('issuer', 'sub')},
},
),
]

70
accounts/models.py
View file

@ -56,9 +56,6 @@ class WebDAVSource(models.Model):
username = models.CharField(max_length=200, blank=True)
password = models.CharField(max_length=500, blank=True)
root_path = models.CharField(max_length=500, blank=True, default='')
# "Eingang" (docs/library.md): new books in it come into the library by themselves —
# a device with the key fetches, encrypts and uploads them (WebDAVSeen keeps which).
inbox = models.BooleanField(default=False)
created_at = models.DateTimeField(auto_now_add=True)
last_used_at = models.DateTimeField(null=True, blank=True)
@ -92,36 +89,6 @@ class WebDAVSource(models.Model):
return url
class WebDAVSeen(models.Model):
"""A file of an "Eingang" source a device took (or is taking) into the library: taken
files aren't offered again; a claim keeps two devices from taking the same file at once
and runs out when a device didn't finish."""
source = models.ForeignKey(WebDAVSource, on_delete=models.CASCADE, related_name='seen')
path = models.CharField(max_length=1000)
size = models.BigIntegerField(default=0)
modified = models.CharField(max_length=100, blank=True)
claimed_at = models.DateTimeField(null=True, blank=True)
done_at = models.DateTimeField(null=True, blank=True)
result = models.CharField(max_length=20, blank=True) # imported | known | too_big | failed
class Meta:
unique_together = ('source', 'path')
class UserVault(models.Model):
"""Small private settings of the devices, end-to-end encrypted with the book key
(docs/ai.md: the Anthropic API key of the "KI" features). The server keeps the blob,
one per name, and never sees what is in it."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='vault')
name = models.CharField(max_length=40)
ct = models.TextField()
iv = models.CharField(max_length=64)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
unique_together = ('user', 'name')
def _generate_token():
return secrets.token_hex(32)
@ -142,40 +109,3 @@ class ApiToken(models.Model):
def __str__(self):
return f"ApiToken({self.user.username})"
class OidcIdentity(models.Model):
"""An account of the sign-in provider (Authelia) linked to a diora account
(accounts/oidc.py). `sub` is the provider's stable id; name and mail are only shown."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='oidc_identities')
issuer = models.CharField(max_length=200)
sub = models.CharField(max_length=255)
username = models.CharField(max_length=150, blank=True)
email = models.CharField(max_length=254, blank=True)
created_at = models.DateTimeField(auto_now_add=True)
last_login_at = models.DateTimeField(null=True, blank=True)
class Meta:
unique_together = ('issuer', 'sub')
def __str__(self):
return f"OidcIdentity({self.username or self.sub} -> {self.user_id})"
class KeyShare(models.Model):
"""A device without the book key asks an unlocked device of the same account for it
(accounts/keyshare.py). The server only passes on public keys and ciphertext: the new
device sends an ECDH public key, the unlocked one answers with its own public key and
the book key encrypted for the shared secret. Both show a code made from the request's
public key, so the user can see the answer goes to the right device."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='key_shares')
token = models.CharField(max_length=64, unique=True, default=_generate_token)
device = models.CharField(max_length=100, blank=True)
request_pub = models.TextField() # base64 raw P-256 point of the asking device
answer_pub = models.TextField(blank=True)
answer_iv = models.CharField(max_length=64, blank=True)
answer_ct = models.TextField(blank=True)
declined = models.BooleanField(default=False)
created_at = models.DateTimeField(auto_now_add=True)
polled_at = models.DateTimeField(null=True, blank=True) # the asking device still waits
answered_at = models.DateTimeField(null=True, blank=True)

165
accounts/oidc.py
View file

@ -1,165 +0,0 @@
"""Sign-in with an OpenID Connect provider (Authelia, docs/accounts.md).
An extra way in next to username + password: an account is linked once in the settings
while signed in normally, after that the provider's login leads into it. Nobody gets a new
account from the provider — an unlinked identity is turned away.
Authorization code flow with PKCE (S256) and a confidential client (client_secret_basic).
The id_token is checked here (RS256 against the provider's JWKS, iss, aud, exp, nonce)
with `cryptography`, so no OIDC library is needed.
The book key is not touched: it is derived from the diora password on the device. A device
that signs in this way gets it from an unlocked device or by typing the password once
(accounts/keyshare.py, static/js/app/33-unlock.js).
"""
import base64
import hashlib
import json
import secrets
import time
from urllib.parse import urlencode
import requests
from cryptography.exceptions import InvalidSignature
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import padding, rsa
from django.conf import settings
TIMEOUT = 10
_CACHE_SECONDS = 3600
_cache = {} # url -> (fetched_at, json)
class OidcError(Exception):
pass
def enabled() -> bool:
return bool(settings.OIDC_ISSUER and settings.OIDC_CLIENT_ID and settings.OIDC_CLIENT_SECRET)
def key_enabled() -> bool:
return enabled() and bool(settings.OIDC_APP_CLIENT_ID)
def _b64url_decode(s: str) -> bytes:
return base64.urlsafe_b64decode(s + '=' * (-len(s) % 4))
def _b64url(b: bytes) -> str:
return base64.urlsafe_b64encode(b).rstrip(b'=').decode('ascii')
def _get_json(url: str, fresh: bool = False) -> dict:
hit = _cache.get(url)
if hit and not fresh and time.time() - hit[0] < _CACHE_SECONDS:
return hit[1]
try:
resp = requests.get(url, timeout=TIMEOUT)
resp.raise_for_status()
data = resp.json()
except (requests.RequestException, ValueError) as exc:
raise OidcError(f'{url} nicht erreichbar: {exc}') from exc
_cache[url] = (time.time(), data)
return data
def discovery() -> dict:
issuer = settings.OIDC_ISSUER.rstrip('/')
return _get_json(issuer + '/.well-known/openid-configuration')
def new_flow() -> dict:
"""Random state, nonce and PKCE verifier for one sign-in (kept in the session)."""
return {
'state': secrets.token_urlsafe(24),
'nonce': secrets.token_urlsafe(24),
'verifier': secrets.token_urlsafe(48),
}
def authorize_url(flow: dict, redirect_uri: str) -> str:
challenge = _b64url(hashlib.sha256(flow['verifier'].encode('ascii')).digest())
params = {
'response_type': 'code',
'client_id': settings.OIDC_CLIENT_ID,
'redirect_uri': redirect_uri,
'scope': 'openid profile email groups',
'state': flow['state'],
'nonce': flow['nonce'],
'code_challenge': challenge,
'code_challenge_method': 'S256',
}
return discovery()['authorization_endpoint'] + '?' + urlencode(params)
def exchange_code(code: str, flow: dict, redirect_uri: str) -> dict:
"""Trade the code for tokens and return the checked id_token claims."""
try:
resp = requests.post(
discovery()['token_endpoint'],
data={
'grant_type': 'authorization_code',
'code': code,
'redirect_uri': redirect_uri,
'code_verifier': flow['verifier'],
},
auth=(settings.OIDC_CLIENT_ID, settings.OIDC_CLIENT_SECRET),
timeout=TIMEOUT,
)
except requests.RequestException as exc:
raise OidcError(f'Token-Abruf fehlgeschlagen: {exc}') from exc
if resp.status_code != 200:
raise OidcError(f'Token-Abruf abgelehnt ({resp.status_code})')
try:
id_token = resp.json()['id_token']
except (ValueError, KeyError) as exc:
raise OidcError('Antwort ohne id_token') from exc
return verify_id_token(id_token, flow['nonce'])
def _public_key(kid: str):
jwks_uri = discovery()['jwks_uri']
for fresh in (False, True): # a key rotation shows up as an unknown kid: fetch again once
for jwk in _get_json(jwks_uri, fresh=fresh).get('keys', []):
if jwk.get('kty') == 'RSA' and (not kid or jwk.get('kid') == kid):
n = int.from_bytes(_b64url_decode(jwk['n']), 'big')
e = int.from_bytes(_b64url_decode(jwk['e']), 'big')
return rsa.RSAPublicNumbers(e, n).public_key()
raise OidcError('Signaturschlüssel des Anbieters nicht gefunden')
def verify_id_token(token: str, nonce, now: float = None, client_id: str = None) -> dict:
"""Check signature and claims. `nonce` is the expected value, or a function that says
whether the token's nonce is good (the apps' signed ones, oidc_views.app_login)."""
try:
header_b64, payload_b64, sig_b64 = token.split('.')
header = json.loads(_b64url_decode(header_b64))
claims = json.loads(_b64url_decode(payload_b64))
signature = _b64url_decode(sig_b64)
except (ValueError, TypeError) as exc:
raise OidcError('id_token unlesbar') from exc
if header.get('alg') != 'RS256':
raise OidcError(f'nicht unterstützte Signatur {header.get("alg")!r} (erwartet RS256)')
try:
_public_key(header.get('kid', '')).verify(
signature, f'{header_b64}.{payload_b64}'.encode('ascii'),
padding.PKCS1v15(), hashes.SHA256())
except InvalidSignature as exc:
raise OidcError('Signatur des id_token ungültig') from exc
now = time.time() if now is None else now
if claims.get('iss', '').rstrip('/') != settings.OIDC_ISSUER.rstrip('/'):
raise OidcError('falscher Aussteller')
aud = claims.get('aud')
if (client_id or settings.OIDC_CLIENT_ID) not in (aud if isinstance(aud, list) else [aud]):
raise OidcError('id_token ist nicht für diora')
if not isinstance(claims.get('exp'), (int, float)) or claims['exp'] < now - 60:
raise OidcError('id_token abgelaufen')
good = nonce(claims.get('nonce') or '') if callable(nonce) else (nonce and claims.get('nonce') == nonce)
if not good:
raise OidcError('nonce passt nicht')
if not claims.get('sub'):
raise OidcError('id_token ohne sub')
return claims

245
accounts/oidc_views.py
View file

@ -1,245 +0,0 @@
"""Sign-in with Authelia and linking it to an existing account (accounts/oidc.py)."""
import json
import logging
import secrets
from django.conf import settings
from django.contrib import messages
from django.contrib.auth import login
from django.contrib.auth.decorators import login_required
from django.core import signing
from django.http import JsonResponse
from django.shortcuts import redirect, render
from django.utils import timezone
from django.utils.http import url_has_allowed_host_and_scheme
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from . import oidc
from .models import ApiToken, OidcIdentity
log = logging.getLogger(__name__)
SESSION_KEY = 'oidc_flow'
def _redirect_uri(request):
return settings.OIDC_REDIRECT_URI or request.build_absolute_uri('/accounts/oidc/callback/')
def _safe_next(request, value):
if value and url_has_allowed_host_and_scheme(value, allowed_hosts={request.get_host()},
require_https=request.is_secure()):
return value
return '/'
def oidc_context(request):
"""What the settings page shows about the provider."""
return {
'oidc_enabled': oidc.enabled(),
'oidc_provider': settings.OIDC_PROVIDER_NAME,
'oidc_identities': list(request.user.oidc_identities.all()) if request.user.is_authenticated else [],
}
@require_http_methods(['GET'])
def oidc_start(request):
"""Off to the provider. `?link=1` (signed in) links the account instead of signing in."""
if not oidc.enabled():
messages.error(request, 'Die Anmeldung über Authelia ist auf diesem Server nicht eingerichtet.')
return redirect('login')
link = request.GET.get('link') == '1'
if link and not request.user.is_authenticated:
return redirect('login')
flow = oidc.new_flow()
flow['link'] = link
flow['next'] = _safe_next(request, request.GET.get('next', ''))
try:
url = oidc.authorize_url(flow, _redirect_uri(request))
except oidc.OidcError as exc:
log.warning('oidc start: %s', exc)
messages.error(request, 'Authelia ist gerade nicht erreichbar.')
return redirect('settings' if link else 'login')
request.session[SESSION_KEY] = flow
return redirect(url)
@require_http_methods(['GET'])
def oidc_callback(request):
flow = request.session.pop(SESSION_KEY, None)
fail_to = 'login'
if flow and flow.get('link'):
fail_to = 'settings'
if not flow or request.GET.get('state') != flow.get('state'):
messages.error(request, 'Die Anmeldung über Authelia ist abgelaufen. Bitte noch einmal.')
return redirect(fail_to)
if request.GET.get('error'):
messages.error(request, 'Authelia hat die Anmeldung abgebrochen.')
return redirect(fail_to)
code = request.GET.get('code', '')
try:
claims = oidc.exchange_code(code, flow, _redirect_uri(request))
except oidc.OidcError as exc:
log.warning('oidc callback: %s', exc)
messages.error(request, 'Die Anmeldung über Authelia hat nicht geklappt.')
return redirect(fail_to)
issuer = settings.OIDC_ISSUER.rstrip('/')
sub = claims['sub']
shown = {
'username': (claims.get('preferred_username') or '')[:150],
'email': (claims.get('email') or '')[:254],
}
identity = OidcIdentity.objects.filter(issuer=issuer, sub=sub).select_related('user').first()
if flow.get('link'):
if not request.user.is_authenticated:
return redirect('login')
if identity and identity.user_id != request.user.id:
messages.error(request, 'Dieses Authelia-Konto ist schon mit einem anderen diora-Konto verknüpft.')
return redirect('settings')
if identity is None:
OidcIdentity.objects.create(user=request.user, issuer=issuer, sub=sub, **shown)
else:
OidcIdentity.objects.filter(pk=identity.pk).update(**shown)
messages.success(request, f'Authelia-Konto „{shown["username"] or sub}“ verknüpft. '
'Du kannst dich jetzt auch darüber anmelden.')
if oidc.key_enabled(): # leave the book key with Authelia's help right away
return redirect('/accounts/oidc/key/?next=/accounts/settings/')
return redirect('settings')
if identity is None:
messages.error(request, 'Dieses Authelia-Konto ist mit keinem diora-Konto verknüpft. '
'Melde dich einmal mit Passwort an und verknüpfe es in den Einstellungen.')
return redirect('login')
if not identity.user.is_active:
messages.error(request, 'Dieses diora-Konto ist gesperrt.')
return redirect('login')
OidcIdentity.objects.filter(pk=identity.pk).update(last_login_at=timezone.now(), **shown)
login(request, identity.user, backend='django.contrib.auth.backends.ModelBackend')
# A password login brings the book key along, this one cannot: the device fetches it
# (oidc_key below), else the page asks (static/js/app/33-unlock.js).
next_url = flow.get('next') or '/'
if oidc.key_enabled():
from urllib.parse import urlencode
return redirect('/accounts/oidc/key/?' + urlencode({'next': next_url}))
return redirect(next_url)
@login_required
@require_http_methods(['POST'])
def oidc_unlink(request, pk):
OidcIdentity.objects.filter(user=request.user, pk=pk).delete()
messages.success(request, 'Verknüpfung mit Authelia gelöst.')
return redirect('settings')
@login_required
@require_http_methods(['GET'])
def oidc_key(request):
"""The book key with the provider's help, without asking (docs/accounts.md).
Runs entirely on the device: it signs in once more as the public client
(OIDC_APP_CLIENT_ID, PKCE, prompt=none — the provider session from just before makes it
invisible), reads its `diora_key` claim from userinfo and with it opens the book key the
server keeps wrapped (vault "authelia-key") — or, when the device has the key, wraps and
leaves it there. The server never sees the claim; this view only hands out the page.
The same page is the redirect target (`/accounts/oidc/key/callback/`)."""
next_url = _safe_next(request, request.GET.get('next', ''))
if not oidc.key_enabled():
return redirect(next_url)
try:
disco = oidc.discovery()
except oidc.OidcError as exc:
log.warning('oidc key: %s', exc)
return redirect(next_url)
config = {
'authorize': disco['authorization_endpoint'],
'token': disco['token_endpoint'],
'userinfo': disco.get('userinfo_endpoint', ''),
'client_id': settings.OIDC_APP_CLIENT_ID,
'redirect_uri': request.build_absolute_uri('/accounts/oidc/key/callback/')
if not settings.OIDC_REDIRECT_URI
else settings.OIDC_REDIRECT_URI.replace('/oidc/callback/', '/oidc/key/callback/'),
'subs': [i.sub for i in request.user.oidc_identities.all()],
'user_id': request.user.id,
'next': next_url,
}
return render(request, 'accounts/oidc_key.html', {'config': config})
# ---------------------------------------------------------------------------
# The apps (Android, desktop): sign in with Authelia as the public client themselves
# ---------------------------------------------------------------------------
#
# 1. GET /accounts/oidc/app/ -> where Authelia is, the client id and a nonce
# signed by this server (good for APP_NONCE_AGE).
# 2. The app runs the code flow with PKCE in the system browser (redirect to its own
# scheme or loopback), trades the code itself and reads `diora_key` from userinfo.
# 3. POST /accounts/oidc/app/login/ {id_token} -> the account's API token, as the
# password login (diora-android Auth.kt) hands it out.
# The book key never passes here: the app opens the vault "authelia-key" itself.
APP_NONCE_SALT = 'diora.oidc.app'
APP_NONCE_AGE = 600
@require_http_methods(['GET'])
def app_config(request):
if not oidc.key_enabled():
return JsonResponse({'enabled': False})
try:
disco = oidc.discovery()
except oidc.OidcError as exc:
log.warning('oidc app config: %s', exc)
return JsonResponse({'enabled': False, 'error': 'provider unreachable'}, status=502)
return JsonResponse({
'enabled': True,
'provider': settings.OIDC_PROVIDER_NAME,
'issuer': settings.OIDC_ISSUER.rstrip('/'),
'authorize': disco['authorization_endpoint'],
'token': disco['token_endpoint'],
'userinfo': disco.get('userinfo_endpoint', ''),
'client_id': settings.OIDC_APP_CLIENT_ID,
'scope': 'openid profile diora_key',
'nonce': signing.dumps(secrets.token_urlsafe(16), salt=APP_NONCE_SALT),
})
def _app_nonce_ok(value):
try:
signing.loads(value, salt=APP_NONCE_SALT, max_age=APP_NONCE_AGE)
return True
except signing.BadSignature:
return False
@csrf_exempt # a native client without cookies; the signed id_token is the proof
@require_http_methods(['POST'])
def app_login(request):
if not oidc.key_enabled():
return JsonResponse({'error': 'not enabled'}, status=404)
try:
body = json.loads(request.body or b'{}')
id_token = body['id_token']
if not isinstance(id_token, str):
raise TypeError
except (ValueError, KeyError, TypeError):
return JsonResponse({'error': 'id_token required'}, status=400)
try:
claims = oidc.verify_id_token(id_token, _app_nonce_ok, client_id=settings.OIDC_APP_CLIENT_ID)
except oidc.OidcError as exc:
log.warning('oidc app login: %s', exc)
return JsonResponse({'error': 'Die Anmeldung über Authelia war ungültig.'}, status=401)
identity = OidcIdentity.objects.filter(
issuer=settings.OIDC_ISSUER.rstrip('/'), sub=claims['sub']).select_related('user').first()
if identity is None:
return JsonResponse({'error': 'Dieses Authelia-Konto ist mit keinem diora-Konto verknüpft. '
'Verknüpfe es einmal im Browser unter Einstellungen.'}, status=403)
if not identity.user.is_active:
return JsonResponse({'error': 'Dieses diora-Konto ist gesperrt.'}, status=403)
OidcIdentity.objects.filter(pk=identity.pk).update(last_login_at=timezone.now())
token, _ = ApiToken.objects.get_or_create(user=identity.user)
return JsonResponse({'token': token.token, 'user_id': identity.user.id,
'username': identity.user.username, 'sub': claims['sub']})

33
accounts/sync.py
View file

@ -3,8 +3,7 @@ from django.http import JsonResponse
from django.utils import timezone
from django.views.decorators.http import require_http_methods
from books.models import EBook, EBookProgress, EBookHighlights, EBookBookmarks, book_rows
from books.views import _now_ms, progress_state
from books.models import EBook, EBookProgress, EBookHighlights, EBookBookmarks
from podcasts.models import PodcastFeed, EpisodeProgress, PodcastQueue
from radio.models import SavedStation
@ -20,22 +19,21 @@ def sync_snapshot(request):
endpoints (e.g. GET /books/<id>/data/), which this same token also
unlocks. Writes (progress, highlights, bookmarks, ...) go through those
existing endpoints too, to reuse their merge semantics rather than
duplicating it here.
?only=books: just the books' part. The apps' book sync needs nothing else,
and the podcast part is by far the biggest (every episode ever played:
megabytes for an account that moved over from AntennaPod)."""
duplicating it here."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
user = request.user
books = [{k: b[k] for k in ('id', 'meta_ct', 'meta_iv', 'uploaded_at', 'is_read')} for b in book_rows(user)]
books = list(EBook.objects.filter(user=user).values('id', 'meta_ct', 'meta_iv', 'uploaded_at', 'is_read'))
for b in books:
b['uploaded_at'] = b['uploaded_at'].isoformat()
book_progress = [
{'book_id': p.book_id, 'updated_at': p.updated_at.isoformat(), **progress_state(p)}
for p in EBookProgress.objects.filter(user=user)
]
book_progress = list(
EBookProgress.objects.filter(user=user).values('book_id', 'scroll_fraction', 'position_anchor', 'updated_at')
)
for p in book_progress:
p['updated_at'] = p['updated_at'].isoformat()
book_highlights = list(EBookHighlights.objects.filter(user=user).values('book_id', 'ct', 'iv', 'updated_at'))
for h in book_highlights:
@ -45,16 +43,6 @@ def sync_snapshot(request):
for bm in book_bookmarks:
bm['updated_at'] = bm['updated_at'].isoformat()
if request.GET.get('only') == 'books':
return JsonResponse({
'server_time': timezone.now().isoformat(),
'server_now': _now_ms(),
'books': books,
'book_progress': book_progress,
'book_highlights': book_highlights,
'book_bookmarks': book_bookmarks,
})
podcast_feeds = list(
PodcastFeed.objects.filter(user=user)
.annotate(latest_episode_at=Max('episodes__pub_date'))
@ -83,7 +71,6 @@ def sync_snapshot(request):
return JsonResponse({
'server_time': timezone.now().isoformat(),
'server_now': _now_ms(),
'books': books,
'book_progress': book_progress,
'book_highlights': book_highlights,

321
accounts/test_oidc.py
View file

@ -1,321 +0,0 @@
import base64
import json
import time
from unittest.mock import patch
from urllib.parse import parse_qs, urlsplit
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import padding, rsa
from django.contrib.auth.models import User
from django.test import TestCase, override_settings
from . import oidc
from .models import ApiToken, KeyShare, OidcIdentity
ISSUER = 'https://auth.example.org'
PRIV = rsa.generate_private_key(public_exponent=65537, key_size=2048)
OTHER_PRIV = rsa.generate_private_key(public_exponent=65537, key_size=2048)
def _b64url(b):
return base64.urlsafe_b64encode(b).rstrip(b'=').decode()
def _int_b64(i):
return _b64url(i.to_bytes((i.bit_length() + 7) // 8, 'big'))
def make_token(claims, key=PRIV, alg='RS256', kid='k1'):
header = _b64url(json.dumps({'alg': alg, 'kid': kid, 'typ': 'JWT'}).encode())
payload = _b64url(json.dumps(claims).encode())
sig = key.sign(f'{header}.{payload}'.encode(), padding.PKCS1v15(), hashes.SHA256())
return f'{header}.{payload}.{_b64url(sig)}'
DISCOVERY = {
'issuer': ISSUER,
'authorization_endpoint': ISSUER + '/api/oidc/authorization',
'token_endpoint': ISSUER + '/api/oidc/token',
'jwks_uri': ISSUER + '/jwks.json',
}
pub = PRIV.public_key().public_numbers()
JWKS = {'keys': [{'kty': 'RSA', 'kid': 'k1', 'n': _int_b64(pub.n), 'e': _int_b64(pub.e)}]}
class FakeResp:
def __init__(self, data, status=200):
self._data, self.status_code = data, status
def json(self):
return self._data
def raise_for_status(self):
if self.status_code >= 400:
raise oidc.requests.HTTPError(str(self.status_code))
def fake_get(url, timeout=None):
return FakeResp(DISCOVERY if url.endswith('openid-configuration') else JWKS)
@override_settings(OIDC_ISSUER=ISSUER, OIDC_CLIENT_ID='diora', OIDC_CLIENT_SECRET='s3cret',
OIDC_REDIRECT_URI='https://diora.test/accounts/oidc/callback/')
class OidcFlowTests(TestCase):
def setUp(self):
oidc._cache.clear()
self.user = User.objects.create_user(username='crm', password='pw12345678')
self.get = patch('accounts.oidc.requests.get', side_effect=fake_get)
self.get.start()
self.addCleanup(self.get.stop)
def _start(self, query=''):
resp = self.client.get('/accounts/oidc/start/' + query)
self.assertEqual(resp.status_code, 302)
q = parse_qs(urlsplit(resp['Location']).query)
self.assertEqual(q['code_challenge_method'], ['S256'])
self.assertEqual(q['client_id'], ['diora'])
return q['state'][0], q['nonce'][0]
def _callback(self, state, claims, token_status=200):
token = make_token(claims)
with patch('accounts.oidc.requests.post',
return_value=FakeResp({'id_token': token}, token_status)) as post:
resp = self.client.get('/accounts/oidc/callback/', {'state': state, 'code': 'c0de'})
return resp, post
def _claims(self, nonce, **extra):
c = {'iss': ISSUER, 'aud': ['diora'], 'sub': 'uuid-1', 'exp': time.time() + 300,
'nonce': nonce, 'preferred_username': 'marwin', 'email': 'm@example.org'}
c.update(extra)
return c
def test_unlinked_identity_is_turned_away(self):
state, nonce = self._start()
resp, _ = self._callback(state, self._claims(nonce))
self.assertEqual(resp['Location'], '/accounts/login/')
self.assertNotIn('_auth_user_id', self.client.session)
self.assertFalse(User.objects.exclude(pk=self.user.pk).exists())
def test_link_then_login(self):
self.client.force_login(self.user)
state, nonce = self._start('?link=1')
resp, post = self._callback(state, self._claims(nonce))
self.assertEqual(resp['Location'], '/accounts/settings/')
ident = OidcIdentity.objects.get()
self.assertEqual((ident.user, ident.sub, ident.username), (self.user, 'uuid-1', 'marwin'))
# client_secret_basic + PKCE verifier
self.assertEqual(post.call_args.kwargs['auth'], ('diora', 's3cret'))
self.assertIn('code_verifier', post.call_args.kwargs['data'])
self.client.logout()
state, nonce = self._start('?next=/books/')
resp, _ = self._callback(state, self._claims(nonce))
self.assertEqual(resp['Location'], '/books/')
self.assertEqual(int(self.client.session['_auth_user_id']), self.user.pk)
ident.refresh_from_db()
self.assertIsNotNone(ident.last_login_at)
def test_next_to_other_host_is_dropped(self):
OidcIdentity.objects.create(user=self.user, issuer=ISSUER, sub='uuid-1')
state, nonce = self._start('?next=https://evil.example/')
resp, _ = self._callback(state, self._claims(nonce))
self.assertEqual(resp['Location'], '/')
def test_identity_of_other_account_is_not_moved(self):
other = User.objects.create_user(username='bob', password='pw12345678')
OidcIdentity.objects.create(user=other, issuer=ISSUER, sub='uuid-1')
self.client.force_login(self.user)
state, nonce = self._start('?link=1')
self._callback(state, self._claims(nonce))
self.assertEqual(OidcIdentity.objects.get().user, other)
def test_wrong_state_nonce_audience_or_signature_fail(self):
OidcIdentity.objects.create(user=self.user, issuer=ISSUER, sub='uuid-1')
cases = [
('state', {}),
('nonce', {'nonce': 'other'}),
('aud', {'aud': 'someone-else'}),
('iss', {'iss': 'https://evil.example'}),
('exp', {'exp': time.time() - 3600}),
]
for name, extra in cases:
with self.subTest(name):
state, nonce = self._start()
claims = {**self._claims(nonce), **extra}
resp, _ = self._callback('wrong' if name == 'state' else state, claims)
self.assertEqual(resp['Location'], '/accounts/login/')
self.assertNotIn('_auth_user_id', self.client.session)
state, nonce = self._start()
with patch('accounts.oidc.requests.post',
return_value=FakeResp({'id_token': make_token(self._claims(nonce), key=OTHER_PRIV)})):
self.client.get('/accounts/oidc/callback/', {'state': state, 'code': 'x'})
self.assertNotIn('_auth_user_id', self.client.session)
def test_state_is_single_use(self):
OidcIdentity.objects.create(user=self.user, issuer=ISSUER, sub='uuid-1')
state, nonce = self._start()
self._callback(state, self._claims(nonce))
self.client.logout()
resp, _ = self._callback(state, self._claims(nonce))
self.assertNotIn('_auth_user_id', self.client.session)
def test_unlink(self):
ident = OidcIdentity.objects.create(user=self.user, issuer=ISSUER, sub='uuid-1')
other = User.objects.create_user(username='bob', password='pw12345678')
foreign = OidcIdentity.objects.create(user=other, issuer=ISSUER, sub='uuid-2')
self.client.force_login(self.user)
self.client.post(f'/accounts/oidc/{foreign.pk}/unlink/')
self.client.post(f'/accounts/oidc/{ident.pk}/unlink/')
self.assertEqual(list(OidcIdentity.objects.all()), [foreign])
def test_with_app_client_login_and_link_go_by_the_key_page(self):
OidcIdentity.objects.create(user=self.user, issuer=ISSUER, sub='uuid-1')
with override_settings(OIDC_APP_CLIENT_ID='diora-app'):
state, nonce = self._start('?next=/books/')
resp, _ = self._callback(state, self._claims(nonce))
self.assertEqual(resp['Location'], '/accounts/oidc/key/?next=%2Fbooks%2F')
page = self.client.get(resp['Location'])
self.assertContains(page, 'oidc-key-config')
cfg = page.context['config']
self.assertEqual(cfg['client_id'], 'diora-app')
self.assertEqual(cfg['redirect_uri'], 'https://diora.test/accounts/oidc/key/callback/')
self.assertEqual(cfg['subs'], ['uuid-1'])
self.assertEqual(cfg['next'], '/books/')
self.assertEqual(cfg['token'], DISCOVERY['token_endpoint'])
self.assertEqual(self.client.get('/accounts/oidc/key/callback/?code=x&state=y').status_code, 200)
state, nonce = self._start('?link=1')
resp, _ = self._callback(state, self._claims(nonce))
self.assertEqual(resp['Location'], '/accounts/oidc/key/?next=/accounts/settings/')
def test_key_page_without_app_client_goes_straight_on(self):
self.client.force_login(self.user)
resp = self.client.get('/accounts/oidc/key/?next=/books/')
self.assertEqual(resp['Location'], '/books/')
def test_login_page_shows_button_only_when_configured(self):
self.assertContains(self.client.get('/accounts/login/'), 'Mit Authelia anmelden')
with override_settings(OIDC_CLIENT_SECRET=''):
self.assertNotContains(self.client.get('/accounts/login/'), 'Mit Authelia anmelden')
@override_settings(OIDC_ISSUER=ISSUER, OIDC_CLIENT_ID='diora', OIDC_CLIENT_SECRET='s3cret',
OIDC_APP_CLIENT_ID='diora-app')
class OidcAppLoginTests(TestCase):
def setUp(self):
oidc._cache.clear()
self.user = User.objects.create_user(username='crm', password='pw12345678')
OidcIdentity.objects.create(user=self.user, issuer=ISSUER, sub='uuid-1')
p = patch('accounts.oidc.requests.get', side_effect=fake_get)
p.start()
self.addCleanup(p.stop)
def _nonce(self):
cfg = self.client.get('/accounts/oidc/app/').json()
self.assertEqual((cfg['client_id'], cfg['token']), ('diora-app', DISCOVERY['token_endpoint']))
return cfg['nonce']
def _login(self, **claims):
c = {'iss': ISSUER, 'aud': ['diora-app'], 'sub': 'uuid-1', 'exp': time.time() + 300,
'nonce': self._nonce()}
c.update(claims)
return self.client.post('/accounts/oidc/app/login/', json.dumps({'id_token': make_token(c)}),
content_type='application/json')
def test_linked_account_gets_its_existing_token(self):
existing = ApiToken.objects.create(user=self.user)
resp = self._login()
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.json(), {'token': existing.token, 'user_id': self.user.id,
'username': 'crm', 'sub': 'uuid-1'})
def test_token_is_made_when_there_is_none(self):
resp = self._login()
self.assertEqual(resp.json()['token'], ApiToken.objects.get(user=self.user).token)
def test_refused(self):
for name, claims, code in [
('unlinked', {'sub': 'other'}, 403),
('web client token', {'aud': 'diora'}, 401),
('own nonce', {'nonce': 'made-up'}, 401),
('expired', {'exp': time.time() - 3600}, 401),
]:
with self.subTest(name):
self.assertEqual(self._login(**claims).status_code, code)
self.assertEqual(self.client.post('/accounts/oidc/app/login/', '{}',
content_type='application/json').status_code, 400)
def test_old_nonce_is_refused(self):
nonce = self._nonce()
with patch('django.core.signing.time.time', return_value=time.time() + 3600):
resp = self.client.post('/accounts/oidc/app/login/', json.dumps({'id_token': make_token(
{'iss': ISSUER, 'aud': 'diora-app', 'sub': 'uuid-1', 'exp': time.time() + 7200, 'nonce': nonce})}),
content_type='application/json')
self.assertEqual(resp.status_code, 401)
def test_off_without_app_client(self):
with override_settings(OIDC_APP_CLIENT_ID=''):
self.assertEqual(self.client.get('/accounts/oidc/app/').json(), {'enabled': False})
PUB = base64.b64encode(b'\x04' + b'\x01' * 64).decode()
class KeyShareTests(TestCase):
def setUp(self):
self.user = User.objects.create_user(username='crm', password='pw12345678')
self.client.force_login(self.user)
def _post(self, url, body):
return self.client.post(url, json.dumps(body), content_type='application/json')
def test_round_trip_hands_answer_out_once(self):
token = self._post('/accounts/keyshare/', {'pub': PUB, 'device': 'Firefox auf Linux'}).json()['token']
self.assertEqual(self.client.get(f'/accounts/keyshare/{token}/').json(), {'state': 'pending'})
pending = self.client.get('/accounts/keyshare/pending/').json()['requests']
self.assertEqual([(r['token'], r['pub'], r['device']) for r in pending], [(token, PUB, 'Firefox auf Linux')])
answer = {'pub': PUB, 'iv': base64.b64encode(b'i' * 12).decode(),
'ct': base64.b64encode(b'c' * 48).decode()}
self.assertEqual(self._post(f'/accounts/keyshare/{token}/answer/', answer).status_code, 200)
self.assertEqual(self._post(f'/accounts/keyshare/{token}/answer/', answer).status_code, 409)
self.assertEqual(self.client.get('/accounts/keyshare/pending/').json()['requests'], [])
got = self.client.get(f'/accounts/keyshare/{token}/').json()
self.assertEqual(got, {'state': 'answered', **answer})
self.assertEqual(self.client.get(f'/accounts/keyshare/{token}/').json(), {'state': 'expired'})
def test_requests_of_devices_that_stopped_asking_are_not_offered(self):
token = self._post('/accounts/keyshare/', {'pub': PUB}).json()['token']
KeyShare.objects.update(polled_at=KeyShare.objects.get().created_at.replace(year=2000))
self.assertEqual(self.client.get('/accounts/keyshare/pending/').json()['requests'], [])
self.client.get(f'/accounts/keyshare/{token}/') # asks again
self.assertEqual(len(self.client.get('/accounts/keyshare/pending/').json()['requests']), 1)
def test_decline(self):
token = self._post('/accounts/keyshare/', {'pub': PUB}).json()['token']
self._post(f'/accounts/keyshare/{token}/answer/', {'decline': True})
self.assertEqual(self.client.get(f'/accounts/keyshare/{token}/').json(), {'state': 'declined'})
def test_bad_input_is_refused(self):
self.assertEqual(self._post('/accounts/keyshare/', {'pub': 'nope'}).status_code, 400)
token = self._post('/accounts/keyshare/', {'pub': PUB}).json()['token']
bad = {'pub': PUB, 'iv': base64.b64encode(b'i' * 12).decode(), 'ct': base64.b64encode(b'c' * 10).decode()}
self.assertEqual(self._post(f'/accounts/keyshare/{token}/answer/', bad).status_code, 400)
def test_other_accounts_see_nothing(self):
token = self._post('/accounts/keyshare/', {'pub': PUB}).json()['token']
other = User.objects.create_user(username='bob', password='pw12345678')
self.client.force_login(other)
self.assertEqual(self.client.get('/accounts/keyshare/pending/').json()['requests'], [])
self.assertEqual(self._post(f'/accounts/keyshare/{token}/answer/', {'decline': True}).status_code, 404)
self.client.logout()
self.assertEqual(self.client.get('/accounts/keyshare/pending/').status_code, 401)
def test_open_requests_are_limited_and_run_out(self):
tokens = [self._post('/accounts/keyshare/', {'pub': PUB}).json()['token'] for _ in range(7)]
self.assertEqual(KeyShare.objects.count(), 5)
self.assertEqual(self.client.get(f'/accounts/keyshare/{tokens[0]}/').json(), {'state': 'expired'})
self.assertEqual(self.client.get(f'/accounts/keyshare/{tokens[-1]}/').json(), {'state': 'pending'})
KeyShare.objects.update(created_at=KeyShare.objects.first().created_at.replace(year=2000))
self.assertEqual(self.client.get('/accounts/keyshare/pending/').json()['requests'], [])
self.assertEqual(self._post('/accounts/keyshare/', {'pub': PUB}).status_code, 200)

139
accounts/tests.py
View file

@ -85,12 +85,6 @@ class SyncSnapshotTests(TestCase):
resp = self.client.get('/api/sync/')
self.assertEqual(resp.status_code, 401)
def test_only_books(self):
resp = self.client.get('/api/sync/?only=books', HTTP_AUTHORIZATION=f'Bearer {self.token.token}')
data = resp.json()
self.assertEqual(set(data), {'server_time', 'server_now', 'books', 'book_progress', 'book_highlights', 'book_bookmarks'})
self.assertEqual(data['book_highlights'][0]['ct'], 'hl-ct')
def test_snapshot_shape_and_data(self):
resp = self.client.get('/api/sync/', HTTP_AUTHORIZATION=f'Bearer {self.token.token}')
self.assertEqual(resp.status_code, 200)
@ -546,136 +540,3 @@ class CloudFetchCachingTests(TestCase):
with patch('books.webdav.fetch_file', return_value=b'EPUB-BYTES'):
resp = self.client.get(f'/books/cloud/{self.source.pk}/fetch/?path=Dune.epub')
self.assertEqual(resp['Cache-Control'], 'no-store')
class WebDAVEditTests(TestCase):
def setUp(self):
self.user = User.objects.create_user(username='alice', password='pw12345678')
self.other = User.objects.create_user(username='bob', password='pw12345678')
self.source = WebDAVSource.objects.create(
user=self.user, label='cloud', base_url='https://cloud.example.com',
username='alice', password='app-pw',
)
def _edit(self, pk, **data):
form = {'label': 'cloud', 'base_url': 'https://cloud.example.com', 'username': 'alice', 'root_path': ''}
form.update(data)
with patch('accounts.views.list_directory', return_value=[]):
return self.client.post(f'/accounts/webdav/{pk}/edit/', form)
def test_sets_start_folder_and_keeps_password(self):
self.client.force_login(self.user)
resp = self._edit(self.source.pk, root_path='Books', password='')
self.assertEqual(resp.status_code, 302)
self.source.refresh_from_db()
self.assertEqual(self.source.root_path, 'Books')
self.assertEqual(self.source.password, 'app-pw')
def test_new_password_replaces_the_old_one(self):
self.client.force_login(self.user)
self._edit(self.source.pk, password='new-pw')
self.source.refresh_from_db()
self.assertEqual(self.source.password, 'new-pw')
def test_name_and_url_are_required(self):
self.client.force_login(self.user)
self._edit(self.source.pk, label='', root_path='Books')
self.source.refresh_from_db()
self.assertEqual(self.source.root_path, '')
def test_foreign_source_is_not_editable(self):
self.client.force_login(self.other)
self._edit(self.source.pk, root_path='x')
self.source.refresh_from_db()
self.assertEqual(self.source.root_path, '')
def test_settings_page_does_not_show_the_password(self):
self.client.force_login(self.user)
html = self.client.get('/accounts/settings/').content.decode()
self.assertIn(f'/accounts/webdav/{self.source.pk}/edit/', html)
self.assertNotIn('app-pw', html)
class CloudInboxTests(TestCase):
"""An "Eingang" source (docs/library.md): its new books are handed out once, claimed for
one device for ten minutes, and not again once a device took them."""
def setUp(self):
self.user = User.objects.create_user('inbox', password='pw12345678')
self.client.force_login(self.user)
self.src = WebDAVSource.objects.create(user=self.user, label='Bücher', base_url='https://b.example.org/dav/',
username='d', password='x', inbox=True)
WebDAVSource.objects.create(user=self.user, label='andere', base_url='https://c.example.org/dav/')
def listing(self, path=''):
if path == '':
return [{'name': 'Sub', 'path': 'Sub', 'is_dir': True, 'size': 0, 'modified': '', 'is_book': False},
{'name': 'A - Eins (2020).epub', 'path': 'A - Eins (2020).epub', 'is_dir': False,
'size': 100, 'modified': 'Mon', 'is_book': True},
{'name': 'notiz.txt', 'path': 'notiz.txt', 'is_dir': False, 'size': 1, 'modified': '', 'is_book': False}]
return [{'name': 'B.pdf', 'path': 'Sub/B.pdf', 'is_dir': False, 'size': 200, 'modified': '', 'is_book': True}]
def inbox(self):
with patch('books.webdav.list_directory', side_effect=lambda source, path='': self.listing(path)):
return self.client.get('/books/cloud/inbox/').json()
def done(self, path, result):
return self.client.post(f'/books/cloud/{self.src.pk}/done/', json.dumps({'path': path, 'result': result}),
content_type='application/json')
def test_new_books_once_then_claimed_then_done(self):
files = self.inbox()['files']
self.assertEqual(sorted(f['path'] for f in files), ['A - Eins (2020).epub', 'Sub/B.pdf'])
self.assertEqual(self.inbox()['files'], []) # claimed for this device: no one else gets them now
self.assertEqual(self.done('A - Eins (2020).epub', 'imported').status_code, 200)
self.assertEqual(self.done('Sub/B.pdf', 'failed').status_code, 200)
from accounts.models import WebDAVSeen
WebDAVSeen.objects.filter(path='Sub/B.pdf').update(claimed_at=None) # the claim ran out
self.assertEqual([f['path'] for f in self.inbox()['files']], ['Sub/B.pdf'])
def test_a_changed_file_counts_as_new(self):
self.inbox()
self.done('A - Eins (2020).epub', 'imported')
self.done('Sub/B.pdf', 'known')
self.assertEqual(self.inbox()['files'], [])
old = self.listing
self.listing = lambda path='': [dict(e, size=999) if e['path'].endswith('.epub') else e for e in old(path)]
self.assertEqual([f['path'] for f in self.inbox()['files']], ['A - Eins (2020).epub'])
def test_only_own_sources_and_valid_results(self):
other = User.objects.create_user('x', password='pw12345678')
foreign = WebDAVSource.objects.create(user=other, label='f', base_url='https://d.example.org/dav/', inbox=True)
self.assertEqual(self.client.post(f'/books/cloud/{foreign.pk}/done/', '{}', content_type='application/json').status_code, 404)
self.inbox()
self.assertEqual(self.done('A - Eins (2020).epub', 'whatever').status_code, 400)
self.client.logout()
self.assertEqual(self.client.get('/books/cloud/inbox/').status_code, 401)
def test_settings_switch_the_inbox(self):
r = self.client.post(f'/accounts/webdav/{self.src.pk}/edit/', {'label': 'Bücher', 'base_url': self.src.base_url})
self.assertEqual(r.status_code, 302)
self.src.refresh_from_db()
self.assertFalse(self.src.inbox)
class VaultTests(TestCase):
"""The vault keeps what the devices encrypted, per name; only for its owner."""
def test_keep_replace_remove(self):
u = User.objects.create_user('v', password='pw12345678')
self.client.force_login(u)
post = lambda body: self.client.post('/accounts/vault/ai/', json.dumps(body), content_type='application/json')
self.assertEqual(self.client.get('/accounts/vault/ai/').json(), {'ct': '', 'iv': ''})
self.assertEqual(post({'ct': 'abc', 'iv': 'i1'}).status_code, 200)
self.assertEqual(post({'ct': 'def', 'iv': 'i2'}).status_code, 200)
self.assertEqual(self.client.get('/accounts/vault/ai/').json(), {'ct': 'def', 'iv': 'i2'})
self.assertEqual(post({'iv': 'x'}).status_code, 400)
other = User.objects.create_user('w', password='pw12345678')
self.client.force_login(other)
self.assertEqual(self.client.get('/accounts/vault/ai/').json()['ct'], '')
self.client.force_login(u)
self.assertEqual(post({'ct': ''}).status_code, 200)
self.assertEqual(self.client.get('/accounts/vault/ai/').json()['ct'], '')
self.client.logout()
self.assertEqual(self.client.get('/accounts/vault/ai/').status_code, 401)

15
accounts/urls.py
View file

@ -1,6 +1,6 @@
from django.contrib.auth.views import LogoutView
from django.urls import path
from . import keyshare, oidc_views, views
from . import views
urlpatterns = [
path('register/', views.register, name='register'),
@ -17,19 +17,6 @@ urlpatterns = [
path('change-password/', views.change_password, name='change_password'),
path('api-token/regenerate/', views.regenerate_api_token, name='regenerate_api_token'),
path('webdav/add/', views.webdav_add, name='webdav_add'),
path('webdav/<int:pk>/edit/', views.webdav_edit, name='webdav_edit'),
path('webdav/<int:pk>/test/', views.webdav_test, name='webdav_test'),
path('webdav/<int:pk>/delete/', views.webdav_delete, name='webdav_delete'),
path('vault/<slug:name>/', views.vault, name='vault'),
path('oidc/start/', oidc_views.oidc_start, name='oidc_start'),
path('oidc/callback/', oidc_views.oidc_callback, name='oidc_callback'),
path('oidc/app/', oidc_views.app_config, name='oidc_app_config'),
path('oidc/app/login/', oidc_views.app_login, name='oidc_app_login'),
path('oidc/key/', oidc_views.oidc_key, name='oidc_key'),
path('oidc/key/callback/', oidc_views.oidc_key, name='oidc_key_callback'),
path('oidc/<int:pk>/unlink/', oidc_views.oidc_unlink, name='oidc_unlink'),
path('keyshare/', keyshare.create, name='keyshare_create'),
path('keyshare/pending/', keyshare.pending, name='keyshare_pending'),
path('keyshare/<str:token>/', keyshare.status, name='keyshare_status'),
path('keyshare/<str:token>/answer/', keyshare.answer, name='keyshare_answer'),
]

73
accounts/views.py
View file

@ -12,8 +12,7 @@ from django.views.decorators.http import require_http_methods
from radio import lastfm as lastfm_module
from .models import ApiToken, UserVault, WebDAVSource
from .oidc_views import oidc_context
from .models import ApiToken, WebDAVSource
from .webdav import WebDAVError, list_directory
User = get_user_model()
@ -57,7 +56,7 @@ def login_view(request):
else:
form = AuthenticationForm()
return render(request, 'accounts/login.html', {'form': form, **oidc_context(request)})
return render(request, 'accounts/login.html', {'form': form})
# ---------------------------------------------------------------------------
@ -79,7 +78,6 @@ def settings_view(request):
'password_form': PasswordChangeForm(request.user),
'api_token': getattr(request.user, 'api_token', None),
'webdav_sources': request.user.webdav_sources.all(),
**oidc_context(request),
}
return render(request, 'accounts/settings.html', context)
@ -219,7 +217,6 @@ def change_password(request):
'password_form_open': True,
'api_token': getattr(request.user, 'api_token', None),
'webdav_sources': request.user.webdav_sources.all(),
**oidc_context(request),
})
@ -286,7 +283,6 @@ def webdav_add(request):
username=request.POST.get('username', '').strip()[:200],
password=request.POST.get('password', '')[:500],
root_path=request.POST.get('root_path', '').strip()[:500],
inbox=request.POST.get('inbox') == '1',
)
level, message = _probe_source(source)
@ -294,36 +290,6 @@ def webdav_add(request):
return redirect('settings')
@login_required
@require_http_methods(['POST'])
def webdav_edit(request, pk):
source = WebDAVSource.objects.filter(pk=pk, user=request.user).first()
if not source:
messages.error(request, 'Verbindung nicht gefunden.')
return redirect('settings')
label = request.POST.get('label', '').strip()
base_url = request.POST.get('base_url', '').strip()
if not label or not base_url:
messages.error(request, 'Name und Server-URL sind erforderlich.')
return redirect('settings')
source.label = label[:100]
source.base_url = base_url[:500]
source.username = request.POST.get('username', '').strip()[:200]
source.root_path = request.POST.get('root_path', '').strip()[:500]
source.inbox = request.POST.get('inbox') == '1'
# The stored password is never sent to the page: empty means keep it.
password = request.POST.get('password', '')
if password:
source.password = password[:500]
source.save()
level, message = _probe_source(source)
messages.add_message(request, level, message)
return redirect('settings')
@login_required
@require_http_methods(['POST'])
def webdav_test(request, pk):
@ -344,38 +310,3 @@ def webdav_delete(request, pk):
if deleted:
messages.success(request, 'Verbindung entfernt.')
return redirect('settings')
# ---------------------------------------------------------------------------
# Vault: small device settings, end-to-end encrypted (docs/ai.md)
# ---------------------------------------------------------------------------
VAULT_MAX = 64 * 1024
@csrf_exempt
@require_http_methods(['GET', 'POST'])
def vault(request, name):
"""GET {ct, iv} ('' while there is none); POST {ct, iv} replaces it, {ct: ''} removes it.
The blob is encrypted on the devices with the book key; the server only keeps it."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
if request.method == 'GET':
v = UserVault.objects.filter(user=request.user, name=name).first()
return JsonResponse({'ct': v.ct if v else '', 'iv': v.iv if v else ''})
if len(request.body) > VAULT_MAX:
return JsonResponse({'error': 'too large'}, status=413)
try:
body = json.loads(request.body or b'{}')
except (json.JSONDecodeError, ValueError):
return JsonResponse({'error': 'invalid JSON'}, status=400)
ct = body.get('ct') if isinstance(body, dict) else None
iv = body.get('iv') if isinstance(body, dict) else None
if ct == '':
UserVault.objects.filter(user=request.user, name=name).delete()
return JsonResponse({'ok': True})
if not isinstance(ct, str) or not isinstance(iv, str) or not iv or len(iv) > 64:
return JsonResponse({'error': 'ct and iv required'}, status=400)
UserVault.objects.update_or_create(user=request.user, name=name, defaults={'ct': ct, 'iv': iv})
return JsonResponse({'ok': True})

43
books/migrations/0005_progress_last_read_wins.py
View file

@ -1,43 +0,0 @@
from django.db import migrations, models
def keep_what_we_know(apps, schema_editor):
"""The continue point so far was the furthest place: it is that, and it was
last saved at updated_at."""
EBookProgress = apps.get_model('books', 'EBookProgress')
for p in EBookProgress.objects.all().iterator():
p.furthest_fraction = p.scroll_fraction
p.furthest_anchor = p.position_anchor
p.read_at = int(p.updated_at.timestamp() * 1000)
p.save(update_fields=['furthest_fraction', 'furthest_anchor', 'read_at'])
class Migration(migrations.Migration):
dependencies = [
('books', '0004_ebook_is_read'),
]
operations = [
migrations.AddField(
model_name='ebookprogress',
name='read_at',
field=models.BigIntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='ebookprogress',
name='device',
field=models.CharField(blank=True, default='', max_length=40),
),
migrations.AddField(
model_name='ebookprogress',
name='furthest_fraction',
field=models.FloatField(default=0.0),
),
migrations.AddField(
model_name='ebookprogress',
name='furthest_anchor',
field=models.CharField(blank=True, default='', max_length=30),
),
migrations.RunPython(keep_what_we_know, migrations.RunPython.noop),
]

36
books/migrations/0006_covers_and_prefs.py
View file

@ -1,36 +0,0 @@
# Generated by Django 6.1 on 2026-10-16 18:21
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('books', '0005_progress_last_read_wins'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddField(
model_name='ebook',
name='cover_ct',
field=models.TextField(blank=True, default=''),
),
migrations.AddField(
model_name='ebook',
name='cover_iv',
field=models.CharField(blank=True, default='', max_length=32),
),
migrations.CreateModel(
name='LibraryPrefs',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('prefs_ct', models.TextField(blank=True, default='')),
('prefs_iv', models.CharField(blank=True, default='', max_length=32)),
('updated_at', models.DateTimeField(auto_now=True)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='library_prefs', to=settings.AUTH_USER_MODEL)),
],
),
]

23
books/migrations/0007_list_indexes.py
View file

@ -1,23 +0,0 @@
# Generated by Django 6.1 on 2026-10-19 13:24
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('books', '0006_covers_and_prefs'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddIndex(
model_name='ebook',
index=models.Index(fields=['user', 'uploaded_at', 'is_read'], name='ebook_list_idx'),
),
migrations.AddIndex(
model_name='ebook',
index=models.Index(fields=['user', 'cover_iv'], name='ebook_cover_idx'),
),
]

38
books/migrations/0008_ebook_file.py
View file

@ -1,38 +0,0 @@
# Generated by Django 6.1 on 2026-10-19 15:02
# The books' files move out of their rows into a table of their own (docs/database.md):
# copied in one statement, then the two columns are dropped (ALTER TABLE DROP COLUMN).
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('books', '0007_list_indexes'),
]
operations = [
migrations.CreateModel(
name='EBookFile',
fields=[
('book', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='file', serialize=False, to='books.ebook')),
('data_ct', models.TextField()),
('data_iv', models.CharField(max_length=32)),
],
),
migrations.RunSQL(
'INSERT INTO books_ebookfile (book_id, data_ct, data_iv) SELECT id, data_ct, data_iv FROM books_ebook',
reverse_sql='UPDATE books_ebook SET '
'data_ct = (SELECT f.data_ct FROM books_ebookfile f WHERE f.book_id = books_ebook.id), '
'data_iv = (SELECT f.data_iv FROM books_ebookfile f WHERE f.book_id = books_ebook.id)',
),
migrations.RemoveField(
model_name='ebook',
name='data_ct',
),
migrations.RemoveField(
model_name='ebook',
name='data_iv',
),
]

68
books/models.py
View file

@ -2,91 +2,27 @@ from django.db import models
from django.contrib.auth.models import User
class EBookManager(models.Manager):
def create(self, data_ct=None, data_iv=None, **kwargs):
"""A book, and with data_ct/data_iv its file (EBookFile) in the same go."""
book = super().create(**kwargs)
if data_ct is not None:
EBookFile.objects.create(book=book, data_ct=data_ct, data_iv=data_iv or '')
return book
class EBook(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ebooks')
meta_ct = models.TextField() # base64 AES-GCM ciphertext of {title, author, filename}
meta_iv = models.CharField(max_length=32) # hex IV for metadata
# The file itself is in EBookFile (book.file), not here (docs/database.md).
data_ct = models.TextField() # base64 AES-GCM ciphertext of raw EPUB bytes
data_iv = models.CharField(max_length=32) # hex IV for EPUB data
uploaded_at = models.DateTimeField(auto_now_add=True)
is_read = models.BooleanField(default=False)
# The cover, end-to-end encrypted like the rest (docs/library.md): a client that has
# the file extracts it, encrypts it with the user's key and sends it once.
cover_ct = models.TextField(blank=True, default='')
cover_iv = models.CharField(max_length=32, blank=True, default='')
objects = EBookManager()
class Meta:
ordering = ['uploaded_at']
# The list reads these from indexes (book_rows()); they made it fast while the file
# was still in this row (docs/database.md), and cost nothing now.
indexes = [
models.Index(fields=['user', 'uploaded_at', 'is_read'], name='ebook_list_idx'),
models.Index(fields=['user', 'cover_iv'], name='ebook_cover_idx'),
]
def __str__(self):
return f"EBook #{self.pk} (user={self.user_id})"
class EBookFile(models.Model):
"""A book's file, encrypted like everything else: base64 AES-GCM of the EPUB/PDF bytes.
In a table of its own (docs/database.md): with the file in the book's row, every change of a
title, a folder or "gelesen" rewrote megabytes, and the list read past every file."""
book = models.OneToOneField(EBook, on_delete=models.CASCADE, primary_key=True, related_name='file')
data_ct = models.TextField()
data_iv = models.CharField(max_length=32)
def __str__(self):
return f"EBookFile of #{self.book_id}"
def book_rows(user):
"""Every book of the user as {id, meta_ct, meta_iv, uploaded_at (ISO), is_read, has_cover},
oldest first — read without passing a book's file: meta_ct and meta_iv stand before it in
the row, the rest comes from the covering indexes ebook_list_idx and ebook_cover_idx."""
books = {b['id']: b for b in EBook.objects.filter(user=user).order_by().values('id', 'meta_ct', 'meta_iv')}
for book_id, at, read in EBook.objects.filter(user=user).order_by().values_list('id', 'uploaded_at', 'is_read'):
if book_id in books:
books[book_id].update(uploaded_at=at, is_read=read)
covered = set(EBook.objects.filter(user=user).exclude(cover_iv='').order_by().values_list('id', flat=True))
out = sorted((b for b in books.values() if 'uploaded_at' in b), key=lambda b: (b['uploaded_at'], b['id']))
for b in out:
b['uploaded_at'] = b['uploaded_at'].isoformat()
b['has_cover'] = b['id'] in covered
return out
class LibraryPrefs(models.Model):
"""The library's own settings, encrypted with the user's key (the folder order:
folder names are private, docs/library.md). One blob, replaced as a whole."""
user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='library_prefs')
prefs_ct = models.TextField(blank=True, default='')
prefs_iv = models.CharField(max_length=32, blank=True, default='')
updated_at = models.DateTimeField(auto_now=True)
class EBookProgress(models.Model):
"""Where to continue a book: the place it was last read, on any device
(docs/progress-sync.md), plus the furthest place ever reached."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ebook_progress')
book = models.ForeignKey(EBook, on_delete=models.CASCADE, related_name='progress')
scroll_fraction = models.FloatField(default=0.0)
position_anchor = models.CharField(max_length=30, blank=True, default='')
# When the reader was at that place (ms since epoch, server clock); null before 2026-10-12.
read_at = models.BigIntegerField(null=True, blank=True)
device = models.CharField(max_length=40, blank=True, default='')
furthest_fraction = models.FloatField(default=0.0)
furthest_anchor = models.CharField(max_length=30, blank=True, default='')
updated_at = models.DateTimeField(auto_now=True)
class Meta:

292
books/tests.py
View file

@ -1,292 +0,0 @@
"""The reading position: the place last read wins (docs/progress-sync.md)."""
import json
from unittest.mock import patch
from django.contrib.auth.models import User
from django.test import TestCase
from accounts.models import ApiToken
from .models import EBook, EBookProgress
NOW = 1_760_270_000_000 # the server's clock in these tests (ms)
MIN = 60_000
class ProgressTests(TestCase):
def setUp(self):
self.user = User.objects.create_user(username='alice', password='pw12345678')
self.auth = {'HTTP_AUTHORIZATION': f'Bearer {ApiToken.objects.create(user=self.user).token}'}
self.book = EBook.objects.create(user=self.user, meta_ct='m', meta_iv='i', data_ct='d', data_iv='i')
self.now = NOW
clock = patch('books.views._now_ms', side_effect=lambda: self.now)
clock.start()
self.addCleanup(clock.stop)
def save(self, anchor, frac=None, read_at=None, sent_at=None, device='Handy', **extra):
body = {'scroll_fraction': frac if frac is not None else 0.1, 'position_anchor': anchor, **extra}
if read_at is not None:
body.update(read_at=read_at, sent_at=sent_at if sent_at is not None else self.now, device=device)
resp = self.client.post(f'/books/{self.book.id}/progress/', json.dumps(body),
content_type='application/json', **self.auth)
self.assertEqual(resp.status_code, 200)
return resp.json()
def state(self):
return EBookProgress.objects.get(book=self.book)
# -- the new rule
def test_newer_place_wins_even_if_it_is_earlier_in_the_book(self):
self.save('300:0.5', read_at=NOW - 10 * MIN, device='Handy')
r = self.save('12:0.0', read_at=NOW - 1 * MIN, device='diora Desktop')
self.assertTrue(r['stored'])
p = self.state()
self.assertEqual((p.position_anchor, p.device, p.read_at), ('12:0.0', 'diora Desktop', NOW - MIN))
self.assertEqual(p.furthest_anchor, '300:0.5')
def test_an_older_write_arriving_late_does_not_win(self):
# The phone read offline at 10:00 and syncs at 12:00; the desktop read at 11:00.
self.save('40:0.0', read_at=NOW - 60 * MIN, device='diora Desktop')
r = self.save('90:0.0', read_at=NOW - 120 * MIN, device='Handy')
self.assertFalse(r['stored'])
self.assertEqual(r['position_anchor'], '40:0.0')
p = self.state()
self.assertEqual((p.position_anchor, p.device), ('40:0.0', 'diora Desktop'))
self.assertEqual(p.furthest_anchor, '90:0.0', 'the late write still counts as reached')
def test_resaving_an_unchanged_place_keeps_its_time_and_cannot_win(self):
self.save('50:0.0', read_at=NOW - 5 * MIN, device='Handy')
self.save('70:0.0', read_at=NOW - 2 * MIN, device='diora Desktop')
# The phone, idle since, saves its place again on going to the background.
self.now += MIN
r = self.save('50:0.0', read_at=NOW - 5 * MIN, device='Handy')
self.assertFalse(r['stored'])
self.assertEqual(self.state().position_anchor, '70:0.0')
def test_the_device_clock_error_is_taken_out(self):
# A tolino 3 hours behind: it read one minute ago by its clock, and sends now by its clock.
behind = 3 * 60 * MIN
self.save('10:0.0', read_at=NOW - 30 * MIN, device='Handy')
r = self.save('20:0.0', read_at=NOW - behind - MIN, sent_at=NOW - behind, device='tolino')
self.assertTrue(r['stored'])
self.assertEqual(self.state().read_at, NOW - MIN)
def test_nothing_wins_from_the_future(self):
self.save('10:0.0', read_at=NOW + 24 * 60 * MIN, sent_at=NOW)
self.assertEqual(self.state().read_at, NOW)
self.now += MIN
self.assertTrue(self.save('11:0.0', read_at=self.now, sent_at=self.now)['stored'])
def test_equal_times_take_the_new_write(self):
self.save('10:0.0', read_at=NOW - MIN)
self.assertTrue(self.save('11:0.0', read_at=NOW - MIN)['stored'])
def test_force_sets_the_place_now(self):
self.save('300:0.0', read_at=NOW - MIN)
self.assertTrue(self.save('0:0.0', frac=0.0, force=True)['stored'])
p = self.state()
self.assertEqual((p.position_anchor, p.read_at), ('0:0.0', NOW))
self.assertEqual(p.furthest_anchor, '300:0.0')
def test_furthest_only_grows(self):
self.save('80:0.0', read_at=NOW - 3 * MIN)
self.save('20:0.0', read_at=NOW - 2 * MIN)
self.save('81:0.0', read_at=NOW - 9 * MIN)
p = self.state()
self.assertEqual((p.position_anchor, p.furthest_anchor), ('20:0.0', '81:0.0'))
def test_pdfs_compare_by_fraction(self):
self.save('', frac=0.7, read_at=NOW - 2 * MIN)
self.save('', frac=0.3, read_at=NOW - MIN)
p = self.state()
self.assertEqual((p.scroll_fraction, p.furthest_fraction), (0.3, 0.7))
def test_response_is_the_state_after_the_write(self):
r = self.save('5:0.25', frac=0.05, read_at=NOW - MIN, device='Browser')
self.assertEqual(r['server_now'], NOW)
for key in ('scroll_fraction', 'position_anchor', 'read_at', 'device',
'furthest_fraction', 'furthest_anchor', 'last_read'):
self.assertIn(key, r)
self.assertEqual(r['read_at'], NOW - MIN)
def test_implausible_times_count_as_an_old_app(self):
self.save('10:0.0', read_at=NOW - MIN)
# read_at=True or seconds instead of ms are not times: only forward, like an old app.
self.assertFalse(self.save('5:0.0', read_at=True)['stored'])
self.assertFalse(self.save('5:0.0', read_at=1_760_270_000)['stored'])
# -- apps from before the change (no read_at)
def test_old_apps_only_move_forward(self):
self.save('50:0.0', read_at=NOW - MIN)
self.assertFalse(self.save('20:0.0')['stored'])
self.assertTrue(self.save('60:0.0')['stored'])
self.assertEqual(self.state().read_at, NOW)
def test_old_apps_resaving_the_same_place_change_nothing(self):
self.save('50:0.0', read_at=NOW - 5 * MIN, device='Handy')
self.now += MIN
self.assertFalse(self.save('50:0.0')['stored'])
p = self.state()
self.assertEqual((p.read_at, p.device), (NOW - 5 * MIN, 'Handy'))
def test_first_write_of_an_old_app_is_stored(self):
self.assertTrue(self.save('0:0.0', frac=0.0)['stored'])
def test_old_force_still_works(self):
self.save('50:0.0', read_at=NOW - MIN)
self.assertTrue(self.save('1:0.0', force=True)['stored'])
# -- reading
def test_book_list_and_snapshot_carry_the_state(self):
self.save('7:0.5', frac=0.2, read_at=NOW - MIN, device='Browser')
resp = self.client.get('/books/', **self.auth)
b = resp.json()[0]
self.assertEqual(resp['X-Server-Now'], str(NOW))
self.assertEqual((b['position_anchor'], b['read_at'], b['device'], b['furthest_anchor']),
('7:0.5', NOW - MIN, 'Browser', '7:0.5'))
self.assertTrue(b['last_read'].startswith('2025-10-12T'))
snap = self.client.get('/api/sync/?only=books', **self.auth).json()
p = snap['book_progress'][0]
self.assertEqual((p['book_id'], p['read_at'], p['device']), (self.book.id, NOW - MIN, 'Browser'))
self.assertIn('server_now', snap)
def test_rows_from_before_have_their_save_time_as_read_at(self):
p = EBookProgress.objects.create(user=self.user, book=self.book, scroll_fraction=0.4, position_anchor='9:0.0')
b = self.client.get('/books/', **self.auth).json()[0]
self.assertEqual(b['read_at'], int(p.updated_at.timestamp() * 1000))
def test_a_never_opened_book(self):
b = self.client.get('/books/', **self.auth).json()[0]
self.assertEqual((b['read_at'], b['last_read'], b['position_anchor']), (None, None, ''))
class CoverAndPrefsTests(TestCase):
"""Covers and the library's settings travel encrypted (docs/library.md)."""
def setUp(self):
self.user = User.objects.create_user(username='alice', password='pw12345678')
self.auth = {'HTTP_AUTHORIZATION': f'Bearer {ApiToken.objects.create(user=self.user).token}'}
self.book = EBook.objects.create(user=self.user, meta_ct='m', meta_iv='i', data_ct='d', data_iv='i')
def post(self, url, body, **auth):
return self.client.post(url, json.dumps(body), content_type='application/json', **(auth or self.auth))
def test_a_cover_is_kept_and_listed(self):
url = f'/books/{self.book.id}/cover/'
self.assertEqual(self.client.get(url, **self.auth).status_code, 404)
self.assertFalse(self.client.get('/books/', **self.auth).json()[0]['has_cover'])
self.assertEqual(self.post(url, {'cover_ct': 'Q0lQSEVS', 'cover_iv': 'a1b2c3d4e5f60718293a4b5c'}).status_code, 200)
self.assertEqual(self.client.get(url, **self.auth).json(),
{'cover_ct': 'Q0lQSEVS', 'cover_iv': 'a1b2c3d4e5f60718293a4b5c'})
self.assertTrue(self.client.get('/books/', **self.auth).json()[0]['has_cover'])
def test_bad_or_huge_covers_are_refused(self):
url = f'/books/{self.book.id}/cover/'
self.assertEqual(self.post(url, {'cover_ct': 'x', 'cover_iv': 'not hex!'}).status_code, 400)
self.assertEqual(self.post(url, {'cover_ct': 'x' * 300_000, 'cover_iv': 'a1b2c3d4'}).status_code, 400)
def test_covers_are_ones_own(self):
bob = User.objects.create_user(username='bob', password='pw12345678')
bob_auth = {'HTTP_AUTHORIZATION': f'Bearer {ApiToken.objects.create(user=bob).token}'}
url = f'/books/{self.book.id}/cover/'
self.assertEqual(self.post(url, {'cover_ct': 'x', 'cover_iv': 'a1b2c3d4'}, **bob_auth).status_code, 404)
self.assertEqual(self.client.get(url, **bob_auth).status_code, 404)
def test_prefs_are_one_encrypted_blob(self):
self.assertEqual(self.client.get('/books/prefs/', **self.auth).json(), {'prefs_ct': '', 'prefs_iv': ''})
self.post('/books/prefs/', {'prefs_ct': 'AAA', 'prefs_iv': 'a1b2c3d4e5f6'})
self.post('/books/prefs/', {'prefs_ct': 'BBB', 'prefs_iv': 'a1b2c3d4e5f7'})
self.assertEqual(self.client.get('/books/prefs/', **self.auth).json(), {'prefs_ct': 'BBB', 'prefs_iv': 'a1b2c3d4e5f7'})
class BookRowsReadPastNoFileTests(TestCase):
"""The list must not read a column that stands after the file (data_ct) in a book's row:
SQLite would walk through the whole file to get there (docs/database.md)."""
def test_the_list_queries_use_covering_indexes(self):
from django.contrib.auth.models import User
from django.db import connection
from django.test.utils import CaptureQueriesContext
from books.models import EBook, book_rows
user = User.objects.create_user('rows', password='x')
EBook.objects.create(user=user, meta_ct='m', meta_iv='i', data_ct='d' * 100000, data_iv='j', cover_iv='c', cover_ct='cc')
EBook.objects.create(user=user, meta_ct='m2', meta_iv='i2', data_ct='d', data_iv='j')
with CaptureQueriesContext(connection) as q:
rows = book_rows(user)
self.assertEqual([r['meta_ct'] for r in rows], ['m', 'm2'])
self.assertEqual([r['has_cover'] for r in rows], [True, False])
late = ('uploaded_at', 'is_read', 'cover_iv', 'cover_ct', 'data_ct', 'data_iv')
with connection.cursor() as cur:
for query in q.captured_queries:
sql = query['sql']
if any(col in sql for col in late):
cur.execute('EXPLAIN QUERY PLAN ' + sql.replace('%s', '?'), [user.id] if '?' in sql or '%s' in sql else [])
plan = ' '.join(str(r) for r in cur.fetchall())
self.assertIn('COVERING INDEX', plan, sql)
class EBookFileTests(TestCase):
"""The file lives in EBookFile; upload, data, replace-data and rekey go there."""
def setUp(self):
from django.contrib.auth.models import User
self.user = User.objects.create_user('filer', password='pw-filer-123')
self.client.force_login(self.user)
def test_upload_read_replace_rekey_delete(self):
import base64, json
from books.models import EBook, EBookFile
data = base64.b64encode(b'x' * 100).decode()
r = self.client.post('/books/upload/', json.dumps({'meta_ct': 'm', 'meta_iv': 'i', 'data_ct': data, 'data_iv': 'd'}),
content_type='application/json')
pk = r.json()['id']
self.assertEqual(EBookFile.objects.get(book_id=pk).data_ct, data)
self.assertEqual(self.client.get(f'/books/{pk}/data/').json(), {'data_ct': data, 'data_iv': 'd'})
self.client.post(f'/books/{pk}/replace-data/', json.dumps({'data_ct': 'bmV3', 'data_iv': 'd2'}), content_type='application/json')
self.assertEqual(self.client.get(f'/books/{pk}/data/').json()['data_ct'], 'bmV3')
self.client.post(f'/books/{pk}/rekey/', json.dumps({'meta_ct': 'm3', 'meta_iv': 'i3', 'data_ct': 'cmVr', 'data_iv': 'd3'}),
content_type='application/json')
self.assertEqual(EBook.objects.get(pk=pk).meta_ct, 'm3')
self.assertEqual(EBookFile.objects.get(book_id=pk).data_iv, 'd3')
self.client.post(f'/books/{pk}/delete/')
self.assertFalse(EBookFile.objects.filter(book_id=pk).exists())
def test_someone_elses_file_is_not_found(self):
from django.contrib.auth.models import User
from books.models import EBook
other = User.objects.create_user('other-filer', password='pw-other-123')
b = EBook.objects.create(user=other, meta_ct='m', meta_iv='i', data_ct='ZA==', data_iv='i')
self.assertEqual(self.client.get(f'/books/{b.id}/data/').status_code, 404)
class StaleMetaTests(TestCase):
"""A meta write that starts from an older meta (base_iv) is refused with the current one."""
def setUp(self):
from django.contrib.auth.models import User
from books.models import EBook
self.user = User.objects.create_user('metauser', password='pw-meta-123')
self.client.force_login(self.user)
self.book = EBook.objects.create(user=self.user, meta_ct='m1', meta_iv='aa', data_ct='ZA==', data_iv='i')
def post(self, body):
import json
return self.client.post(f'/books/{self.book.id}/meta/', json.dumps(body), content_type='application/json')
def test_current_base_is_taken(self):
self.assertEqual(self.post({'meta_ct': 'm2', 'meta_iv': 'bb', 'base_iv': 'aa'}).status_code, 200)
self.book.refresh_from_db()
self.assertEqual((self.book.meta_ct, self.book.meta_iv), ('m2', 'bb'))
def test_old_base_is_refused_with_the_current_meta(self):
self.post({'meta_ct': 'm2', 'meta_iv': 'bb', 'base_iv': 'aa'})
r = self.post({'meta_ct': 'old', 'meta_iv': 'cc', 'base_iv': 'aa'})
self.assertEqual(r.status_code, 409)
self.assertEqual((r.json()['meta_ct'], r.json()['meta_iv']), ('m2', 'bb'))
self.book.refresh_from_db()
self.assertEqual(self.book.meta_ct, 'm2')
def test_without_base_as_before(self):
self.assertEqual(self.post({'meta_ct': 'm3', 'meta_iv': 'dd'}).status_code, 200)

4
books/urls.py
View file

@ -5,8 +5,6 @@ urlpatterns = [
path('', views.book_list, name='book_list'),
path('upload/', views.upload_book, name='upload_book'),
path('cloud/sources/', webdav.cloud_sources, name='cloud_sources'),
path('cloud/inbox/', webdav.cloud_inbox, name='cloud_inbox'),
path('cloud/<int:pk>/done/', webdav.cloud_done, name='cloud_done'),
path('cloud/<int:pk>/browse/', webdav.cloud_browse, name='cloud_browse'),
path('cloud/<int:pk>/fetch/', webdav.cloud_fetch, name='cloud_fetch'),
path('metadata-lookup/', views.lookup_book_metadata, name='lookup_book_metadata'),
@ -14,8 +12,6 @@ urlpatterns = [
path('<int:pk>/delete/', views.delete_book, name='delete_book'),
path('<int:pk>/read/', views.set_book_read, name='set_book_read'),
path('<int:pk>/meta/', views.update_book_meta, name='update_book_meta'),
path('<int:pk>/cover/', views.book_cover, name='book_cover'),
path('prefs/', views.library_prefs, name='library_prefs'),
path('<int:pk>/replace-data/', views.replace_book_data, name='replace_book_data'),
path('<int:pk>/rekey/', views.rekey_book, name='rekey_book'),
path('<int:pk>/progress/', views.save_progress, name='save_book_progress'),

238
books/views.py
View file

@ -1,18 +1,15 @@
import base64
import json
import re
import time
import xml.etree.ElementTree as ET
from datetime import datetime, timezone as dt_timezone
import requests
from django.conf import settings
from django.db import transaction
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from .models import EBook, EBookFile, EBookProgress, EBookHighlights, EBookBookmarks, LibraryPrefs, book_rows
from .models import EBook, EBookProgress, EBookHighlights, EBookBookmarks
def _require_auth(request):
@ -250,40 +247,11 @@ def _progress_is_further(new_anchor, new_frac, old_anchor, old_frac):
"""True if the new reading position is at least as far into the book as the
old one. Compares by anchor block index (precise, decoupled from layout);
falls back to scroll_fraction only when an anchor is missing (PDFs)."""
return _cmp_progress(new_anchor, new_frac, old_anchor, old_frac) >= 0
def _cmp_progress(a_anchor, a_frac, b_anchor, b_frac):
"""> 0 if A is further into the book than B, < 0 if behind, 0 if level."""
ab, ai = _anchor_parts(a_anchor)
bb, bi = _anchor_parts(b_anchor)
if ab >= 0 and bb >= 0:
if ab != bb:
return ab - bb
return (ai > bi) - (ai < bi)
return (a_frac > b_frac) - (a_frac < b_frac)
def _now_ms():
return int(time.time() * 1000)
def progress_state(p):
"""A book's reading state as every client sees it (docs/progress-sync.md);
p may be None (never opened)."""
if p is None:
return {'scroll_fraction': 0.0, 'position_anchor': '', 'read_at': None, 'device': '',
'furthest_fraction': 0.0, 'furthest_anchor': '', 'last_read': None}
read_at = p.read_at if p.read_at is not None else int(p.updated_at.timestamp() * 1000)
return {
'scroll_fraction': p.scroll_fraction,
'position_anchor': p.position_anchor,
'read_at': read_at,
'device': p.device,
'furthest_fraction': p.furthest_fraction,
'furthest_anchor': p.furthest_anchor,
'last_read': datetime.fromtimestamp(read_at / 1000, tz=dt_timezone.utc).isoformat(),
}
nb, ni = _anchor_parts(new_anchor)
ob, oi = _anchor_parts(old_anchor)
if nb >= 0 and ob >= 0:
return ni >= oi if nb == ob else nb >= ob
return new_frac >= old_frac
@require_http_methods(['GET'])
@ -291,18 +259,26 @@ def book_list(request):
err = _require_auth(request)
if err:
return err
books = book_rows(request.user)
progress_map = {p.book_id: p for p in EBookProgress.objects.filter(user=request.user)}
books = list(
request.user.ebooks.values('id', 'meta_ct', 'meta_iv', 'uploaded_at', 'is_read')
)
for b in books:
b['uploaded_at'] = b['uploaded_at'].isoformat()
# Include saved scroll_fraction for each book
progress_map = {
p.book_id: (p.scroll_fraction, p.updated_at, p.position_anchor)
for p in EBookProgress.objects.filter(user=request.user)
}
highlighted_ids = set(
EBookHighlights.objects.filter(user=request.user).values_list('book_id', flat=True)
)
for b in books:
b.update(progress_state(progress_map.get(b['id'])))
prog = progress_map.get(b['id'])
b['scroll_fraction'] = prog[0] if prog else 0.0
b['last_read'] = prog[1].isoformat() if prog else None
b['position_anchor'] = prog[2] if prog else ''
b['has_highlights'] = b['id'] in highlighted_ids
response = JsonResponse(books, safe=False)
# The list is a bare array (old clients expect that): the server clock goes in a header.
response['X-Server-Now'] = str(_now_ms())
return response
return JsonResponse(books, safe=False)
@csrf_exempt
@ -350,77 +326,12 @@ def update_book_meta(request, pk):
if not meta_ct or not meta_iv:
return JsonResponse({'error': 'meta_ct, meta_iv required'}, status=400)
# base_iv: the IV of the meta the client changed. Another one now (changed elsewhere
# since): refused with the current meta, for the client to apply its change to that.
# A queued write from a device with an old list (a title or folder set meanwhile)
# used to put the old meta back. Clients that don't send base_iv write as before.
base_iv = body.get('base_iv')
if base_iv and base_iv != book.meta_iv:
return JsonResponse({'error': 'stale', 'meta_ct': book.meta_ct, 'meta_iv': book.meta_iv}, status=409)
book.meta_ct = meta_ct
book.meta_iv = meta_iv
book.save(update_fields=['meta_ct', 'meta_iv'])
return JsonResponse({'ok': True})
MAX_COVER = 200_000 # characters of base64: a 300×450 JPEG is far below
MAX_PREFS = 100_000
def _hex_iv(value):
return isinstance(value, str) and 8 <= len(value) <= 32 and all(c in '0123456789abcdefABCDEF' for c in value)
@csrf_exempt
@require_http_methods(['GET', 'POST'])
def book_cover(request, pk):
"""A book's cover, encrypted on the device with the user's key (docs/library.md):
GET {cover_ct, cover_iv} (404 without one), POST the same to set it."""
err = _require_auth(request)
if err:
return err
book = EBook.objects.filter(pk=pk, user=request.user).only('id', 'cover_ct', 'cover_iv').first()
if book is None:
return JsonResponse({'error': 'not found'}, status=404)
if request.method == 'GET':
if not book.cover_ct:
return JsonResponse({'error': 'no cover'}, status=404)
return JsonResponse({'cover_ct': book.cover_ct, 'cover_iv': book.cover_iv})
try:
body = json.loads(request.body)
except (json.JSONDecodeError, ValueError):
return JsonResponse({'error': 'invalid JSON'}, status=400)
ct, iv = body.get('cover_ct'), body.get('cover_iv')
if not isinstance(ct, str) or not ct or len(ct) > MAX_COVER or not _hex_iv(iv):
return JsonResponse({'error': 'cover_ct, cover_iv required (cover at most 150 KB)'}, status=400)
book.cover_ct, book.cover_iv = ct, iv
book.save(update_fields=['cover_ct', 'cover_iv'])
return JsonResponse({'ok': True})
@csrf_exempt
@require_http_methods(['GET', 'POST'])
def library_prefs(request):
"""The library's settings, encrypted with the user's key ({folder_order: [...]}):
GET {prefs_ct, prefs_iv} ('' while there are none), POST the same, replacing them."""
err = _require_auth(request)
if err:
return err
if request.method == 'GET':
p = LibraryPrefs.objects.filter(user=request.user).first()
return JsonResponse({'prefs_ct': p.prefs_ct if p else '', 'prefs_iv': p.prefs_iv if p else ''})
try:
body = json.loads(request.body)
except (json.JSONDecodeError, ValueError):
return JsonResponse({'error': 'invalid JSON'}, status=400)
ct, iv = body.get('prefs_ct'), body.get('prefs_iv')
if not isinstance(ct, str) or not ct or len(ct) > MAX_PREFS or not _hex_iv(iv):
return JsonResponse({'error': 'prefs_ct, prefs_iv required'}, status=400)
LibraryPrefs.objects.update_or_create(user=request.user, defaults={'prefs_ct': ct, 'prefs_iv': iv})
return JsonResponse({'ok': True})
@csrf_exempt
@require_http_methods(['POST'])
def upload_book(request):
@ -449,17 +360,16 @@ def upload_book(request):
return JsonResponse({'error': 'invalid base64 in data_ct'}, status=400)
if raw_size > max_bytes:
return JsonResponse({'error': f'file too large (max {settings.EBOOK_MAX_BYTES // (1024 * 1024)} MB)'}, status=400)
return JsonResponse({'error': 'file too large (max 50 MB)'}, status=400)
with transaction.atomic():
book = EBook.objects.create(
user=request.user,
meta_ct=meta_ct,
meta_iv=meta_iv,
data_ct=data_ct,
data_iv=data_iv,
)
EBookProgress.objects.create(user=request.user, book=book)
book = EBook.objects.create(
user=request.user,
meta_ct=meta_ct,
meta_iv=meta_iv,
data_ct=data_ct,
data_iv=data_iv,
)
EBookProgress.objects.create(user=request.user, book=book)
return JsonResponse({'ok': True, 'id': book.id})
@ -469,10 +379,12 @@ def get_book_data(request, pk):
if err:
return err
f = EBookFile.objects.filter(book_id=pk, book__user=request.user).first()
if f is None:
try:
book = EBook.objects.get(pk=pk, user=request.user)
except EBook.DoesNotExist:
return JsonResponse({'error': 'not found'}, status=404)
return JsonResponse({'data_ct': f.data_ct, 'data_iv': f.data_iv})
return JsonResponse({'data_ct': book.data_ct, 'data_iv': book.data_iv})
@csrf_exempt
@ -500,12 +412,14 @@ def replace_book_data(request, pk):
if not all([data_ct, data_iv]):
return JsonResponse({'error': 'data_ct and data_iv required'}, status=400)
with transaction.atomic():
EBookFile.objects.update_or_create(book=book, defaults={'data_ct': data_ct, 'data_iv': data_iv})
if meta_ct and meta_iv:
book.meta_ct = meta_ct
book.meta_iv = meta_iv
book.save(update_fields=['meta_ct', 'meta_iv'])
update_fields = ['data_ct', 'data_iv']
book.data_ct = data_ct
book.data_iv = data_iv
if meta_ct and meta_iv:
book.meta_ct = meta_ct
book.meta_iv = meta_iv
update_fields += ['meta_ct', 'meta_iv']
book.save(update_fields=update_fields)
return JsonResponse({'ok': True})
@ -534,11 +448,11 @@ def rekey_book(request, pk):
if not all([meta_ct, meta_iv, data_ct, data_iv]):
return JsonResponse({'error': 'meta_ct, meta_iv, data_ct, data_iv required'}, status=400)
with transaction.atomic():
EBookFile.objects.update_or_create(book=book, defaults={'data_ct': data_ct, 'data_iv': data_iv})
book.meta_ct = meta_ct
book.meta_iv = meta_iv
book.save(update_fields=['meta_ct', 'meta_iv'])
book.meta_ct = meta_ct
book.meta_iv = meta_iv
book.data_ct = data_ct
book.data_iv = data_iv
book.save(update_fields=['meta_ct', 'meta_iv', 'data_ct', 'data_iv'])
return JsonResponse({'ok': True})
@ -583,50 +497,22 @@ def save_progress(request, pk):
if isinstance(raw_anchor, str) and re.fullmatch(r'\d{1,7}:\d(\.\d{1,6})?', raw_anchor):
position_anchor = raw_anchor
force = body.get('force') is True
device = body.get('device') if isinstance(body.get('device'), str) else ''
now = _now_ms()
read_at = _client_ms(body.get('read_at'))
sent_at = _client_ms(body.get('sent_at'))
force = bool(body.get('force', False))
with transaction.atomic():
progress, created = EBookProgress.objects.select_for_update().get_or_create(user=request.user, book=book)
if created:
progress.read_at = None
if force:
when, store = now, True
elif read_at is not None:
# The device's clock may be off: move its time onto ours by the
# difference at sending, and nothing counts as later than now.
when = read_at + (now - sent_at) if sent_at is not None else read_at
when = min(when, now)
store = progress.read_at is None or when >= progress.read_at
else:
# An app from before 2026-10-12: only forward, as it used to be — but
# strictly, so re-saving the same place changes nothing.
when = now
store = created or _cmp_progress(position_anchor, scroll_fraction,
progress.position_anchor, progress.scroll_fraction) > 0
if store:
progress.scroll_fraction = scroll_fraction
progress.position_anchor = position_anchor
progress.read_at = when
progress.device = device[:40]
if _cmp_progress(position_anchor, scroll_fraction,
progress.furthest_anchor, progress.furthest_fraction) > 0 or created:
progress.furthest_fraction = scroll_fraction
progress.furthest_anchor = position_anchor
progress.save()
progress, _ = EBookProgress.objects.get_or_create(
user=request.user,
book=book,
)
# Always advance to the furthest-read position (by anchor block index, so a
# transiently wrong scroll_fraction can't freeze the position) unless the
# client explicitly forces a reset (e.g. "start over" button in the reader).
if force or _progress_is_further(position_anchor, scroll_fraction,
progress.position_anchor, progress.scroll_fraction):
progress.scroll_fraction = scroll_fraction
progress.position_anchor = position_anchor
progress.save(update_fields=['scroll_fraction', 'position_anchor', 'updated_at'])
return JsonResponse({'ok': True, 'stored': store, 'server_now': now, **progress_state(progress)})
def _client_ms(value):
"""A time from a client (ms since epoch) or None if it isn't a plausible one."""
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
value = int(value)
return value if 10 ** 12 <= value < 10 ** 14 else None
return JsonResponse({'ok': True})
@csrf_exempt

91
books/webdav.py
View file

@ -12,16 +12,12 @@ browser and only then POSTed to /books/upload/ as ciphertext, exactly like a
local upload — nothing readable is ever stored server-side.
"""
import json
from datetime import timedelta
from django.conf import settings
from django.http import HttpResponse, JsonResponse
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from accounts.models import WebDAVSeen, WebDAVSource
from accounts.models import WebDAVSource
from accounts.webdav import WebDAVError, WebDAVInputError, list_directory, fetch_file
from .views import _require_auth
@ -38,94 +34,11 @@ def cloud_sources(request):
sources = request.user.webdav_sources.all()
return JsonResponse({'sources': [
{'id': s.id, 'label': s.label, 'host': s.base_url, 'inbox': s.inbox}
{'id': s.id, 'label': s.label, 'host': s.base_url}
for s in sources
]})
# ---- "Eingang" (docs/library.md): new books of a source into the library by themselves ----
INBOX_DEPTH = 3 # sub-folders looked into
INBOX_MAX = 20 # files handed out per call
INBOX_CLAIM = timedelta(minutes=10)
def _inbox_files(source):
"""The source's book files, its sub-folders too (INBOX_DEPTH deep)."""
out, todo = [], [('', 0)]
while todo:
path, depth = todo.pop(0)
for e in list_directory(source, path):
if e['is_dir'] and depth < INBOX_DEPTH:
todo.append((e['path'], depth + 1))
elif e['is_book']:
out.append(e)
return out
@require_http_methods(['GET'])
def cloud_inbox(request):
"""The files of the user's "Eingang" sources not taken into the library yet, claimed for
this device for ten minutes (another device asking meanwhile doesn't get them):
{files: [{source, label, path, name, size, modified}], errors: [{source, label, error}]}.
A file that changed (other size) counts as new again."""
unauthorized = _require_auth(request)
if unauthorized:
return unauthorized
now = timezone.now()
files, errors = [], []
for source in request.user.webdav_sources.filter(inbox=True):
try:
entries = _inbox_files(source)
except (WebDAVError, WebDAVInputError) as exc:
errors.append({'source': source.id, 'label': source.label, 'error': str(exc)})
continue
seen = {s.path: s for s in source.seen.all()}
for e in entries:
if len(files) >= INBOX_MAX:
break
s = seen.get(e['path'])
if s and s.size == e['size']:
if s.done_at or (s.claimed_at and now - s.claimed_at < INBOX_CLAIM):
continue
WebDAVSeen.objects.update_or_create(
source=source, path=e['path'][:1000],
defaults={'size': e['size'], 'modified': e['modified'][:100], 'claimed_at': now,
'done_at': None, 'result': ''})
files.append({'source': source.id, 'label': source.label, 'path': e['path'],
'name': e['name'].rsplit('/', 1)[-1], 'size': e['size'], 'modified': e['modified']})
source.last_used_at = now
source.save(update_fields=['last_used_at'])
return JsonResponse({'files': files, 'errors': errors})
@csrf_exempt
@require_http_methods(['POST'])
def cloud_done(request, pk):
"""{path, result}: a device finished with an "Eingang" file — imported, known (already in the
library), too_big: not offered again; failed: offered again after the claim ran out."""
unauthorized = _require_auth(request)
if unauthorized:
return unauthorized
source = _get_source(request, pk)
if not source:
return JsonResponse({'error': 'unknown source'}, status=404)
try:
body = json.loads(request.body or b'{}')
except (json.JSONDecodeError, ValueError):
return JsonResponse({'error': 'invalid JSON'}, status=400)
result = body.get('result') if isinstance(body, dict) else None
if result not in ('imported', 'known', 'too_big', 'failed'):
return JsonResponse({'error': 'result: imported, known, too_big or failed'}, status=400)
seen = WebDAVSeen.objects.filter(source=source, path=str(body.get('path', ''))[:1000]).first()
if not seen:
return JsonResponse({'error': 'unknown file'}, status=404)
seen.result = result
seen.done_at = None if result == 'failed' else timezone.now()
seen.save(update_fields=['result', 'done_at'])
return JsonResponse({'ok': True})
@require_http_methods(['GET'])
def cloud_browse(request, pk):
unauthorized = _require_auth(request)

10
deploy/whisper/Dockerfile
View file

@ -1,10 +0,0 @@
# diora's built-in Whisper (docs/transcription.md). Built on the server next to diora-prod:
# docker compose build whisper
FROM python:3.12-slim
RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/*
RUN pip install --no-cache-dir faster-whisper==1.2.1 fastapi==0.143.1 uvicorn==0.54.0 python-multipart==0.0.32
WORKDIR /app
COPY server.py .
ENV WHISPER_CACHE=/models HF_HOME=/models/hf
EXPOSE 8000
CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"]

85
deploy/whisper/server.py
View file

@ -1,85 +0,0 @@
"""diora's built-in Whisper: a small OpenAI-compatible transcription server.
POST /v1/audio/transcriptions (multipart: file, model, language, prompt, response_format)
returns {text} or, with response_format=verbose_json, {language, duration, text, segments}.
The `model` field is ignored; WHISPER_MODEL decides. The model is loaded on the first request
and freed again after WHISPER_IDLE seconds without work, so it holds no memory while unused.
One transcription at a time; parallel requests wait (the server has few cores to share).
See docs/transcription.md.
"""
import os
import subprocess
import threading
import time
# Windows with a GPU: where the CUDA libraries (cuBLAS, cuDNN) are, separated by ';'.
for _d in filter(None, os.environ.get('WHISPER_DLL_DIRS', '').split(';')):
os.add_dll_directory(_d)
os.environ['PATH'] = _d + os.pathsep + os.environ['PATH']
import numpy as np # noqa: E402
from fastapi import FastAPI, File, Form, HTTPException, UploadFile # noqa: E402
from faster_whisper import WhisperModel # noqa: E402
MODEL = os.environ.get('WHISPER_MODEL', 'large-v3-turbo')
THREADS = int(os.environ.get('WHISPER_THREADS', '4'))
DEVICE = os.environ.get('WHISPER_DEVICE', 'cpu') # cpu or cuda
COMPUTE = os.environ.get('WHISPER_COMPUTE', 'int8') # int8_float16 on a GPU
IDLE = int(os.environ.get('WHISPER_IDLE', '600'))
CACHE = os.environ.get('WHISPER_CACHE', '/models')
app = FastAPI()
work = threading.Lock() # one transcription at a time
state = {'model': None, 'last': 0.0}
def model():
if state['model'] is None:
state['model'] = WhisperModel(MODEL, device=DEVICE, compute_type=COMPUTE, cpu_threads=THREADS,
download_root=CACHE)
return state['model']
def unloader():
while True:
time.sleep(30)
if state['model'] is not None and time.time() - state['last'] > IDLE and work.acquire(blocking=False):
try:
state['model'] = None
finally:
work.release()
threading.Thread(target=unloader, daemon=True).start()
def decode(data: bytes) -> np.ndarray:
p = subprocess.run(['ffmpeg', '-nostdin', '-loglevel', 'error', '-i', 'pipe:0', '-ac', '1', '-ar', '16000',
'-f', 'f32le', 'pipe:1'], input=data, capture_output=True)
if p.returncode != 0:
raise HTTPException(400, 'not audio: ' + p.stderr.decode('utf-8', 'replace')[-200:])
return np.frombuffer(p.stdout, np.float32)
@app.get('/health')
def health():
return {'ok': True, 'model': MODEL, 'device': DEVICE, 'loaded': state['model'] is not None}
@app.post('/v1/audio/transcriptions')
def transcriptions(file: UploadFile = File(...), language: str = Form(''), prompt: str = Form(''),
response_format: str = Form('json')):
audio = decode(file.file.read())
t0 = time.time()
with work:
segs, info = model().transcribe(audio, language=language or None, initial_prompt=prompt or None,
beam_size=5, vad_filter=True)
segs = [{'id': i, 'start': round(s.start, 2), 'end': round(s.end, 2), 'text': s.text}
for i, s in enumerate(segs)]
state['last'] = time.time()
dur = len(audio) / 16000
print(f'{dur:.0f} s audio in {time.time() - t0:.0f} s', flush=True)
text = ''.join(s['text'] for s in segs).strip()
if response_format == 'verbose_json':
return {'task': 'transcribe', 'language': info.language, 'duration': dur, 'text': text, 'segments': segs}
return {'text': text}

2
diora/__init__.py
View file

@ -1,2 +0,0 @@
# SQLite in WAL mode for every connection (diora/sqlite.py).
from . import sqlite # noqa: E402,F401

101
diora/search.py
View file

@ -1,101 +0,0 @@
"""The search across diora (docs/search.md): one query, what the server knows of every area —
podcasts and audiobooks one follows, podcast episodes (the full-text index, podcasts/discover.py),
news and blog sources, their articles, saved radio stations. The books aren't here: their titles
are encrypted, the page searches them itself (static/js/search.js).
Every word of the query has to be there, as the start of a word ("plattf" finds "Plattform");
case and diacritics don't count."""
from django.db.models import Q
from django.http import JsonResponse
from django.views.decorators.http import require_http_methods
from feeds.models import Item, Source
from podcasts import discover
from podcasts.models import PodcastFeed
from radio.models import SavedStation
LIMIT = 8 # per area; the area's own view has the rest
ARTICLE_SCAN = 400 # articles looked at after the database's rough match
def _words(q):
return [w for w in discover._words(q) if w][:8]
def _matches(words, *texts):
"""Every word as the start of a word in the texts ("plattf" finds "Plattform")."""
have = discover._words(' '.join(t or '' for t in texts))
return bool(words) and all(any(h.startswith(w) for h in have) for w in words)
def _rank(words, title):
"""Title starting with the query first, then title words, then the rest."""
t = discover._words(title)
if t[:len(words)] == words or (t and t[0].startswith(words[0])):
return 0
return 1 if all(any(h.startswith(w) for h in t) for w in words) else 2
def _feeds(user, words):
out = {'podcasts': [], 'audiobooks': []}
rows = PodcastFeed.objects.filter(user=user).values('id', 'title', 'author', 'artwork_url', 'kind', 'rss_url',
'favorite', 'new_episodes')
found = [f for f in rows if _matches(words, f['title'], f['author'])]
found.sort(key=lambda f: (_rank(words, f['title']), (f['title'] or '').lower()))
for f in found:
key = 'audiobooks' if f['kind'] == PodcastFeed.AUDIOBOOK else 'podcasts'
out[key].append(f)
return {k: v[:LIMIT] for k, v in out.items()} | {'podcasts_total': len(out['podcasts']),
'audiobooks_total': len(out['audiobooks'])}
def _sources(user, words):
rows = Source.objects.filter(user=user).values('id', 'title', 'url', 'kind', 'icon_url', 'category')
found = [s for s in rows if _matches(words, s['title'], s['url'], s['category'])]
found.sort(key=lambda s: (_rank(words, s['title']), (s['title'] or '').lower()))
return found[:LIMIT]
def _articles(user, words):
"""Articles whose title or author has every word, the newest first. The database narrows by
the longest word (LIKE, as case-blind as SQLite is), Python decides."""
longest = max(words, key=len)
rough = Q(title__icontains=longest) | Q(author__icontains=longest)
if longest != longest.capitalize():
rough |= Q(title__contains=longest.capitalize())
rows = (Item.objects.filter(user=user, gone=False).filter(rough).order_by('-published')
.values('id', 'title', 'author', 'kind', 'published', 'image_url', 'read_at', 'source__title')[:ARTICLE_SCAN])
found = [i for i in rows if _matches(words, i['title'], i['author'])]
return [{'id': i['id'], 'title': i['title'], 'author': i['author'], 'kind': i['kind'],
'published': i['published'].isoformat() if i['published'] else None, 'image_url': i['image_url'],
'read': i['read_at'] is not None, 'source': i['source__title']} for i in found[:LIMIT * 2]], len(found)
def _stations(user, words):
rows = SavedStation.objects.filter(user=user).values('id', 'name', 'url', 'tags', 'country', 'favicon_url',
'is_favorite')
found = [s for s in rows if _matches(words, s['name'], s['tags'], s['country'])]
found.sort(key=lambda s: (_rank(words, s['name']), not s['is_favorite'], s['name'].lower()))
return found[:LIMIT]
def search(user, q):
words = _words(q)
if not words or len(''.join(words)) < 2:
return {'q': q, 'empty': True}
out = {'q': q}
out.update(_feeds(user, words))
ids = discover.search_ids(user, q, limit=LIMIT * 2)
out['episodes'] = discover._rows(user, ids)
out['sources'] = _sources(user, words)
out['articles'], out['articles_total'] = _articles(user, words)
out['stations'] = _stations(user, words)
return out
@require_http_methods(['GET'])
def search_view(request):
"""?q=…: what matches in every area (see the module's text)."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
return JsonResponse(search(request.user, request.GET.get('q', '')[:200]))

26
diora/settings.py
View file

@ -31,14 +31,12 @@ INSTALLED_APPS = [
'books',
'gpodder',
'tts',
'feeds',
'projects',
]
EBOOK_MAX_BYTES = 100 * 1024 * 1024 # 100 MB (big scanned PDFs, complete editions)
EBOOK_MAX_BYTES = 50 * 1024 * 1024 # 50 MB
# Encrypted uploads are base64-encoded (~33% overhead) so allow ~140 MB body
DATA_UPLOAD_MAX_MEMORY_SIZE = 140 * 1024 * 1024
# Encrypted uploads are base64-encoded (~33% overhead) so allow ~75 MB body
DATA_UPLOAD_MAX_MEMORY_SIZE = 75 * 1024 * 1024
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
@ -78,15 +76,10 @@ DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR / 'data' / os.environ.get('DIORA_DB_NAME', 'db.sqlite3'),
# WAL (diora/sqlite.py); a transaction takes the write lock when it begins, so one that
# reads first never fails on upgrading it (WAL's busy timeout can't help there).
'OPTIONS': {'timeout': 20, 'transaction_mode': 'IMMEDIATE'},
'OPTIONS': {'timeout': 20},
}
}
# The built-in Whisper instance (OpenAI-compatible, base such as http://whisper:8000/v1);
# empty = none (docs/transcription.md).
WHISPER_URL = os.environ.get('DIORA_WHISPER_URL', '').strip().rstrip('/')
PODCAST_MAX_EPISODES_PER_FEED = int(os.environ.get('PODCAST_MAX_EPISODES_PER_FEED', '200'))
AUTH_PASSWORD_VALIDATORS = [
@ -141,17 +134,6 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
LOGIN_URL = '/accounts/login/'
LOGIN_REDIRECT_URL = '/'
# Sign-in with Authelia (accounts/oidc.py); off while one of the three is empty.
OIDC_ISSUER = os.environ.get('DIORA_OIDC_ISSUER', '')
OIDC_CLIENT_ID = os.environ.get('DIORA_OIDC_CLIENT_ID', '')
OIDC_CLIENT_SECRET = os.environ.get('DIORA_OIDC_CLIENT_SECRET', '')
# Fixed callback address; empty = built from the request (needs the real https host).
OIDC_REDIRECT_URI = os.environ.get('DIORA_OIDC_REDIRECT_URI', '')
OIDC_PROVIDER_NAME = os.environ.get('DIORA_OIDC_PROVIDER_NAME', 'Authelia')
# Public client the devices use to fetch their `diora_key` claim from the provider themselves
# (docs/accounts.md, "Schlüssel über Authelia"); empty = off.
OIDC_APP_CLIENT_ID = os.environ.get('DIORA_OIDC_APP_CLIENT_ID', '')
LOGOUT_REDIRECT_URL = '/'
# Last.fm

23
diora/sqlite.py
View file

@ -1,23 +0,0 @@
"""SQLite in write-ahead-log mode (docs/database.md): readers no longer wait for a writer.
In the old rollback-journal mode a long write (the podcast refresh every 15 minutes, a book of
30 MB going in) kept every reader out until it was done, and requests failed with "database is
locked" — even GET /books/. WAL is set once per database file and stays; synchronous=NORMAL is
the safe pairing for it. A connection that can't switch (another one holds the file) just goes
on as it is; the next one switches.
"""
from django.db.backends.signals import connection_created
def _sqlite_wal(sender, connection, **kwargs):
if connection.vendor != 'sqlite':
return
with connection.cursor() as cursor:
try:
cursor.execute('PRAGMA journal_mode=WAL')
except Exception: # noqa: BLE001 - locked right now; another connection will do it
pass
cursor.execute('PRAGMA synchronous=NORMAL')
connection_created.connect(_sqlite_wal, dispatch_uid='diora_sqlite_wal')

62
diora/test_search.py
View file

@ -1,62 +0,0 @@
"""The search across diora (diora/search.py)."""
from datetime import datetime, timezone
from django.contrib.auth.models import User
from django.test import TestCase
from feeds.models import Item, Source
from podcasts import discover
from podcasts.models import PodcastEpisode, PodcastFeed
from radio.models import SavedStation
class SearchTests(TestCase):
def setUp(self):
self.user = User.objects.create_user(username='sucher', password='pw12345678')
self.client.force_login(self.user)
other = User.objects.create_user(username='andere', password='pw12345678')
when = datetime(2026, 3, 1, tzinfo=timezone.utc)
self.pod = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/p', title='Ökonomie heute',
author='Peter Decker')
self.book = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/b', title='Das Kapital',
author='Karl Marx', kind=PodcastFeed.AUDIOBOOK)
PodcastFeed.objects.create(user=other, rss_url='https://example.org/p', title='Ökonomie heute')
self.ep = PodcastEpisode.objects.create(feed=self.pod, guid='e1', title='Die Ökonomie der Plattformen',
audio_url='https://example.org/e1.mp3', pub_date=when)
PodcastEpisode.objects.create(feed=self.book, guid='k1', title='Kapitel 1: Die Ware',
audio_url='https://example.org/k1.mp3', pub_date=when)
discover.index_new()
blog = Source.objects.create(user=self.user, url='https://blog.example/feed', title='Plattform-Blog', kind=Source.BLOG)
Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='a', title='Plattformarbeit und Streik',
published=when)
Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='b', title='Gartenarbeit', published=when)
Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='c', title='Plattformen, alt',
published=when, gone=True)
SavedStation.objects.create(user=self.user, name='Radio Plattform', url='https://radio.example/s', tags='jazz')
def find(self, q):
return self.client.get('/search/', {'q': q}).json()
def test_every_area(self):
r = self.find('plattf')
self.assertEqual([e['id'] for e in r['episodes']], [self.ep.id])
self.assertEqual([a['title'] for a in r['articles']], ['Plattformarbeit und Streik'])
self.assertEqual([s['title'] for s in r['sources']], ['Plattform-Blog'])
self.assertEqual([s['name'] for s in r['stations']], ['Radio Plattform'])
def test_podcasts_and_audiobooks_apart_case_and_accents_blind(self):
r = self.find('okonomie')
self.assertEqual([f['id'] for f in r['podcasts']], [self.pod.id])
self.assertEqual(r['audiobooks'], [])
r = self.find('marx kap')
self.assertEqual([f['id'] for f in r['audiobooks']], [self.book.id])
self.assertEqual(r['episodes'], [], 'an audiobook\'s chapters are no podcast episodes')
def test_every_word_and_nothing_for_too_little(self):
self.assertEqual(self.find('plattform garten')['articles'], [])
self.assertTrue(self.find('a')['empty'])
self.assertEqual(self.client.get('/search/', {'q': 'x'}).status_code, 200)
def test_only_signed_in(self):
self.client.logout()
self.assertEqual(self.client.get('/search/', {'q': 'plattform'}).status_code, 401)

4
diora/urls.py
View file

@ -5,19 +5,15 @@ from django.urls import path, include
from django.views.static import serve as serve_static
from accounts.sync import sync_snapshot
from diora.search import search_view
urlpatterns = [
path('admin/', admin.site.urls),
path('accounts/', include('accounts.urls')),
path('podcasts/', include('podcasts.urls')),
path('books/', include('books.urls')),
path('feeds/', include('feeds.urls')),
path('projects/', include('projects.urls')),
path('tts/', include('tts.urls')),
path('api/2/', include('gpodder.urls')),
path('api/sync/', sync_snapshot, name='api_sync'),
path('search/', search_view, name='search'),
# Served at the root (not /static/js/sw.js) so its default scope covers
# the whole app — a service worker's scope is limited to its own script's
# directory unless the server sends Service-Worker-Allowed, so registering

91
docs/accounts.md
View file

@ -1,91 +0,0 @@
# Anmeldung und Bücher-Schlüssel
## Zwei Wege hinein
**Benutzername und Passwort** (`accounts/views.py` `login_view`): Das Login-Formular leitet im
Browser aus dem Passwort den Bücher-Schlüssel ab (PBKDF2-SHA256, Salt `diora:<username>`,
200 000 Runden) und legt ihn bis zur nächsten Seite in `localStorage['diora_pending_enc_key']`
ab. Die App-Seite übernimmt ihn als `diora_enc_key_<userId>` (`32-crypto.js`, `99-init.js`).
**Authelia (OpenID Connect)** (`accounts/oidc.py`, `accounts/oidc_views.py`): als zusätzlicher
Weg, die Anmeldung mit Passwort bleibt. Der Ablauf ist Authorization Code mit PKCE (S256) und
einem vertraulichen Client (`client_secret_basic`). Das id_token wird hier geprüft (RS256 gegen
die JWKS des Anbieters, `iss`, `aud`, `exp`, `nonce`), dafür reicht `cryptography`.
- Ein Konto wird **einmal verknüpft**: in den Einstellungen „Mit Authelia verknüpfen“, während
man normal angemeldet ist (`/accounts/oidc/start/?link=1`). Gespeichert wird `OidcIdentity`
(`issuer`, `sub`; Name und Mail nur zur Anzeige). Danach führt „Mit Authelia anmelden“ auf der
Login-Seite in dieses Konto.
- Eine nicht verknüpfte Authelia-Identität wird abgewiesen. Es entstehen **keine neuen
Konten** über Authelia, und es wird auch nicht über Name oder Mail automatisch verknüpft.
- Wer hinein darf, regelt Authelia (Policy `mitglieder` = Gruppen `admins` oder `freunde`).
- Einstellungen per Umgebung: `DIORA_OIDC_ISSUER`, `DIORA_OIDC_CLIENT_ID`,
`DIORA_OIDC_CLIENT_SECRET`, `DIORA_OIDC_REDIRECT_URI` (fest, weil hinter dem Proxy kein
https erkannt wird), optional `DIORA_OIDC_PROVIDER_NAME`. Fehlt eins der ersten drei, gibt es
keinen Knopf. Auf prod stehen sie in `~/docker-compose/diora-prod/.env` (Strato), das
Gegenstück liegt auf Hetzner in `/opt/authelia/oidc-clients.env`.
- Redirect-URI im Authelia-Client: `https://diora.creamfresh.xyz/accounts/oidc/callback/`.
## Der Schlüssel nach einer Authelia-Anmeldung
### Normalfall: über Authelia, ohne Nachfrage
Authelia hält je Nutzer ein geheimes Attribut `diora_key` (32 Byte, base64). Es geht nur an den
öffentlichen Client `diora-app` (`DIORA_OIDC_APP_CLIENT_ID`, PKCE, Scope `diora_key`) und nur
über userinfo, nicht im id_token. Nach jedem Authelia-Login (und nach dem Verknüpfen) führt
diora auf `/accounts/oidc/key/` (`templates/accounts/oidc_key.html`). Die Seite meldet sich
**im Browser** noch einmal mit `prompt=none` als `diora-app` an, unsichtbar, weil die
Authelia-Sitzung noch besteht, und holt per CORS Token und userinfo. Daraus leitet sie mit
HKDF-SHA256 (Salt = `sub`, info `diora authelia key v1`) einen AES-GCM-Schlüssel ab:
- Hat das Gerät den Bücher-Schlüssel (und passt er zu einem Buch), wird er damit verschlüsselt
und im Vault `authelia-key` abgelegt, bei jedem Authelia-Login neu. So heilt ein geänderter
`diora_key` von selbst.
- Hat es keinen, öffnet sie den Vault-Eintrag damit und legt den Schlüssel ab.
- Es zählt nur das Authelia-Konto, das mit diesem diora-Konto verknüpft ist (`sub`).
Der diora-Server sieht `diora_key` nie. **Grenze, ehrlich:** Wer Authelia (Hetzner-Root) oder
dein Authelia-Konto (Passwort, Mail-Reset) übernimmt, kommt damit an die Bücher. Wer Strato
übernimmt, kann im Browser ohnehin manipuliertes JS ausliefern. Geschützt ist gegen geleakte
Daten **einer** Seite (diora-DB/Backup ohne Authelia-Nutzerdatei oder umgekehrt). Geht
`diora_key` verloren, sind die Bücher nicht weg: Der Schlüssel kommt weiter aus dem Passwort.
Entschieden 2026-11-01 vom Nutzer (Komfort vor stärkerer Trennung).
### Notweg: Code oder Passwort
Ohne Passwort gibt es keinen Schlüssel. Deshalb erzeugt `getOrCreateEncKey()` **nie mehr
selbst einen neuen Schlüssel**: Früher machte es einen zufälligen, wenn keiner da war, und damit
waren die vorhandenen Bücher auf diesem Gerät verloren. Jetzt fragt es `dioraUnlock()`
(`static/js/app/33-unlock.js`). Es gibt zwei Wege:
1. **Von einem entsperrten Gerät übergeben** (`accounts/keyshare.py`, Modell `KeyShare`). Das
neue Gerät schickt einen ECDH-P-256-Public-Key und zeigt einen 6-stelligen Code, der aus
diesem Key berechnet wird. Entsperrte Geräte fragen alle 15 s
`/accounts/keyshare/pending/` ab und zeigen denselben Code mit „Freigeben/Ablehnen“. Die
Freigabe verschlüsselt den Bücher-Schlüssel mit AES-GCM unter dem gemeinsamen Geheimnis
(HKDF-SHA256, info `diora keyshare v1`). Der Server sieht nur Public Keys und Chiffrat und
gibt die Antwort genau einmal heraus. Anträge laufen nach 10 min ab. Angeboten werden nur
solche, deren Gerät in den letzten 20 s nachgefragt hat, und je Konto höchstens 5, die
ältesten weichen.
2. **Das diora-Passwort einmal eingeben**: dieselbe Ableitung wie beim Login.
In beiden Fällen wird der Schlüssel an einem Buch geprüft, bevor er gespeichert wird. Mit
„Später“ bleiben Bücher gesperrt, und die Bibliothek zeigt einen Knopf „Entsperren“. Die
Einstellungsseite erzeugt ebenfalls keinen Schlüssel mehr.
## Apps
Android und Desktop melden sich bisher nur mit Passwort an (eigene Login-Seiten in
`diora-android/web-native/login.html` und `diora-desktop/src/web/login.js`, die einen
API-Token holen). Mit Authelia melden sie sich selbst als öffentlicher Client `diora-app` an:
1. `GET /accounts/oidc/app/` sagt, wo Authelia ist, gibt die Client-ID und eine vom Server
signierte nonce (10 min gültig).
2. Die App führt den Code-Flow mit PKCE im Systembrowser aus. Redirect:
`xyz.creamfresh.diora:/oauth` bzw. `xyz.creamfresh.diora.reader:/oauth` (Android),
Loopback `http://127.0.0.1:<port>/oauth` (Desktop, jeder Port). Den Code tauscht sie selbst
und liest `diora_key` aus userinfo.
3. `POST /accounts/oidc/app/login/ {id_token}` prüft das Token (`aud` = `diora-app`, nonce) und
gibt den API-Token des verknüpften Kontos heraus (den vorhandenen, kein neuer).
4. Die App holt mit dem Token `/accounts/vault/authelia-key/` und öffnet den Bücher-Schlüssel
wie die Webseite (HKDF, Salt `sub`). Ist dort noch nichts, fragt sie nach dem diora-Passwort.

62
docs/ai.md
View file

@ -1,62 +0,0 @@
# KI: Claude schlägt vor, der Nutzer entscheidet
Since 2026-10-24. Organisation help with Claude (Anthropic) across diora: books and magazines
first; podcasts, news/blogs and projects follow (same building blocks).
## The key: the user's own, end-to-end encrypted, only on the devices
Two providers, chosen in the settings (decided 2026-10-25): **Anthropic** (the official SDK) or
**OpenRouter** (the same Claude model, `anthropic/claude-opus-5.5`). OpenRouter's
Anthropic-compatible endpoint refuses the headers a browser call needs (`anthropic-version` …,
checked by a CORS preflight), so diora talks to OpenRouter's chat completions instead: plain
`fetch`, `response_format` json_schema (strict), `reasoning.effort`; no fallback on refusals, and
OpenRouter sees what is sent too. The vault keeps `{provider, anthropic_key, openrouter_key}`
(an old `{anthropic_key}` still reads as Anthropic); a key is checked before it is kept
(Anthropic: the model's info; OpenRouter: `GET /api/v1/key`).
Decided with the user: the API key is entered in the app ("KI-Assistent", the ✦ of a
feature or the app menu) and stored like the books — encrypted with the book key on the device,
kept by the server as a blob it can't read:
- `GET/POST /accounts/vault/<name>/` (`accounts.UserVault`, name `ai`): `{ct, iv}`; `{ct: ''}`
removes it. Its own store, not the library prefs (`/books/prefs/` is rewritten whole when
the folder order is saved).
- The devices decrypt it into memory only and call Claude **directly** from the page
(browser, the apps' WebView, the desktop). The diora server sees neither the key nor what is
sent; Anthropic sees what a feature needs (titles, authors, folder names …). Anthropic bills
the user.
- Without the book key on a device (or after a password change, which changes the book key)
the key can't be read: the settings ask for it again. Not on diora Reader (e-readers).
- Server-side jobs (in the background, without an app open) were left out on purpose: they
would need the key on the server. The project Radar has its own server key (docs/projects.md).
Podcast articles wait on the server as a wish until a device with the key is open
(docs/transcription.md).
## Building blocks (`static/js/ai.js`, an ES module)
- The official SDK, `@anthropic-ai/sdk` 0.133.0, bundled for the browser with esbuild 0.28.2:
`static/vendor/anthropic-sdk-0.133.0.mjs` (licence beside it). `dangerouslyAllowBrowser`:
the key is the user's own. To update: `npm install @anthropic-ai/sdk@<v> esbuild` in an empty
folder, `esbuild entry.mjs --bundle --format=esm --platform=browser --target=es2020 --minify`
with `export { default } from '@anthropic-ai/sdk'`, and change the import in ai.js.
- `DioraAI.ask({system, prompt, schema, effort})`: model `claude-opus-5-5`, structured output
(`output_config.format` json_schema), effort `low` by default, streamed, `fallbacks: "default"`
(beta `server-side-fallback-2026-07-01`) for declined requests; refusals and over-long
answers become plain German messages.
- `ask` also takes `onProgress(words)` (words written so far while streaming; OpenRouter is not streamed and reports once at the end)
and `signal` (an `AbortSignal`), both optional; `DioraAI.modelName()` is the model of the chosen provider. Used by the podcast articles.
- `DioraAI.review({title, intro, items, onApply})`: the suggestions with checkboxes — nothing
changes until the user ticks and presses "Übernehmen".
- `DioraAI.settings()`, `withKey(fn)`, `progress()`, `showError()`, `toast()`; the sheets close
with Android's back key; ⚙ in each sheet opens the settings.
## Features
| area | where | what Claude gets | what it suggests |
|---|---|---|---|
| Bücher, Zeitschriften | ✦ in the library's head (`static/js/app/55-library-tools.js`) | per book: id, title, author, folders, series, magazine mark; the existing folders | folders to add (existing first; new ones only for ≥ 3 books), series with volume, cleaned title/author where broken, magazine mark |
| Podcasts | ✦ in Abos (diora-android `web-native/static/native-podcast.js`, copied to `static/podcasts/`) | per subscription: title, author, folders, level, heard in 12 months, publishing interval, three newest episode titles | folders to add (≤ 2), level Stamm / Strom / Auf Abruf (only with a clear reason) |
| Nachrichten, Blogs | ✦ in the feeds' bar (`static/js/feeds.js`; the apps' section bar presses it) | per source: title, address, kind, Ressort or folders, recent titles shown | Ressort for news, folders for blogs, news ↔ blog when clearly wrong |
| Podcast-Artikel | ✦ Artikel in an episode (`static/js/podcast-article.js`, docs/transcription.md) | the episode's transcript (feed or Whisper), title, author, shownotes, chapters | an article from the speakers' standpoint with summary and sections, saved to the episode and to Blogs (no review sheet: it is new text, removable) |
Applied through `_updateBookMeta` (the encrypted meta, as when editing by hand).

112
docs/audiobooks.md
View file

@ -1,112 +0,0 @@
# Audiobooks (Hörbücher & Hörspiele)
A podcast subscription can be marked as an audiobook. It then leaves the
podcast world (no "Neu", no queue automation, not in the podcast lists) and
shows up in the library ("Bücher") next to the e-books, where it is listened to
like a book: its episodes are chapters in order, "Weiterhören" goes on where
the last chapter stopped, the whole book has one progress, places can be kept
as bookmarks with a note, and the sleep timer can stop at the end of a chapter.
## Server (diora-web)
### The mark
`PodcastFeed.kind`: `"podcast"` (default) or `"audiobook"`. A feed setting like
the others: in `state.feeds[]` (`/podcasts/state/`), set with
`POST /podcasts/feeds/<id>/settings/ {"kind": "audiobook"}`.
Marked as an audiobook:
- its episodes leave "Neu" (dismissed, not played); new ones never get there
and are never queued, whatever `new_episodes` says;
- it is left out of "Folgen" (`/podcasts/episodes/` without `feed_id`);
- nothing else changes: places, played marks, downloads, settings stay.
Episodes of it already in the queue stay there until played or removed.
### Chapters and progress
Chapter order: `episode_number` (if set), then `pub_date`, then the title in
natural order ("Kapitel 2" before "Kapitel 10"), then id.
`GET /podcasts/audiobooks/` → `{"books": [book, …]}`, the last listened first:
```
book = {
"feed_id", "title", "author", "artwork_url",
"chapters": 24, "done": 6, // played chapters
"total_seconds": 32400, "heard_seconds": 11400,
"next": {"episode_id", "index", "position_seconds"} | null, // null: finished
"last_played_at": ISO | null
}
```
`heard_seconds` = played chapters' durations + the places in the others.
Durations the feed doesn't have count as 0 until a player reports them
(`progress/save` with `duration_seconds`, see youtube.md).
`GET /podcasts/audiobooks/<feed_id>/` → `{"book": book, "chapters": [ch, …],
"bookmarks": [bm, …]}` with
```
ch = { // the queue entry's shape, so it can be handed to the player as it is
"episode__id", "episode__title", "episode__audio_url", "episode__duration_seconds",
"episode__artwork_url", "episode__feed__id", "episode__feed__title",
"episode__feed__artwork_url", "episode__pub_date", "position_seconds", "played",
"index"
}
bm = {"id", "episode_id", "position_ms", "note", "created_at"} // in chapter order, then place
```
### Bookmarks
- `POST /podcasts/bookmarks/ {"episode_id", "position_ms", "note"}` → `{"ok", "bookmark": bm}`
- `POST /podcasts/bookmarks/<id>/ {"note"}` → `{"ok", "bookmark": bm}`
- `POST /podcasts/bookmarks/<id>/delete/` → `{"ok"}`
Any episode can have bookmarks; the library shows those of audiobooks.
## The player (both hosts: podcasts-web.js and Android)
One new host call, next to `podcastPlayEpisode`:
`podcastPlayBook(json)` with `json = {"feed_id", "chapters": [ch, …], "index", "position_ms"}`
(`position_ms` optional: else the chapter's own place)
- plays `chapters[index]` **without putting anything in the queue**;
- the book is what plays next: when a chapter ends it counts as played and the
next chapter follows from its own place; after the last one playing stops;
"next" (⏭, media keys) goes to the next chapter without marking the one left
as played;
- playing anything else (a queue episode, the radio's own handover aside) ends
the book context; the book context survives a restart of the page/app;
- the sleep timer needs nothing new: "end of chapter" is `podcastSleep(-1)`
(end of the episode) or, for an episode with chapters of its own, the minutes
until that chapter's end (`podcastSleep(ms / 60000)`, fractions are fine).
`playerState()` (unchanged) tells the library which chapter plays where.
## The library (diora-web `static/js/audiobooks.js`, everywhere)
One file for the web page, the desktop app and the Android app (like
feeds.js; full flavour only, it needs the podcast host). It draws into
`#tab-books`, above `#book-list`, in its own container `#audiobook-shelf`:
- "Weiterhören": the audiobook listened to last (unless finished);
- "Hörbücher": a card per audiobook — picture, title, author,
"3 h 10 von 9 h · Kapitel 7/24" and a progress bar;
- "+ Hörbuch": pick one of the podcast subscriptions to mark as an audiobook;
- the book view: picture, progress, ▶ Weiterhören / Von vorn, the chapter list
(tap: play it from its place; played ones ✓), bookmarks (tap: play from there;
note editable; ✕), "+ Lesezeichen" (at the place where this book plays now),
"Schlummer: Kapitelende", and ⋮ → "Wieder als Podcast", "Alles als ungehört".
It talks to the server with fetch and to the player through the host
(`window.DioraNative` on Android, `window.DioraPodcastHost` elsewhere):
`podcastState`, `podcastFeedSettings`, `podcastPlayBook`, `playerState`,
`podcastSleep`, `podcastMarkPlayedAll`.
## The podcast UI (native-podcast.js, diora-android)
- feed settings sheet: "Als Hörbuch" (sets `kind`);
- audiobook feeds left out of Abos, Neu, Folgen, search results of the
subscriptions; a hint "In Bücher → Hörbücher" where one was moved;
- the player shows a book chapter as "Kapitel 7/24 · <book>" when it plays one.

47
docs/database.md
View file

@ -1,47 +0,0 @@
# The database
One SQLite file, `data/db.sqlite3` (on the server `/home/marwin/diora-prod-data`, mounted into
`diora-prod` at `/app/data`). The books live in it too, encrypted, so it is large (2.5 GB in
October 2026) and some writes are big (a scanned book of 30 MB).
## Write-ahead log (since 2026-10-19)
`diora/sqlite.py` puts every connection in WAL mode (`PRAGMA journal_mode=WAL`, set once per
file, it stays) with `synchronous=NORMAL`; `settings.DATABASES` has `transaction_mode:
IMMEDIATE`, so a transaction takes the write lock when it begins.
Why: in the rollback-journal mode before, a long write kept all readers out. The podcast
refresh (cron, every 15 minutes) and big uploads made requests fail with "database is
locked", GET /books/ among them — the apps then saw no new books and no changed titles.
With WAL, readers read on while one writer writes; writers still take turns (20 s timeout).
Next to `db.sqlite3` there are now `db.sqlite3-wal` and `db.sqlite3-shm`. They belong to the
database: a copy of `db.sqlite3` alone may miss the latest writes. To copy it safely:
docker exec diora-prod python -c "import sqlite3; s=sqlite3.connect('data/db.sqlite3'); d=sqlite3.connect('data/backup.sqlite3'); s.backup(d)"
or stop the container first. Back to the old mode, if ever needed:
`PRAGMA journal_mode=DELETE` (with no other connection open) and remove `diora/sqlite.py`'s hook.
## The book list never reads past the file
A book's row is `user, meta_ct, meta_iv, data_ct (the file), data_iv, uploaded_at, is_read,
cover_ct, cover_iv`. SQLite stores a big value in a chain of overflow pages, and to reach a
column *after* it, it walks the whole chain: listing 427 books read gigabytes and took 37 s
(the desktop app gives up after 10 s, so it never synced). Since then `books.models.book_rows()`
reads `id, meta_ct, meta_iv` from the rows (they stand before the file) and `uploaded_at,
is_read` / whether there is a cover from the covering indexes `ebook_list_idx` and
`ebook_cover_idx` — no list query touches a file. `books/tests.py` checks the query plans.
## The files in a table of their own (since 2026-10-19)
`books.models.EBookFile` (one row per book, `book_id` its key) holds `data_ct`/`data_iv`;
`EBook` has only the small fields. Before, every change of a title, a folder or "gelesen"
rewrote the whole row with its file (megabytes, for a scanned volume tens of them), and every
`EBook.objects.get()` loaded the file. `EBook.objects.create(data_ct=…, data_iv=…)` still
works (the manager makes the EBookFile too); the API is unchanged.
Migration `books/0008_ebook_file` copied 430 files in one statement and dropped the two
columns: about 2 minutes on the server (rehearsed on a copy), the site down meanwhile. The
pages the old columns used are free now and get reused; `VACUUM` (exclusive, minutes, at a
quiet time) would give them back to the disk.

226
docs/library.md
View file

@ -1,226 +0,0 @@
# Folders and the library (redesign)
## Folders everywhere: tags shown as filter chips
One idea of a folder for every area — Bücher, Zeitschriften, Hörbücher,
Podcasts, Blogs (and the news' Ressorts, which already work this way):
- A thing can be in **several folders** (they work like tags); none is fine.
- Folders are **filters**, not rooms: a row of chips on top —
`Alle · <folder> n · … · Ohne Ordner` — tapping one filters in place; no
"inside a folder" view, no back button. On a wide window the row wraps.
- **Assigning**: mark one or more things (long press / ☐ select mode, or ⋮ on
one), then "In Ordner …": a sheet with every folder of the area as checkboxes
and "Neuer Ordner". No typing a name that must match.
- **Managing** (⋮ on the chip row → "Ordner verwalten"): rename, merge (rename
onto an existing name), delete (the things stay, only the folder goes),
reorder by dragging. The order is the chip order.
- **Automatic groups** stay automatic and look different from folders (no
chip; a stack/group heading): audiobook series, magazine periodicals.
### Where folders are kept
| Area | Field | Notes |
|------|-------|-------|
| E-books, magazines | encrypted meta `folders: [names]` | `folder` (the first) kept for older clients; read `folders` or else `[folder]` |
| Audiobooks, podcasts | `PodcastFeed.folders` (JSON list) | feed setting `folders`; `folder` (string) still accepted |
| Blogs | `Source.folders` (exists) | |
| News | `Source.category` (one Ressort) | already chips in Nachrichten |
The folder order per area: e-books in the encrypted user settings blob (names
are private), the others on the server (`/podcasts/folders/`, `/feeds/folders/`,
`{order: [names]}`).
### Podcasts: a category's episodes, by time
The folder chips stand over „Folgen“ too: with one chosen it lists every episode of
that category (`GET /podcasts/episodes/?folder=<name>`, empty: the podcasts in none).
„Folgen“, „Neu“ and one podcast's page group their episodes by time — Heute, Gestern,
Diese Woche (the last 7 days), Diesen Monat (30 days), then the months — after the
Kino feed. In „Folgen“ (sort „Nach Zeitraum, seltene zuerst“, the default) and „Neu“
the podcasts that seldom come go first within a group: `interval_days` of each feed
in `/podcasts/state/`, the mean gap between its newest 10 episodes (one episode: its
age, at least 30 days, the same when all came within an hour; kept an hour). Heard is a filter chip, not an order.
### Shared code: `static/js/folders.js` (diora-web, every platform)
```
DioraFolders.chips(el, {folders: [{name, count}], active, none: count|0, onPick(name|''|null), onManage()})
DioraFolders.assign({title, folders: [names], selected: [names], onDone(names)}) // the sheet
DioraFolders.manage({folders: [names], onRename(old, new), onDelete(name), onOrder(names)})
```
Each area keeps its own storage and calls these for the look and the dialogs.
## The library (Bücher, Zeitschriften)
- **Header**: title, search (title, author), sort (zuletzt gelesen · Titel ·
Autor · hinzugefügt · kürzeste / längste zuerst), ⊞/☰ grid or list, `+` (Buch hinzufügen, Aus Cloud).
- **Length** (since 2026-10-25): the pages in the encrypted meta (`pages`) — a PDF's pages, an
EPUB's text by 1800 characters a page (`bookPagesOfFile`, 40-epub.js; the reader's own
estimate). Set on upload and when a book is opened; for older books the device works it out
once when a length sort is chosen ("Länge wird ermittelt … 3 von 80": file from the offline
copy or the server, decrypted on the device). Unknown lengths come last; the cards show
"· 320 S." while sorted by length.
- **Filter chips**: Alle · Angefangen · Ungelesen · Gelesen — then the folders.
- **Weiterlesen**: one card on top (cover, title, "Seite 132 von 186 · gestern",
▶), not repeated in the list below.
- **Grid** (default on phone and desktop): covers with a progress line, title.
**List** (default on e-ink / diora Reader): one row per book, small cover,
title, author, % / ✓.
### Covers, end-to-end encrypted
A client that has a book's file extracts its cover (EPUB: the OPF cover image;
PDF: page 1), scales it to 240 px wide JPEG, **encrypts it with the user's key**
(AES-GCM like the meta) and uploads it once: `POST /books/<id>/cover/
{cover_ct, cover_iv}`. `GET /books/` lists `has_cover`; `GET /books/<id>/cover/`
returns `{cover_ct, cover_iv}`. Clients cache the decrypted image locally. The
server never sees a cover.
### Editions
Two or more books can be **editions of one work** (the PDF scan and the EPUB, two
translations …): the same random key in each one's encrypted meta, `work: "w…"`
(`''` or missing = none). The list shows a work once — the edition read last, else an EPUB,
else the first added — with its formats on the cover ("EPUB · PDF", in the list view after
the author). Each edition keeps its own file, position and highlights.
- Joining: choose several books → "Als Ausgaben verbinden", or a book's ⋮ → "Als Ausgabe
verbinden …" (other books listed, similar titles first, with a search).
- ⋮ → "Ausgaben (n) …" lists the editions: open one, "Lösen" sets one apart (the last one
left is set apart with it), "Weitere Ausgabe verbinden …".
- For the apps' own book sheets: `window.bookEditions(id)` opens that sheet,
`window.bookEditionsOf(id)` returns the editions (the book alone if it has none).
### Generated covers
A book without a cover (or one set to `cover: 'generated'` in its meta) gets one
drawn by `static/js/gencover.js`: an SVG, the same on every device, never stored.
A **concept** (motif, palettes, typeface) is chosen by score:
- words in the title (3), the shelf category (2) and the author (2, only at a
name's start). A stem of five letters or more counts anywhere (German
compounds), a shorter one at a word's start, one of three letters as a whole
word; `^` / `$` in the list force a word's start / end.
- the folders, by name (`FOLDER_THEMES`): the first counts 2.5 — more than an
author or shelf word, less than a title word — further ones 1.
A work of Marx or Engels (author, or a volume of the MEW series) is always a **blue volume**
(blau: blue cloth, gold rules and capitals, the spine on the left with the volume number);
no other book is.
Concepts: konstruktiv (Marxist and political titles), geometrie (Philosophie: rings, turning squares,
triangle/circle/square), psyche (Psychoanalyse: two ellipses, the Borromean
knot, a spiral), karte (Geschichte), kosmos (Science-Fiction), ornament
(Fantasy), raster (Politik, Soziologie), schatten (Krimi: a red line, a lit
doorway), grusel (Horror), klassiker (patterned paper with a title plate),
sakral (Religion & Mythologie), spiel (Kinder & Jugend), code (Informatik),
typo, wellen, botanik, berge, zellen, and since 2026-10-25 funk (talk and radio: waveform,
mast, speech bubbles), musik (record, staff), kunst (colour fields, brushstrokes), essen (plate
on a checked cloth, citrus), sport (track, pitch), geld (price chart, coins), reise (road,
route with compass) — and bauhaus when nothing matches (shapes, quarter-circle tiles, a
circle crossed by diagonals). Most concepts have two or three motifs. A hash of title and
author picks motif, palette, layout and every detail.
**Square** (`{square: true}`, since 2026-10-25): audiobooks and podcasts without a picture, or
whose picture doesn't load (audiobooks.js; diora-android native-podcast.js `art()`, before the
initial letter). 300 × 300, three title lines at most; the band of the motif shown is centred
in what the text leaves free. Podcasts (`podcast: true`) nothing matches get funk, not
bauhaus; a Marx/Engels audiobook gets konstruktiv (the blue volume is a binding).
### Export as XML (since 2026-10-24)
⇩ in the library's head (`static/js/app/55-library-tools.js`) saves `diora-bibliothek-<date>.xml`:
made on the device from the list it has decrypted (nothing new goes to the server), every book
and magazine issue with its diora metadata — title, author, file name, ISBN, pages, shelf
category, series and volume, work (editions), magazine mark, cover choice, folders —, the
reading position (`<progress percent anchor read read-at last-read device furthest-percent
furthest-anchor>`), when it was uploaded, whether it has highlights, and an issue's articles with
their state. The folder order heads the file. Root `<diora-library version="1">`.
✦ beside it: ordering with Claude (docs/ai.md).
### Eingang: books from a cloud folder by themselves (since 2026-10-24)
A cloud source (Einstellungen → Cloud-Verbindungen, WebDAV: `accounts.WebDAVSource`,
`accounts/webdav.py`) can be marked **Eingang**: its new books come into the library without
anyone importing them. The server can't do it alone — the books are encrypted with the key
on the devices — so a device does:
1. `static/js/app/57-cloud-inbox.js`: after the book list has loaded (at most every 5
minutes; only with the real key in `localStorage`, never on diora Reader) it asks
`GET /books/cloud/inbox/` → `{files: [{source, label, path, name, size, modified}], errors}`:
the `.epub`/`.pdf` files of the Eingang sources (three folders deep) no device has taken.
The server claims them for this device for ten minutes (another device asking meanwhile
doesn't get them).
2. Each file: fetched through the server (`/books/cloud/<id>/fetch/`), imported like any file
(`_importEbookBuffer`, `skipKnown`: same title and author → left out), then
`POST /books/cloud/<id>/done/ {path, result}` with `imported`, `known`, `too_big` (not
offered again) or `failed` (again once the claim ran out).
3. The server keeps them in `accounts.WebDAVSeen` (source, path, size, modified): a file whose
size changed counts as new. The folder itself is never changed.
The first such source: Shelfmark's e-book downloads on the Hetzner server, served read-only by
dufs (set up by the "Media" session; its notes: `C:\Users\m\homelab-handoff\04-hoerbuecher.md`).
## Zeitschriften: das Heft und seine Artikel
An issue is a unit (the editors' selection and order, its date, its theme) and
so is each article in it. Neither is reduced to the other
(`static/js/app/54-issues.js`):
- **The issue stays the document.** It opens and reads through as before, in
the editors' order. The reader marks the article one is in: a running head
under the bar ("Schwerpunkt · Artikel 8 von 23 · Titel"), in the apps' footer
"Artikel 8/23", and a **pause at each article's end** (`aside.iss-end`):
"✓ gelesen", the issue's count, "Weiter im Heft" (the next article not read
yet, skipping those read or skipped), "Zum Inhalt", "überspringen",
"Doch nicht gelesen". In a PDF the pause is a bar that comes up when one
turns from an article's last page onto the next.
- **The states determine each other.** An article one was in and whose end one
reached counts as read (a jump — from the contents, "Weiter" — reads
nothing). The issue counts "7 von 23 gelesen · 2 übersprungen"; "Im Heft
weiterlesen" goes to the article one was in, else the next one open.
"Heft abschließen" marks the rest as skipped, never as read.
- **In the library** an issue shows one bar with a segment per article (read,
skipped, open, the one one is in). A tap opens its contents: Rubriken, the
articles with their state, a teaser and the reading time; the ⋮ / mark
sets read / unread / skipped by hand.
**The index** comes from the file, on the device:
- **EPUB**: the table of contents, by file — an article is a file (the first
entry pointing into it); the entries after it into the same file are its
sections, whatever their depth (GegenStandpunkt puts some at the top level);
an entry whose children point into other files is a Rubrik. A one-file EPUB:
the leaves. No table of contents: each spine file with text. The teaser is
the headline under the title (the headings before the first paragraph that
the title doesn't say, not numbered sections), else the first paragraph.
- **PDF**: a real outline (word processors' `_Hlk…`/`_Toc…` leftovers don't
count); most magazines' PDFs have none, so mostly **the contents page
printed in the issue**: among the first 16 pages the one with most numbered
rows (numbers that merely count 1, 2, 3 … are a list, not a contents), plus
its neighbours with entries. A row: the page number at the start
("14 Der Krieg gegen Palästina", Jacobin), or at the end, maybe with the
author in a column before the title ("Nathan Sperber | Beyond Neoliberalism?
| 23", NLR); lines under it in its type size continue the title, an author
line ("Von …", a name) gives the author, lines in other type the teaser; a
title in capitals may begin on the lines above its number, a lone word on top
is its kicker (LuXemburg: GESPRÄCH, INFOGRAFIK); "Name: Titel"
(manuskripte). Capitals in the contents' type are Rubriken. The printed
numbers are mapped to the file's pages by the offset most titles agree on
(found in the page's text, a headline in capitals preferred); running heads
and the contents page's own head are dropped.
Reading times count 200 words a minute. It is kept in the encrypted meta as
`issue: {v: 2, a: [{h, t, r, x, m, a?, p?, e?}]}` (href, title, Rubrik, teaser,
minutes, author, a PDF's first and last page), or `{v: 2, a: [], none: true}`
for an issue without a usable table of contents (read as a whole). An index of
another version is built anew. Built in the
background for issues without one (not on e-ink: there when the issue is
opened, or it comes from another device).
**The states** are in the meta too: `issueRead: {href: 'r' | 's'}`,
`issueAt: href` (the article one was in). A change goes to the server after
2.5 s; offline it waits in `localStorage.diora_issue_pending` and goes with the
next save (or when the connection is back).

111
docs/navigation.md
View file

@ -1,111 +0,0 @@
# Navigation: Lesen and Hören
diora's areas are sorted by what one does, not where things come from:
| Group | Sections |
|-------|----------|
| — | Home |
| Lesen | Bücher · Zeitschriften · Nachrichten · Blogs |
| Hören | Hörbücher · Podcasts · Radio · Ambient (· Fokus, web page only) |
| Projekte | Projekte (docs/projects.md; `side`) |
| — (foot) | Suche (docs/search.md; `footer`, `transient`) · Abos & Quellen (docs/sources.md; `footer`) |
- **Desktop / wide window (sidebar):** every section is its own entry, under
its group's heading. Nothing inside a section repeats the choice.
- **Phone (tab bar):** four tabs, **Home · Lesen · Hören · Projekte** (since 2026-10-19).
Lesen and Hören show a row of their sections at the top (segments); each group remembers
the section last used. The search is a magnifier in the bar on top (the apps' section bars,
the page's narrow bar), never a tab; Abos & Quellen is in the app menu. The mini player
stays below as it is.
- **Sidebar / wide bar:** `side` sections (Projekte) and then the `footer` ones (Suche,
Abos & Quellen) stand at its foot. A `transient` section (Suche) is never remembered as
the one to open next.
- **diora Reader** (tolino, e-readers): only Lesen, without Home. **TV:** Hören.
Anything new goes into one of the groups. Managing (subscribing, settings,
folders) is in "Abos & Quellen" (docs/sources.md; done).
## One definition: `static/js/nav.js` (diora-web)
Loaded by the page on every platform, before the other area scripts. It owns
the sections, which panel shows each, switching between them, the URL hash
and the remembered choice. The bars are drawn from it. **It is the one source for the
navigation.**
Underneath, `static/js/app/20-tabs.js` keeps `TOP_TABS`: the list of *panels*
(`#tab-<name>`) that `showTab(name)` can show — home, radio, focus, podcasts, books, feeds,
ambient, audiobooks, sources, suche, projekte. That is not a second navigation: nav.js
maps each section to one of these panels (several sections share one: Bücher/Zeitschriften
→ books, Nachrichten/Blogs → feeds) and calls `showTab`. A new area needs both: its panel in
`TOP_TABS` (and the template) and its section in nav.js.
```
window.DioraNav = {
sections(), // [{id, group, label, icon, available}] in order (only those the page has)
groups(), // [{id: 'home'|'lesen'|'hoeren'|'projekte', label, icon, sections: [id…]}]
icon(name), // a section's or group's icon as SVG (also 'menu')
current(), // the section id shown now
go(id), // show a section ('lesen'/'hoeren': that group's last section)
onChange(fn), // fn(sectionId) after every change
}
```
| Section id | Group | Label | Panel | View inside the panel |
|------------|-------|-------|-------|-----------------------|
| `home` | — | Home | `#tab-home` | |
| `books` | lesen | Bücher | `#tab-books` | e-books (not magazines) |
| `magazines` | lesen | Zeitschriften | `#tab-books` | magazines |
| `news` | lesen | Nachrichten | `#tab-feeds` | news |
| `blogs` | lesen | Blogs | `#tab-feeds` | blogs |
| `audiobooks` | hoeren | Hörbücher | `#tab-audiobooks` | (audiobooks.js) |
| `podcasts` | hoeren | Podcasts | `#tab-podcasts` | |
| `radio` | hoeren | Radio | `#tab-radio` | |
| `ambient` | hoeren | Ambient | `#tab-ambient` | |
| `focus` | hoeren | Fokus | `#tab-focus` | web page only (`webOnly`) |
| `projekte` | projekte | Projekte | `#tab-projekte` | (projects.js) |
| `suche` | (foot) | Suche | `#tab-suche` | (search.js); never remembered |
| `sources` | (foot) | Abos & Quellen | `#tab-sources` | (sources.js) |
`go()` calls the page's `showTab(panel)` as before (so everything hooked onto
showTab keeps working) and then the panel's view hook:
`window.__dioraBooksView('books'|'magazines')`,
`window.__dioraFeedsView('news'|'blogs')`. The panels' own switches for that
choice (feeds.js News/Blogs) are hidden. `_currentTopTab` stays the panel name.
Hash: `#books`, `#magazines`, `#news`, `#blogs`, `#audiobooks`, … (old hashes
`#feeds`, `#radio/saved` keep working). Remembered: `localStorage`
`diora_nav_section` and `diora_nav_last_<group>`.
**Zeitschriften**: the books marked as magazines (encrypted meta `magazine`), or, unmarked,
those in a library folder called "Zeitschriften" (or "Magazine"); issues and articles:
docs/library.md.
### The segment row (phone)
nav.js draws `#nav-segments` above the panels: the current group's sections,
when the layout is the tab bar (not `html.nui-sidebar`) and the group has more
than one available section. Styled with the theme tokens; the native CSS may
restyle it.
## Who draws what
| Where | Bar | Drawn by |
|-------|-----|----------|
| Android (phone) | `#nui-tabbar`: Home · Lesen · Hören · Projekte | native-radio.js, from `DioraNav.groups()` |
| Android tablet, desktop (`html.nui-sidebar`) | rail: Home, headings Lesen/Hören with their sections, at the foot Projekte · Suche · Abos & Quellen, Menü | native-radio.js, from `DioraNav.sections()` |
| Apps, every section | a bar on top (`.nui-tab-bar`; `.nui-section-bar` for the generic ones) with the section's name, its actions and the search magnifier | native-ui.js `SECTION_BARS`, native-radio.js, native-podcast.js |
| Web page | `#tabs` (narrow: groups + segments; wide: all sections under headings) | nav.js |
A tab/rail button calls `DioraNav.go(id)`; the active state comes from
`DioraNav.onChange`.
## Phases
1. Navigation (this document), Hörbücher as their own panel, Zeitschriften by folder.
2. "Abos & Quellen": podcasts, YouTube channels, news sources, blogs and the
audiobook mark in one place; the everyday views without managing.
3. Library (done): audiobook series are found in the titles ("Das Kapital –
Erster Band" …) and shown as one stack; audiobooks have folders
(`PodcastFeed.folders`, a list; `folder` is its first, for older clients); a book is marked as a magazine in its encrypted
meta (`magazine: true/false`; unset: the "Zeitschriften" folder decides), and
Zeitschriften groups the issues by periodical.

46
docs/news-text.md
View file

@ -1,46 +0,0 @@
# Lesehilfen für Nachrichten (feeds/textproc.py)
Klassische Textverfahren, kein Sprachmodell; reines Python auf dem Server, für die Nachrichten
eines Nutzers auf einmal. Ausprobiert an echten Nachrichten (Oktober 2026), übernommen wurde,
was dort trug.
## Floskeln
Sätze, die eine Quelle unter mindestens drei ihrer letzten 40 Artikel setzt (Spendenboxen,
„Diese Nachricht wurde … gesendet“, Abo-Hinweise), gehören zu keinem Artikel. `GET
/feeds/items/<id>/` liefert den Artikel ohne die Absätze, die nur aus solchen Sätzen bestehen; der
Überblick sieht sie gar nicht erst. Je Quelle sechs Stunden zwischengespeichert.
## Überblick einer Meldung
`GET /feeds/stories/<id>/overview/` – nur für Meldungen aus mindestens zwei Quellen:
- `summary`: drei Sätze aus allen Berichten, die am meisten von dem sagen, was alle sagen
(LexRank), ohne einander zu wiederholen (MMR); `[{source, text}]`;
- `novel`: je Quelle bis zu zwei Sätze, denen in keinem anderen Bericht etwas ähnelt (tf-idf-Kosinus
< 0,2), ohne die schon im Überblick; `{source: [text]}`;
- `words`: je Quelle ihre eigenen Wörter für das Ereignis (Log-Odds gegen die übrigen Berichte, mit
Dirichlet-Prior; nur großgeschriebene Formen, also Substantive und Namen); `{source: [word]}`;
- `related`: dieselbe Meldung in der anderen Sprache (Deutsch/Englisch) – Meldungen der letzten zwei
Tage, die mindestens drei seltene Namen teilen (nach idf gewichtet, Summe ≥ 17); Story-Objekte wie
in `/feeds/news/`.
Die App zeigt das über dem Artikel als „Überblick aus N Quellen“, darunter die Meldungen in der
anderen Sprache. Android lädt die Überblicke ungelesener Meldungen mit vor (Repo.kt), damit sie auch
offline da sind.
## Themen
Die erste Seite von `GET /feeds/news/` hat `topics: [{label, ids}]`: Meldungen der letzten zwei
Tage, die von derselben Sache handeln, nicht nur vom selben Ereignis. Gruppiert nach Titel und
Anriss (Leader-Clustering gegen die Mittelpunkte, drei Runden). Ein Thema hat mindestens drei
Meldungen aus mindestens zwei Quellen, liegt dicht beieinander (mittlere Ähnlichkeit zum Mittelpunkt
≥ 0,3) und hat ein Etikett aus mindestens zwei großgeschriebenen Wörtern, die die meisten seiner
Meldungen teilen. Mit `category` nur der Anteil der Kategorie. `?ids=1,2,3` liefert genau diese
Meldungen (ein Thema antippen).
## Verworfen
Stimmung (SentiWS), steigende Begriffe, Fortsetzungsketten (Nachrichten laufen nach zwei Tagen ab),
Clickbait-Erkennung: im Versuch zu ungenau oder ohne Nutzen. Persönliche Reihenfolge: erst, wenn
genug gelesen ist, um daraus zu lernen.

193
docs/podcasts.md
View file

@ -1,193 +0,0 @@
# Podcasts: Strom, Merkliste, Wiedergabeliste
The plan agreed on 2026-10-19 (with the user, from their listening history). It replaces
the inbox model ("Neu" to work through, "Folgen" beside it, one queue that is both a store
and the play order). Server: diora-web (podcasts/). The podcast UI is diora-android's
`web-native/static/native-podcast.js`, copied into `static/podcasts/` — built there.
## Why
- The subscriptions (about 540) are a self-chosen part of the podcast world: episodes to see
go by and dip into now and then — not a list to work through. "Neu" had 3360 unplayed
episodes; nobody works through that.
- The queue held 74 episodes (100+ hours): a store of "want to hear some day", yet the
player played it in order as if it were the play order.
- In AntennaPod "download automatically" and "add to queue automatically" were one thing
for the user: a regular podcast is one whose new episodes are on the phone. diora split
them (177 feeds downloading, 8 queueing).
- "Silent" subscriptions came from AntennaPod fetching every feed on the phone, slowly. The
server fetches all feeds anyway now (conditional, every 15 minutes).
## The lists
**Wiedergabeliste** — short, what plays next. The player goes through it in order and
**stops when it is empty** (it does not take from the Merkliste). Every episode in it is
downloaded. Filled by hand: ▶ on an episode (plays it, puts it first), or "In die
Wiedergabeliste" from the Merkliste or the Strom.
**Merkliste** — long, "want to hear". It may grow; it **never expires**. Grouped by podcast,
each podcast **collapsible** (newest first inside). New episodes of Stamm subscriptions land
here and are downloaded; so is what the user adds with ＋. An episode played from it moves
to the front of the Wiedergabeliste; finished, it leaves both.
When an episode leaves both lists (heard, removed), its download is deleted (setting
`delete_dequeue`). Downloads follow the lists, not the subscriptions.
## Subscriptions: three levels
| level | new episodes | today's mapping |
|---|---|---|
| **Stamm** | into the Merkliste and downloaded (per feed: or straight into the Wiedergabeliste — an option, used by none yet) | `new_episodes='queue'` (8) plus the regularly heard ones (about 50, list below) |
| **Strom** (default) | in the Strom; nothing to work through, nothing downloaded | `'inbox'`, and the former silent `'ignore'` ones |
| **Auf Abruf** | not in the Strom; on the podcast's page and in search | — (for exceptions) |
The per-feed "download automatically" switch goes: Stamm means it.
**Fetch interval** per subscription: *Automatisch* (by publishing rhythm, `interval_days`:
daily publishers hourly, rare ones daily), *stündlich*, *täglich*, *wöchentlich*. The former
silent subscriptions become Strom with *täglich*.
## The Strom (replaces "Neu" and "Folgen")
All episodes of all Strom and Stamm subscriptions, by period (Heute, Gestern, Diese Woche,
…). Within a period the podcasts are ordered by **how much the user has listened to them**
(episodes heard): every podcast with at least one heard episode before every one without —
those count for infinitely more (heard in the last 12 months, `heard_365`); among the never-heard ones, rarely publishing first.
Each podcast appears **once per period, always bundled**: its newest episode, and "+n ›" for the others. Episodes new since
they were last seen carry a **small mark** (no counts). Per episode: ▶ (Wiedergabeliste,
first), ＋ (Merkliste), ★. Filters stay: Favoriten, Verlauf, Angefangen, Gehört.
Tabs: **Wiedergabeliste · Merkliste · Strom · Abos**.
## Favourite subscriptions
★ on a subscription (`PodcastFeed.favorite`): a mark and a filter (Abos: only the favourites;
episode lists: `filter=fave_feeds`), whatever its level — it puts nothing on the Merkliste by
itself. Started with the Stamm subscriptions marked.
## Stöbern (decided 2026-10-19)
Within the subscriptions:
- **Search** in titles and shownotes: `GET /podcasts/discover/search/?q=…&stream=1` (all
words, the last as a prefix; titles weigh more). SQLite FTS5 index, contentless
(`podcasts/discover.py`), filled after every refresh; `manage.py index_episodes` fills it.
- **Similar episodes** from the other podcasts: `GET /podcasts/discover/similar/<id>/`.
- **People** (who talks, who gives a talk), from episode titles only (`podcasts/people.py`):
`GET /podcasts/discover/people/` → `[{name, episodes, feeds, score}]`, the ones in the most
listened podcasts first; `GET /podcasts/discover/person/?name=` → their episodes.
- **Topics** of the Strom's last days, chips as for the news: `GET /podcasts/discover/topics/?days=7`
→ `[{label, ids}]`.
- **Forgotten**: `GET /podcasts/discover/forgotten/` → `{loved: [...], rare: [...]}` — unheard
episodes older than 60 days of the podcasts listened to most, and of the rare ones; the same
all day.
One controlled step out (Apple's episode search and fyyd; no key):
- **People elsewhere**: `GET /podcasts/discover/elsewhere/?name=` → episodes naming the person
in podcasts the user doesn't have (`{source, podcast, feed_url, title, audio_url, guid,
pub_date, duration_seconds, artwork_url, link}`). `POST /podcasts/discover/take/` with such an
item puts the episode on the Merkliste; its podcast becomes an *Auf Abruf* subscription.
- **Recommendations**: `GET /podcasts/discover/recommend/` → podcasts like the most listened ones
(fyyd), recommended for several first: `{title, feed_url, artwork_url, description, link,
because: [titles], count}`. Subscribing: the usual feeds/add/ (then *Auf Abruf* or Strom).
Show search stays on Apple (best coverage).
## Podcast-Radio (since 2026-10-25)
Listening without committing: one button plays the Strom like a station.
- **The rule**: the radio always plays the **newest** (publication date) Strom episode that is
neither played, nor started (normally), nor done for the radio. So an episode that comes out
while one plays is next (the one playing isn't interrupted); otherwise it goes back in time,
without limit. Auf Abruf podcasts and audiobooks aren't in the Strom, so not in the radio.
- **Mixed**: several episodes of one podcast in a row are mixed with the others waiting — the
next is the newest of *another* podcast than the one before, if there is one (within the
newest 200).
- What comes next is asked anew at every end and skip (so the newest really comes next); the
list fetched before is for "Danach: …" in the player and for going on offline.
- **Leaves no trace**: no progress (no Verlauf, no "Angefangen", no played mark, not counted in
`heard_365`), the Wiedergabeliste and Merkliste unchanged, nothing downloaded (it streams; a
downloaded file plays if there is one), the Strom's marks unchanged.
- **Done for the radio**: run to its end or skipped (⏭, notification, headset, car). Pausing
isn't: the radio keeps its episode and place for itself, ▶ goes on there.
- **Committing**: in the player while the radio plays, ＋ (Merkliste) and "Normal weiterhören"
(first in the Wiedergabeliste at the current place; from then on it plays as usual, with
progress). Playing anything normally switches the radio off (as an audiobook does).
- **Server** (`podcasts/radio.py`): `PodcastRadioHeard(user, episode, at)`, kept apart from
progress and kept for good (pruned, an old episode would come back when the radio goes back
that far). `GET /podcasts/radio/?after=<playing>&skip=<ids>&limit=5` → the next episodes by
the rule (`{episodes: [...]}`, the rows of the episode lists; `after` isn't among them and its
podcast not first; `skip`: ones the device is done with whose /heard/ may still be on its
way); `POST /podcasts/radio/heard/ {"episode_id"}`. Shared by all devices. Filter `untouched`
(episode lists): neither played nor listened into.
- **Player sync**: a report carries `podcast_radio: true` while the radio plays
(`PlayerState.podcast_radio`); taking it over ("Hier abspielen", a transfer) or adopting it goes
on with the radio on that device.
- **Clients**: the web player (`podcasts-web.js`: an entry with `radio: true`, the radio's own
`podradio` in localStorage — next ones, its place, what it was done with lately; host
`podcastRadio()`, `podcastRadioCommit()`, `podcastRadioLater()`; `playerState().radio =
{next}`) and Android (`PodcastRadio.kt` beside `BookPlay.kt`; RadioService's `save()` /
`markPlayed()` / `followQueue()` leave radio items alone). UI (native-podcast.js): 📻 at the top
of the Strom, "Podcast-Radio · <podcast>" in the mini player and the player, there a box with
"Danach: …", ＋ Merkliste and "Normal weiterhören"; ⏭ only skips.
- Later perhaps: a radio of one folder, a longest length.
## Moving over
- Today's queue (74) becomes the Merkliste; the Wiedergabeliste starts empty.
- Stamm: the 8 automatic subscriptions and the regularly heard ones (≥ 8 episodes a year or
≥ 3 in 90 days, 2026-10-19): Ones and Tooze, Die Neuen Zwanziger – Der Salon, Surplus,
The Deprogram, junge Welt – Tagesüberblick, Speakeasy-Bar, jwpod – Forschung, RLS
Audiodokumente, The Reaction Pod, Geschichten aus der Geschichte, Märkte Mächte Emissionen,
friendlyjordies, Second Thought, Neben der Spur, Forschungsquartett, GegenStandpunkt –
Artikel, Hörsaal, Hakim, Marxistische Vorträge, Blätter, Kontextfenster, New Left Review,
Kommunisten Kneipe, Jung & Naiv, HealthyGamerGG, Dekarldent, Daily Dose Of Dekarldent,
Eine Stunde History, Übertage, Eigentlich, Zwischen Zwei Deckeln, UNFASSBAR, Über Rechts,
TitosLeftLung, New Books in History, ARTEde, Irgendwas mit ARTE, Zwei wie Pech & Schwafel,
Geschichte der kommenden Welten, Vollbild, Zeitfragen, Zeitfragen. Feature, Die Filmanalyse,
Reaction, Wirtschaft am Mittag.
- Everything else: Strom; the silent ones with *täglich*.
## Order of work
1. Wiedergabeliste and Merkliste (server: a second list, moving the queue over, the player's
order; UI: two tabs, grouped Merkliste).
2. The three levels and the fetch interval; downloads following the lists.
3. The Strom replacing "Neu" and "Folgen".
4. Browsing — after more thought.
## API (server, since 2026-10-19)
- `PodcastSettings.lists_v2` turns the two lists on. `POST /podcasts/lists/ {"on": true}`
moves the queue into the Merkliste (the Wiedergabeliste starts empty); `{"on": false}` puts
the Merkliste back at the end of the queue. Off, everything works as before (apps not
updated yet).
- **Wiedergabeliste** = the queue as it was: `state.queue`, `/podcasts/queue/add|remove|reorder/`.
The player goes through it and stops at its end.
- **Merkliste** = `PodcastLater`: `state.later` (rows like the queue's, newest first, plus
`added_at`, `auto` — added by a Stamm feed — and `in_queue`), `POST /podcasts/later/add/` and
`/podcasts/later/remove/` `{"episode_ids": [...]}`. Heard (progress at the end, mark-played)
and "Folge löschen" take an episode off it. Episode lists carry `in_later`; filter `later`.
- **Levels**: `PodcastFeed.new_episodes` — `queue` = Stamm into the Merkliste (with lists_v2;
without it: the queue), `play` = Stamm into the Wiedergabeliste, `inbox` = Strom,
`ignore` = Auf Abruf. `auto_queue` stays true for queue and play.
- **Fetch interval**: `PodcastFeed.refresh_interval` — `''` (by rhythm: every `interval_days`
hours, within 15 minutes and 24 hours), `hourly`, `daily`, `weekly`; the cron's
`refresh_feeds` only fetches feeds that are due (`views.refresh_due`).
- **Strom**: `GET /podcasts/episodes/?filter=stream` (all but Auf Abruf; with `folder=`, `sort=`);
episodes carry `discovered_at`. Per feed in `state.feeds`: `heard_365` (episodes heard in the
last 12 months: played, or listened into for more than a minute) and `interval_days`.
`POST /podcasts/stream/seen/` → `{before, now}`; `state.stream_seen_at`. An episode is marked
when its `discovered_at` is after the time before the current look.
- Web host (`static/js/podcasts-web.js`, `DioraPodcastHost`): `podcastLaterAdd(json episodes)`,
`podcastLaterRemove(json ids)`, `podcastListsSwitch(on)`, `podcastStreamSeen()` (keeps the
previous time as `state.stream_seen_before` for the marks).
## Transcripts
An episode's Transkript tab shows one of two kinds. **From the feed**: `<podcast:transcript>`
(SRT, VTT, JSON, Podlove), fetched and parsed on first use and cached in `EpisodeTranscript`
(`podcasts/transcripts.py`, `GET /podcasts/episodes/<id>/transcript/`). **Made by Whisper**
(since 2026-11): only when the user asks, kept on `TranscriptionJob`; the feed's wins when both
exist. From a transcript the device can write an article that is pinned to the episode and shows
in Blogs. All of that is in docs/transcription.md.

71
docs/progress-sync.md
View file

@ -1,71 +0,0 @@
# Reading position: the last place read wins
Since 2026-10-12 the place to continue a book is **where it was last read, on any device**, not the
furthest place ever reached. The furthest place is still kept, as a hint ("Weiteste Stelle").
This file is the contract between the server (books/views.py `save_progress`), the web reader
(static/js/app/46-readpos.js, with the reader in 62-reader.js), diora-android (Ops.kt / SyncEngine.kt / native-*.js) and diora-desktop
(src/main/ops.js / sync.js). They must apply the same rules.
## The state per book
| field | meaning |
|---|---|
| `scroll_fraction`, `position_anchor` | the continue point (anchor `"block:inner"`, empty for PDFs) |
| `read_at` | when the reader was **at** that place, ms since epoch, **server clock** |
| `device` | who read there (`"Browser"`, `"diora Desktop"`, `"Handy"`, `"tolino"` …), for "auf X weitergelesen" |
| `furthest_fraction`, `furthest_anchor` | the furthest place ever reported (only ever grows) |
`last_read` in `/books/` is `read_at` as ISO time (it used to be the time of the last save attempt).
## Writing: `POST /books/<id>/progress/`
```json
{"scroll_fraction": 0.42, "position_anchor": "187:0.62", "read_at": 1760270000000,
"sent_at": 1760270004000, "device": "diora Desktop"}
```
* `read_at` is **the moment the reader arrived at this position** (the last time the visible position
changed because of the reader), in the **device's** clock, not the moment of saving. Saving an
unchanged position again — every 10 s, on hide, on close, on unload — therefore carries the old time
and can never beat a newer position from another device. A device that only *shows* a position it
got from the server (just opened, nothing moved) reports the server's `read_at` back unchanged.
* `sent_at` is the device clock **when the request leaves** (an outbox sets it at flush time, not
when queuing). The server corrects the device's clock error with it:
`t = read_at + (server_now − sent_at)`, then `t = min(t, server_now)`.
So a tolino whose clock is hours off still orders correctly, and nothing can win "from the future".
* The server stores the position if `t >= stored read_at` (or nothing is stored, or `force`).
Older writes (an offline device catching up late) are **not** stored as the continue point.
* Every write — stored or not — raises `furthest_*` if it is further (same comparison as before:
anchor block, then inner fraction; `scroll_fraction` when an anchor is missing).
* `force: true` (the explicit "Diese Stelle als Fortschritt setzen") stores regardless, with `t = server_now`.
* **Old clients** (no `read_at`): stored only if **strictly further** than the stored position
(the old rule, but `>` so an idle old client re-saving its place changes nothing), with `t = server_now`.
This keeps not-yet-updated apps from pulling the continue point back.
* The response is the state after the write:
`{"ok": true, "stored": bool, "scroll_fraction", "position_anchor", "read_at", "device",
"furthest_fraction", "furthest_anchor", "server_now"}`.
## Reading
`/books/` and `/api/sync/` (`book_progress`) carry all fields of the state, `read_at` in ms (server clock),
plus `server_now` (ms) so a client can estimate its clock offset `server_now − device_now`.
## Clients
* **Local apply / outbox** (Android, desktop, web cache): the same rule as the server, in server time:
a local write converts its `read_at` with the last known offset (0 if none), stores it if newer
than or equal to what the local list holds. Two queued writes for the same book: keep the one with
the later `read_at` (force stays sticky). After a pull, a pending write is reapplied with that rule:
newer than the server's → it shows (and is sent); older → the server's position shows (it's still sent,
for `furthest`).
* **Never save before the position is restored** after opening a book, and never save a position the
reader didn't reach by reading: while a jump is tentative ("Zurück zum Text" after a footnote or link,
"Weit gescrollt — Zurückspringen"), the previous position and its `read_at` are what gets saved, until
the reader returns (nothing changes) or carries on reading there (then the jump target counts, with
its own time).
* **Opening a book** uses the newer of server state and local (not yet sent) state, by `read_at`.
* **Coming back to an open book** (window/app in front again, or after a pull): if the server holds a
newer position from another device than the one shown, offer it: "Auf <device> weitergelesen —
Springen". Do not move the page without asking.
* If `furthest_*` is clearly ahead of the continue point when a book opens, offer
"Weiteste Stelle: 63 % — Springen".

96
docs/projects.md
View file

@ -1,96 +0,0 @@
# Projekte
Projects of a preset: the first is **Forschung** (a research project: sources, citations,
notes; Radar and Schreiben follow). In the navigation (`nav.js`, section and group `projekte`,
flag `side`): a tab beside Home · Lesen · Hören on a phone, at the foot of the sidebar and of
the wide bar above Suche · Abos & Quellen. The search moved out of the phone's tab bar: a
magnifier in the bar on top (`suche` is a footer section; the apps' section bars show it).
## End-to-end encrypted
Like the books (docs/library.md): a project's data (`{name, description}`) and every item's
data are AES-GCM blobs with the user's key (`32-crypto.js`, iv hex, ciphertext base64). The
server stores them and knows only: a project exists, its preset, archived, and what an item
points to — `kind` + `ref` (a book's, episode's or article's id, never its title).
## Items
| kind | ref | data |
|---|---|---|
| `paper` | – | `{key, csl, note, doi}` (CSL-JSON from doi.org) |
| `book` | book id | `{key, csl, note, type}` (CSL from the book's meta) |
| `episode` | episode id | `{key, csl, note, podcast, feed, position}` (position: seconds, "ab 12:34") |
| `article` | feeds item id | `{key, csl, note}` |
| `note` | – | `{title, body (Markdown), folder}` |
A book, episode or article is in a project once (adding again answers `existing: true`).
`key` is the citation key for `[@key]` / `[@key, S. 12]` in notes: first author's family name
(or the podcast's), year, first word of the title — unique in the project, editable.
Notes: Markdown (headings, lists, quotes, bold/italic, links); the view shows citations as
"(Amodio & Jost 2007, S. 3)" and a list of the works cited. Saved as one types (0.9 s).
Export: BibTeX and CSL-JSON of the sources (all or the chosen kind).
## Linking from elsewhere
`DioraProjects.link(kind, ref, info)` opens the sheet "Mit Projekt verknüpfen" (the projects,
the ones it is in ticked, ＋ Neues Projekt). Called from the book sheet (library, apps), the
article reader (◇ in its bar), the episode sheet and the player (apps, native-podcast.js).
`DioraProjects.of(kind, ref)` → `[{id, name}]` the projects it's in; `onChange(fn)` after changes.
## Desktop: a window of its own
The project's menu (⋯) has "In neuem Fenster öffnen" when `dioraDesktop.openWindow` is there:
the desktop opens the page with `?project=<id>`; `projects.js` then shows only that project
(`html.prj-solo` hides the navigation and the dock).
## Writing and conflicts
Every write sends `base_iv` (the IV it changed). Changed elsewhere since: 409 with the current
state. A note's text then goes beside it as a copy "(Konflikt)"; other data reloads.
## API (signed in)
- `GET /projects/` → `{projects: [{id, preset, data_ct, data_iv, archived, position, items, …}], links: [{project, kind, ref}], presets}`
- `POST /projects/create/` `{preset, data_ct, data_iv}` → `{project}`
- `GET /projects/<id>/` → `{project, items: [{id, kind, ref, data_ct, data_iv, created_at, updated_at}]}`
- `POST /projects/<id>/` `{data_ct, data_iv, base_iv}` and/or `{archived}` / `{position}`
- `POST /projects/<id>/delete/`
- `POST /projects/<id>/items/add/` `{kind, ref?, data_ct, data_iv}` → `{item, existing}`
- `POST /projects/items/<id>/` `{data_ct, data_iv, base_iv}` → `{item}` (409 `{item}` when stale)
- `POST /projects/items/<id>/delete/`
- `GET /projects/doi/?q=<DOI | doi.org link | arXiv link or id>` → `{doi, csl}` — fetched from
doi.org (Crossref, DataCite …) as CSL-JSON, cached a week, nothing else kept.
## Radar
A project can have a Radar (`projects/radar.py`, `projects/papers.py`): every week the new papers
from PubMed, arXiv, bioRxiv and medRxiv, sorted by Claude into the config's groups (the first
one — "lesenswert" — with what it's about, how solid it is, the political hook), the week's
news (web search), a basic term explained. One Claude batch (Opus, half price): about 0.40 $
a week. Ported from the "Neuropolitik Grundlagen" session's `F:
europolitik ools`
(fetch_papers.py, digest_batch.py); the digest is their bundle format (version 1).
- Config per radar (`Radar.config`, presets in `projects/radar_presets.py`): PubMed queries per
topic, arXiv query, keyword pattern for the preprint servers, groups and their labels, the
prompts, weekday and hour (Europe/Berlin), days. **Plain on the server**: it searches with them.
The papers and digests are public material, plain too (`RadarDigest.bundle`).
- The decisions (gelesen / später / verworfen / in Quellen) are the user's: an encrypted item
of kind `radar` (`{decisions: {paper id: state}}`). "→ Quellen" makes a `paper` source
(citation data from doi.org when there's a DOI, else from the record).
- Runs from the host's crontab every half hour: `docker exec diora-prod python manage.py radar`
(submits what is due, collects finished batches). Needs `ANTHROPIC_API_KEY` in the container.
`radar --setup <project> neuropolitik` gives a project its Radar; `radar --import <project>
<file>` takes a digest made elsewhere; `radar --seen <project> <file>` ids reported before.
- API: `GET /projects/<id>/radar/` → `{radar: {enabled, title, groups, group_labels, weekday,
hour, days, sources, running, submitted, last_run_at, last_error, next_run, key}, digests:
[{id, generated, bundle}]}` (the latest 12); `POST …/radar/settings/` `{enabled}`;
`POST …/radar/run/` (fetch at the next cron run); `POST …/radar/import/` (a bundle, e.g.
`digest_batch.py --diora-url`).
## Next
Papers as books (a paper's PDF into the library, its highlights
into the project's notes), Schreiben (drafts with the bibliography in a citation style),
the projects in the search, more presets.

137
docs/radio.md
View file

@ -1,137 +0,0 @@
# Radio, Ambient, Fokus
The oldest part of diora (it began as a radio player): internet radio with saved stations,
history and track titles, the generative Ambient player, and the web page's Fokus timer.
Written down 2026-10-24 from the code; it is the reference for the three apps.
## Server: the `radio` app
Models (`radio/models.py`):
| model | what |
|---|---|
| `SavedStation` | a user's station: name, url (unique per user), bitrate, country, tags, `favicon_url` (the logo), `is_favorite`, notes |
| `StationPlay` | one listening: station, started/ended; hour, weekday, weekend filled on save (for the recommendations) |
| `TrackHistory` | a track title heard (user, or `session_key` signed out), `scrobbled` |
| `FeaturedStation` | stations picked in the admin for everybody (order, active) |
| `FocusSession` | a finished Fokus round |
Also: `podcasts.PlayerState` with `kind='radio'` (the player sync between devices) and
`accounts.UserProfile.focus_station_url/_name` (the station the Fokus timer plays).
Endpoints (`radio/urls.py`):
- `GET /` — the page (`templates/radio/player.html`). The saved stations, the history (50) and
the featured stations come rendered into it (`#saved-tbody`, `#history-tbody`,
`.featured-section`) and as `INITIAL_SAVED` (with `play_count`), `INITIAL_FEATURED`,
`USER_FOCUS_STATION`. The apps take these parts from it (see below).
- `GET /radio/sse/?url=&station=` — server-sent events with the track title, `data: {"track": …}` when it
changes, `{"error": "stream ended"}` at the end. The server opens the stream itself with
`Icy-MetaData: 1` and reads `StreamTitle` (`radio/icy.py`). Which addresses it opens
(`radio/streamcheck.py`, since 2026-10-24): never one in a private network (loopback,
10.x, 172.16–31.x, 192.168.x, link-local, the Docker network); **without an account only the
directory's stations** — the page's curated lists (read from `16-stations.js`), the featured
stations, the creamfresh stream, and what radio-browser.info lists (`station=` its station
id, which the page sends for search results; else by the address; kept a day). An address of
one's own is for saving, and saving needs an account: signed out, a pasted address offers
signing in instead of ▶.
- `POST /radio/record/` — a track into the history; `scrobble: true` sends it to Last.fm when the
account has it linked (`radio/lastfm.py`; "Artist - Title", else the station as artist).
- `POST /radio/save/` (by url: create or update), `/radio/remove/<pk>/`, `/radio/favorite/<pk>/`,
`/radio/notes/<pk>/`; `GET /radio/history/` (last 200), `POST /radio/history/<pk>/delete/`.
- `POST /radio/play/start/` → `play_id`, `/radio/play/stop/`; `GET /radio/recommendations/` —
the five stations most played around this hour on this kind of day, with a German label.
- `POST /radio/import/` — an `.m3u`/`.m3u8` file into the saved stations (`#EXTINF` names) →
`{added, skipped}`.
- `POST /radio/focus/record/`, `GET /radio/focus/stats/`.
- `GET /radio/affiliate/?track=` — artwork/album from iTunes (an Amazon link only with
`AMAZON_AFFILIATE_ENABLED`).
- `GET /radio/creamfresh-stream/`, `POST /radio/creamfresh-feedback/` — the user's own station
(radio.creamfresh.xyz) through the server, which adds its login; 👍/👎 for the DJ.
- `GET /radio/stream-player/` — an old standalone player page; nothing links to it.
**Station search** is not on the server: the page asks radio-browser.info directly
(`de1.api.radio-browser.info/json/stations/search`, 50, by click count). A pasted `http(s)://`
address is offered to play or save as it is.
## Web page (`static/js/app/`)
| file | what |
|---|---|
| `01-core.js` | the globals: `audio` (one `<audio>`), `currentStation`, `currentTrack`, `sseSource`, `isPlaying`, `currentPlayId` |
| `10-radio.js` | `playStation(url, name, id)`, `stopPlayback`, `togglePlayStop`, the SSE (`startMetadataSSE` → `recordTrack`), volume (0–255, `diora_volume`), search, saving, favourites, recommendations, the creamfresh votes, Media Session |
| `16-stations.js` | mood chips (they search a tag), curated lists, featured stations, the occasional donation hint, notes |
| `29-m3u.js` | M3U import |
| `80-radio-sidebar.js` | the compact radio player of the sidebar |
| `24-ambient.js` | the Ambient tab (below) and `showRadioTab` |
| `14-focus.js` | Fokus (below) |
Sub-tabs `RADIO_SUB_TABS = ['search', 'saved', 'history']` (`20-tabs.js`), hash `#radio/saved`,
remembered in `diora_active_radio_tab`. On an https page an `http://` stream would be blocked:
the page opens it in a new tab — unless the host plays it (`window.DIORA_PLAYS_HTTP_STREAMS`,
set by the apps). No HLS: what `<audio>` plays.
## Ambient (`static/ambient/index.html`, "Drift")
No sound files: a generative WebAudio synthesizer. Modes (`MODES`): ambient music (chords,
bells, grains), noise colours, nature (rain, storm, sea, wind, stream, fire, crickets),
machines (fan, cabin, train), focus and sleep (binaural delta–beta, singing bowls,
heartbeat); each with two sliders and seeded presets in the URL hash.
The page holds it in an iframe, loaded when first needed (`24-ambient.js`); they talk by
`postMessage` (`{type: 'ambient', cmd: 'start'|'stop'|'mode'|'volume'}` in,
`{state: 'playing'|'stopped'}` out). Mode remembered in `diora_ambient_mode`.
Hooks for the apps: `__dioraAmbientStart/Stop/Cmd/Info/Playing`.
**One sound at a time:** Ambient starting stops the radio (`stopPlayback(false)`) and pauses a
podcast (`__dioraPausePlayer()`; Android: `podcastPause()`, pause only, never toggle); the
radio starting stops Ambient.
## Fokus (web page only)
A Pomodoro timer (25/5 min) in its own section (`nav.js`: `focus`, `webOnly`): it plays the
chosen focus station, records a `FocusSession` after each work phase, pauses the sound for the
break; with a do-not-disturb full-screen mode (`toggleDND`) and a statistics table.
## Android (diora-android)
- **One player** for radio, podcasts and audiobooks: `RadioService.kt`, a Media3
`MediaLibraryService` with one ExoPlayer (15 s connect / 20 s read timeout; diora's own
server gets the login headers — the creamfresh proxy). After a pause of more than 3 s a
station jumps back to live. Android Auto: Saved, Featured, Queue, voice search by name.
Resuming after a restart plays the last station. Radio always streams on mobile data (the
mobile-data rules are for episodes).
- `Radio.kt` — the page's controller: play, stop, `track(title)` (new metadata without
rebuffering), state to the page (`window.__dioraRadio`).
- `RadioPage.kt` — the saved stations, history and featured stations exist only in the
rendered `/`: the app fetches it with its login, cuts those parts out into `radio-page.json`
and splices them into its own page at markers that `tools/build_web.py` set
(`<!--diora:saved|history|featured-->`, `__DIORA_RADIO_*_JSON__`). So they show offline too.
- `LocalServer.kt` passes `/radio/sse/` through as a stream (padded with SSE comments: WebView
hands intercepted data over only when its buffer is full); POSTs go through `native.js`.
- `web-native/static/native-radio.js` — turns the page's `audio` into a remote of the native
player (`radioPlay`/`radioStop`, counters against stale echoes), shows errors ("Sender nicht
erreichbar"), draws the dock/rail (from `nav.js`), the mini player and the radio bar.
- `AmbientService.kt` — Ambient as a foreground media service (notification with Stopp, audio
focus, stops when headphones are pulled); started and stopped from the page
(`ambientState`). Widgets for Ambient and resuming (`CardWidgets.kt`).
- Sleep timer (`SleepTimer.kt`, `SleepRules.kt`): counts only playing time (Ambient too), fades
out over the last 10 s, shaking resets it, can stop after N episodes, can switch itself on at
night (22–6 h); one that counted nothing for 3 h is dropped.
- Data usage: the stream counts as `RADIO_STREAM`, `/radio/` and `/` as `RADIO_DATA`
(`DataUsage.kt`).
## Desktop (diora-desktop)
A copy of `native-radio.js`; `src/web/desktop.js` plays with a hidden `Audio()` of its own
(`DioraNative.radioPlay/Stop/State/Refresh/Track`; media keys: pause = stop) and re-reads `/`
like RadioPage.kt (at most every 20 s). `http://` streams go through the local router
(`/__stream?u=`, `src/main/router.js`) without their ICY data; the SSE is passed through with the
login; `backgroundThrottling: false` keeps the titles coming.
## Known weak spots
- The login for the creamfresh stream is written into `radio/views.py`; it belongs in the
environment like the other secrets.
- Every listener's titles cost the server a second connection to the stream.
- No recording, no alarm; station logos only from radio-browser's favicon.

53
docs/refactoring.md
View file

@ -1,53 +0,0 @@
# Refactoring (started 2026-10-14)
diora grew fast across four code bases that share one web front end:
diora-web (server + app.js and friends), diora-android (Kotlin + web-native/*.js),
diora-desktop (Electron + desktop.js) and the vendored copies between them.
This is the plan for tidying it up without breaking anything.
## Rules
* **No behaviour changes in a refactoring step.** Features and fixes go in their own commits.
* **Tests before moving code.** A step only starts when what it touches is covered.
* **Small steps, each one releasable**, each one checked on web, desktop and Android.
* Whoever owns a file does its refactoring (web-native/* → the Android session;
app.js, home.js, templates, desktop → the desktop session); shared contracts live in docs/.
## The numbers (2026-10-14, before step 2; app.js is now `static/js/app/`)
| file | lines |
|---|---|
| static/js/app.js | 6 864, about 275 functions, about 120 globals, 45 sections |
| static/podcasts/native-podcast.js (copy from diora-android) | 4 554 |
| static/css/app.css | 2 010 |
| static/js/podcasts-web.js | 1 899 |
| podcasts/views.py | 2 505 |
| web-native/native.css | 2 150 |
Web JavaScript has no unit tests and four Playwright smoke tests that run nowhere.
The reading-position rules exist four times (books/views.py, app.js, desktop ops.js,
Android Ops.kt), each tested separately or not at all.
## Steps
1. **Safety net** (in progress)
* Playwright tests that run locally and in CI against a throwaway database: log in, open a
book (an EPUB fixture), read, the position is saved and restored, a highlight, a note
in the margin, the theme switch, radio and podcasts tabs load.
* Shared test vectors for the reading-position rules (`docs/progress-cases.json`), run by
the Django tests, desktop's node tests and Android's OpsTest — one contract, checked
in all four places.
2. **Split app.js** (done: 656520f, sw v106 — `static/js/app/NN-*.js`, 33 files; the
template lists them in order) by its sections into plain scripts loaded in order (radio, focus,
ambient, crypto, epub, books, reading position, reader, tts, highlights, margin, …). They
stay classic scripts sharing globals, so the native layers that wrap `window.*` keep
working unchanged; template, sw.js SHELL and Android's build_web list them. No code
changes inside the moved functions.
3. **Dead code**: features and helpers nothing calls any more (found by the split and by
coverage of the tests), removed one at a time.
4. **Server**: podcasts/views.py into modules by area (feeds, episodes, queue, player sync,
import); shared helpers (`_require_auth`, JSON body parsing) in one place.
5. **CSS**: the remaining fixed colours to the theme tokens (docs/theme.md); app.css split
like app.js.
6. **Copies between repositories**: one sync script per direction with a check in CI that
the copy matches its source revision (today: sync_native.sh, sync_podcast_ui.py).

32
docs/search.md
View file

@ -1,32 +0,0 @@
# Suche
One field for all of diora: a section of its own in the navigation (`static/js/nav.js`,
section and group `suche`, flag `side`):
- phone (the apps' tab bar, the web page's narrow bar): beside Home · Lesen · Hören;
- the apps' sidebar and the web page's wide bar: at the foot, above Abos & Quellen.
It's never the section the app opens in next time (`transient`: `diora_nav_section` isn't written for it).
## What is searched where
| Area | Where | Opens |
|---|---|---|
| Bücher, Zeitschriften | on the device (`_lastBookListData` / `bookMetaCache`): titles are encrypted, the server never sees them; title, author, series, file name, folders; editions of one work once | `openBook(id)` |
| Hörbücher, Podcasts (Abos) | server: title, author | `__dioraAudiobookOpen(id)` / `__dioraPodcastOpenFeed(id)` |
| Folgen | server: the full-text index (`podcasts/discover.py`, titles and shownotes, podcasts only) | `__dioraPodcastOpenEpisode(row)` if the podcast UI has it, else the podcast's page; ▶ plays (`podcastPlayEpisode`) |
| Artikel | server: titles and authors of news and blog items not expired | `__dioraFeedsOpenItem(id)` |
| Nachrichtenquellen, Blogs | server: title, address, category | the news / blogs |
| Radio | server: saved stations, name, tags, country | `playStation()` |
Every word has to be there as the start of a word ("plattf" finds "Plattform"); case and
diacritics don't count. "Darüber hinaus" hands the query to Abos & Quellen (new podcasts,
YouTube, feeds) or the radio directory search. The last eight queries (Enter or a hit opened)
are kept on the device. `DioraSearch.open(q)` opens it from anywhere.
## API
`GET /search/?q=…` (signed in) →
`{q, podcasts, podcasts_total, audiobooks, audiobooks_total, episodes, sources, articles,
articles_total, stations}`; `{q, empty: true}` for less than two letters. Episodes come as the
episode lists give them (`_episode_list_data`). Eight per area (articles sixteen).

47
docs/sources.md
View file

@ -1,47 +0,0 @@
# Abos & Quellen (navigation phase 2)
One place to manage every subscription: podcasts, YouTube channels, audiobooks,
news sources and blogs. The everyday views (Strom, Merkliste, Wiedergabeliste — docs/podcasts.md —,
Nachrichten, Blogs, Hörbücher) show content; adding, changing and removing sources happens
here (their own small entry points lead here).
## Where
`DioraNav` section `sources` ("Abos & Quellen", icon ⚙), panel `#tab-sources`,
`group: 'more'`, `footer: true`:
- not in `groups()` (no tab of its own on a phone);
- in `sections()` with `footer: true`: the sidebar puts it at the bottom, above
"Menü"; the phone's app menu has an entry; diora Reader's bar leaves it out;
- the web page's bar: at the end (wide), as ⚙ (narrow).
## The page (`static/js/sources.js`, diora-web, every platform)
- **Hinzufügen**: one field for an address or search words.
- a YouTube address → its channel (`/podcasts/search/?q=`), as Podcast or Hörbuch;
- another address → `/feeds/discover/`: every feed found, with the kind it
looks like (`suggested_kind`: `podcast` when most entries carry sound or
video, else `news`/`blog`) and whether it's subscribed already
(`subscribed` as a feed, `podcast_subscribed`); the kind can be changed:
Podcast · Hörbuch · Nachrichten · Blog;
- words → the podcast directory (`/podcasts/search/`).
Podcast/Hörbuch subscribe with `POST /podcasts/feeds/add/` (then
`settings {kind: 'audiobook'}`), Nachrichten/Blog with `POST /feeds/sources/add/`.
- **The list**: all sources, filter chips Alle · Podcasts · YouTube · Hörbücher ·
Nachrichten · Blogs · Probleme (fetching fails), and a search over the names.
- **A source's actions** (tapping it): open its content, its settings (the
existing editors), change its kind (Podcast ↔ Hörbuch, Nachrichten ↔ Blog),
unsubscribe.
- **Import & Export**: OPML for podcasts and feeds.
## Hooks it uses (each optional: without it, it goes to the section instead)
| Hook | From | Does |
|------|------|------|
| `__dioraPodcastFeedSettings(feedId)` | native-podcast.js | opens the podcast's settings sheet |
| `__dioraPodcastOpenFeed(feedId)` | native-podcast.js | shows the podcast's episode list (Podcasts section) |
| `__dioraFeedsEdit(sourceId)` | feeds.js | opens the source's editor (Nachrichten/Blogs, manage view) |
| `__dioraFeedsStale()` | feeds.js | its state is old: load again when shown |
| `__dioraAudiobookOpen(feedId)` | audiobooks.js | opens the book |
The host calls it uses: `podcastState`, `podcastFeedSettings`, `podcastRefresh`
(after subscribing), `podcastFeedRemoved` (after unsubscribing).

53
docs/theme.md
View file

@ -1,53 +0,0 @@
# Light and dark
diora has two colour schemes, **dark** (the old look) and **light**, chosen by the reader:
"Dunkel", "Hell" or "Automatisch" (as the operating system says). This file is the contract
between diora-web (app.css, home.js, feeds.js, templates, and static/podcasts/podcast-ui.css —
a copy of diora-android's native.css by tools/sync_podcast_ui.py), diora-android (native.css,
native-*.js; the apps ship diora-web's files too, through build_web.py) and diora-desktop
(desktop.css).
## Switching
* `<html data-theme="dark|light">` is the scheme in effect. Set as early as possible
(an inline script in base.html's `<head>`, before any CSS paints) so nothing flashes.
* The choice is `localStorage.diora_theme` = `"dark"` (default) | `"light"` | `"auto"`.
* `window.__dioraTheme()` → the choice; `window.__dioraSetTheme(choice)` sets and applies it;
`window.__dioraThemeChanged` (event `diora-theme` on `window`, `detail` = the scheme in
effect) lets native code follow (Android status/navigation bars, Electron's window colour).
* "auto" follows `prefers-color-scheme` live — except in the Android app, whose WebView
answers that query from the app's own (dark) theme, not the system's: there the page asks
`DioraNative.systemScheme()` ("dark" | "light") and the app calls
`window.__dioraSystemScheme(scheme)` when the system switches (uiMode change, or on return).
* The desktop app (Electron) sets `nativeTheme.themeSource` to the scheme for a fixed choice
(title bar, scrollbars, form controls match) but leaves it on `"system"` for "auto", so
`prefers-color-scheme` there is the system's; the page tells it the choice along with the
scheme (`dioraDesktop.setTheme(scheme, choice)`, desktop 0.1.31).
* E-ink mode (`html.nui-eink`) keeps its own black-on-white rules and ignores this.
* The reader has its own themes; its default is now "Wie die App" (`readerSettings.theme = 'app'`):
dark → reader dark, light → reader bright. Sepia, Dunkel and Hell stay choosable.
* A wallpaper (the radio page's background image) keeps its own contrast handling
(`body.bright-bg`, the ◑ button); without a wallpaper the scheme decides.
## Colours: only through these variables
| variable | dark | light | for |
|---|---|---|---|
| `--bg` | `#000` | `#f6f5f1` | the page |
| `--fg` | `#fff` | `#17171a` | text |
| `--muted` | `#9a9a9a` | `#66666b` | secondary text, icons |
| `--card` | `#0b0b0b` | `#ffffff` | cards, rows that stand out |
| `--panel` | `#111` | `#ffffff` | sheets, menus, docks, bars |
| `--hover` | `#141414` | `#ecebe6` | hover / pressed |
| `--raised` | `#1a1a1a` | `#e4e2dc` | inputs, tracks of sliders, chips |
| `--border` | `#2a2a2a` | `#d9d6ce` | lines |
| `--border-strong` | `#444` | `#b4b0a6` | lines that must show (inputs, focus off) |
| `--accent` | `#e63946` | `#d0263a` | the red: brand, active, progress |
| `--on-accent` | `#fff` | `#fff` | text on the red |
| `--scrim` | `rgba(0,0,0,.6)` | `rgba(20,20,20,.35)` | behind sheets |
| `--shadow` | `none` | `0 1px 3px rgba(0,0,0,.08)` | cards in light (no lines needed in dark) |
Old names stay as aliases: `--nui-muted: var(--muted)`, `--nui-panel: var(--panel)`,
`--text: var(--fg)`, `--text-muted: var(--muted)`, `--bg-card: var(--card)`.
Images, album art, covers and Ambient's own picture keep their colours. Status colours
(`--green`, `--yellow`, `--red`) may stay as they are.

262
docs/transcription.md
View file

@ -1,262 +0,0 @@
# Smarte Transkription: Transkript und Artikel zu einer Folge
Decided with the user on 2026-11-09. Plan and order of work:
diora-android `plans/smarte-transkription.md`.
## What it is for
Sometimes the shownotes don't tell whether an episode still has something to teach. Then the
user presses a button and, a few hours later at most, gets a readable article written from the
speakers' own standpoint. It is meant to let them judge the episode for themselves, so it keeps
arguments, reasons and evidence and does not just summarise them. The article sits on the
episode and in Blogs.
Nothing happens on its own. The server never transcribes ahead, by cron or on subscribing; the
only way to start a transcription is the `POST` below, sent when the user presses a button.
## Who does what
The **server** transcribes. It already downloads and decodes episodes for skipping silence
(`podcasts/silence.py`), and browsers and the WebView can't fetch podcast audio (CORS). It cuts
the audio at pauses and sends the pieces in parallel to a Whisper instance with an
OpenAI-compatible API (`POST /v1/audio/transcriptions`): the built-in one by default (`DIORA_WHISPER_URL`),
or any instance the user enters. YouTube episodes are
not transcribed: the server never fetches their sound (docs/youtube.md).
The **device** writes the article with the user's own key (docs/ai.md). The key never reaches the
server. It is one LLM call of a few minutes. If diora is open when the transcript is done, it
happens right away; otherwise the next time diora is opened on any of the user's devices. The
server only remembers that an article was wanted.
Transcript and article are stored in plain text on the server, like all podcast and feed data.
## Flow
1. In the episode the user presses "Transkript erstellen" (the Transkript tab) or
"✦ Artikel erstellen" / "✦ Transkribieren und Artikel erstellen". The second sends
`article: true`. With a transcript already there (from the feed or an earlier run) only the
article is written.
2. The server works through the episode (minutes to hours, no GPU on Strato). The UI shows
"Transkribiere 7/38 …" while it is open.
3. Done: a notice for `GET /feeds/notifications/`. Android's BackgroundSync and the desktop show
"Transkript fertig: <Folge>", with "Artikel wird beim Öffnen geschrieben" when one is wanted.
Tapping it opens diora at the episode.
4. A device with a key (`static/js/podcast-article.js`) picks up the open wishes, reserves one,
writes the article and saves it. It then says "Artikel fertig" (on Android as a system
notification through the bridge, `DioraNative.notify(title, text, target)` with target
`episode:<id>` or `item:<id>`; elsewhere a toast) and opens it on a tap.
5. The article is the first tab of the episode ("Artikel") and an item of the favourite blog
"Podcast-Artikel", linking back to the episode.
## API
**Settings.** `GET /podcasts/transcription/settings/` returns
`{engine: "builtin"|"custom", url, model, language, parallel, has_token, builtin_available}`.
`POST` takes the same fields and `token` (write-only; `""` removes it). The token is kept
encrypted with Fernet like `feeds.SiteLogin.secret`. `language` is `""` for automatic or an ISO
code (`de`). `parallel` is 1 to 8, default 3. `builtin_available` is whether `DIORA_WHISPER_URL`
is set. `url` is the instance's base, e.g. `https://whisper.example.org/v1`; the server appends
`/audio/transcriptions`.
An empty `model` is sent as `whisper-1`. The built-in URL is read into `settings.WHISPER_URL` from
`DIORA_WHISPER_URL`. A custom engine needs a `url`; the token is only sent to custom instances.
`POST /podcasts/transcription/test/` sends one second of silence to the configured instance and
returns `{ok, model, error}`. A custom URL goes through the same SSRF check as chapters
(`podcasts/chapters.py` `_open`): no private, loopback or link-local addresses. Only the built-in
URL from `DIORA_WHISPER_URL` is exempt.
**Job.** `POST /podcasts/episodes/<id>/transcribe/` with `{article: true|false}` starts a job, or
updates `want_article` on a running one. With a transcript already there it returns
`{state: "done"}` straight away (and keeps `want_article`). `GET` on the same address returns
`{state, chunks_done, chunks_total, error, want_article, started_at, finished_at}`, with `state`
one of `none`, `queued`, `downloading`, `transcribing`, `done`, `error`. `DELETE` cancels a
queued or running job (its row is deleted, so `GET` then says `none`; a finished one stays). The server runs one job at a time; the others wait as `queued`. A YouTube
episode returns 400 `{error: "youtube"}`, an episode over 800 MB 400 `{error: "too large"}`. The size is asked with a `HEAD`
request (`Content-Length`; if that fails the job starts anyway) and checked again while
downloading, where it ends the job with the error `too large`. `article: true` is only ever
raised, never lowered, by a repeated `POST`; a `POST` after an `error` starts a new job (at the
back of the queue). Jobs run in id order, which is why the notice below can use ids as its cursor.
A feed transcript already there gives a job that is `done` with no `data`: it only holds the wish
for an article and causes no `transcripts` notice.
**Transcript.** The result is kept on the job (`TranscriptionJob.data`) in the format of
`EpisodeTranscript.data`: `{segments: [{start, end, speaker, text}], source: {type: "whisper",
model, language}}`. `speaker` is `""`. It is kept apart from `EpisodeTranscript`, which is a cache
of the feed's transcripts and is rewritten when the feed changes them.
`GET /podcasts/episodes/<id>/transcript/` returns the feed's transcript when the feed has a
usable one, else the Whisper one, else 404 as before. `episode_detail` adds
`transcript_origin` (`"feed"`, `"whisper"` or `null`), `has_transcript` is true for both, and it
adds `article_item_id` (or `null`) and `transcription` (the job's state as above, or `null`).
**Article.** `POST /podcasts/episodes/<id>/article/` takes `{title, summary, body_html, model,
kind}` and returns `{item_id}`. The server cleans `body_html` with `feeds/fetch.py clean_html`.
It creates the user's source "Podcast-Artikel" when missing (`kind` blog, `fav` true,
`synthetic` true, `url` `diora://podcast-artikel`) and adds a `feeds.Item` there, or replaces the
episode's earlier one. Item fields: the episode's link (else its audio URL) as `url`, the feed's
author as `author`, the episode's artwork as `image_url`, now as `published` (so it is on top of
the blog stream under "heute"), `words` counted. Saving clears the wish. `DELETE` on the same
address removes the article and its item. `GET /feeds/items/<id>/` and `_item_data` carry
`episode_id` for such items.
`body_html` may contain `href="#t=<seconds>"` links; `clean_html` keeps exactly this fragment form
(`podcasts/shownotes.py` `TIME_LINK`) and no other `#` links. Validation: `title` 1 to 300
characters, `body_html` non-empty after cleaning and at most 1,000,000 characters, `summary` at most
2000, `model` at most 100, `kind` at most 20 (400 otherwise). The item has `content_full` true (no
page is ever fetched) and, as the reader's furniture filter would strip repeated "▶ ab" blocks,
`GET /feeds/items/<id>/` skips that filter for synthetic sources. The endpoint is
`podcasts/article.py`.
Synthetic sources are skipped by the feed refresh (`refresh_newsfeeds`, `refresh_now`) and by the
pruning of old blog items (`feeds/refresh.py`). They can be hidden or have their bell turned off
like any blog; they can't be refreshed, renamed to another URL or turned into news. `refresh.refresh()` itself drops them, the OPML export leaves them out, and `_source_data` carries `synthetic`.
**Open wishes.** `GET /podcasts/transcription/pending/` returns
`[{episode_id, title, podcast, duration_seconds}]`: the user's episodes whose transcript is done,
that want an article, have none and aren't reserved. `POST
/podcasts/episodes/<id>/article/claim/` reserves one for 15 minutes for the calling device
(200), or answers 409 when another device holds it, so phone and desktop don't both write it.
`DELETE` on the claim releases it (the device gave up, e.g. no key or an error the user saw).
The device names itself with `device` in the JSON body (or, if that is missing, the
`X-Diora-Device` header): a short stable name such as `android`, `desktop` or `browser`, at most 60
characters; a claim without one is 400. The same device claiming again renews its 15 minutes. A
claim on an episode with no open wish (no job done, no article wanted, or an article already
there) is 404. `DELETE` by a device that does not hold the claim is 409.
**Notice.** `GET /feeds/notifications/` additionally takes `t_after` and returns
`transcripts: [{job_id, episode_id, title, podcast, want_article}]` (jobs done after that id) and
`t_last`. Without `t_after` (a device's first time) the list is empty and only `t_last` comes,
like `after`/`last`. Devices that don't know the fields ignore them.
## Transcribing (`podcasts/transcribe.py`)
Download with the same limits as `silence.py`, decode with imageio-ffmpeg to 16 kHz mono. Cut
into pieces of about eight minutes; each cut moves to the longest pause within ±30 seconds
(`silence.py`'s detection), so no word is cut. Pieces go out as FLAC to the built-in instance
and as Opus 32 kbit/s to custom ones, to keep uploads small.
Each piece is sent with `model`, `language` (when set), `response_format=verbose_json` and a
`prompt` of podcast title, episode title and author names, so Whisper spells names right. Pieces
run in a thread pool of `parallel` workers; each is retried up to three times with a growing
pause; progress is written after each. Segment times are shifted by the piece's start; the
segments are joined and smoothed with `transcripts.merge`. A job runs in a daemon thread, like
`silence.ensure()`; one per server. A job found `downloading` or `transcribing` after a restart
is started again. This happens when the first
request (`GET`/`POST` on `/transcribe/` or `/transcription/pending/`) after a start reaches the server.
Cancelling deletes the job's row; the worker notices at the latest after five seconds, stops
sending pieces and removes its temporary files (they live in a temporary directory that is always
removed). Only the worker's main thread writes to the database. The settings used are those of the
episode's owner, read when the job starts. A refusal by the instance with a 4xx status other than
408 and 429 is not retried and ends the job with `Whisper: HTTP <code>`.
## The article (`static/js/podcast-article.js`)
Input: the transcript as plain text, one line per segment group of about 30 seconds,
`[mm:ss] Sprecher: text` (or `[mm:ss] text` without speakers), plus the episode's metadata:
podcast title, author, episode title, date, duration, the shownotes as plain text and the
chapters. Over about 120,000 tokens (more than about nine hours) the device stops with a clear
message for now. A five-hour episode is about 70,000 tokens.
`DioraAI.ask({system, prompt, schema, effort: "medium", maxTokens: 32000, onProgress, signal})`.
`onProgress(words)` reports the words written so far while streaming ("Schreibe Artikel … 2.300
Wörter"); `signal` aborts. Both are optional and new in ai.js; the existing callers don't change.
Schema:
```json
{
"type": "object",
"additionalProperties": false,
"required": ["title", "summary", "kind", "speakers", "sections"],
"properties": {
"title": {"type": "string"},
"summary": {"type": "string"},
"kind": {"type": "string", "enum": ["Interview", "Gespräch", "Monolog", "Diskussion"]},
"speakers": {"type": "array", "items": {"type": "string"}},
"sections": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["heading", "start", "body_markdown"],
"properties": {
"heading": {"type": "string"},
"start": {"type": "integer"},
"body_markdown": {"type": "string"}
}
}
}
}
}
```
The HTML: the summary as a lead paragraph, then per section an `<h2>`, below it a link
"▶ ab 12:34" (`<a href="#t=754">`; `clean_html` keeps fragment links, and feeds.js plays the
item's `episode_id` from there when such a link is tapped) and the body. Markdown is limited to paragraphs, emphasis, lists and block
quotes, converted on the device; the server cleans the result anyway.
System prompt (the user's words decide the perspective; keep its substance when changing it):
```text
You turn the transcript of a podcast episode into an article that reads as if the people
speaking had written it themselves instead of recording the episode.
The reader wants to decide whether this episode still has something to teach them. They do not
want a summary; they want to understand what is argued here well enough to judge it for
themselves. Write for that reader.
Perspective. Write from the speakers' standpoint and state their positions directly, as their
own claims. Never distance yourself from them: no "im Podcast wird die These aufgestellt", "der
Sprecher meint", "in dieser Folge geht es um", "die Gäste diskutieren", "laut X". If the episode
is one person speaking, write as that person argues, in the third person or impersonal voice,
but without reporting verbs. Do not add your own opinion, criticism or fact checks, and do not
soften what is said.
Conversations. Interviews, conversations and discussions are condensed into their core theses;
the form of question and answer disappears. Where people hold different positions, keep them
apart and name who holds which ("Zuckermann hält dagegen, dass …"), but do not turn it into
minutes. Questions from a host matter only where they shape the argument. When the transcript
has no speaker names, infer them from the context (introductions, the shownotes, how people
address each other); if you can't, do not invent names.
Substance over summary. Give the arguments with their reasons, examples and evidence, not just
the conclusions. Keep figures, dates, quotations that carry an argument (short, in quotation
marks, attributed), and every work mentioned (books, articles, studies, films) with author and
title. Make clear what is new, distinctive or surprising in the episode and where it restates
what is commonly said, by how you weight and order it, without grading it.
Leave out: advertising and sponsors, appeals for donations, subscriptions or ratings, notes on
the podcast itself (schedules, technology, community), greetings, small talk and digressions that
lead nowhere.
Form. Write in the language of the episode. `title`: a title for the article, not the episode
title repeated. `summary`: three to six sentences up front that state the central thesis and how
the episode gets there. `sections`: as many as the content needs (usually four to ten), each
with a heading that says something (not "Einleitung"), `start` = the second in the episode where
its content begins (from the [mm:ss] marks), and `body_markdown` in plain paragraphs; lists only
where the speakers list things themselves. Follow the order of the episode unless a different
order makes the argument clearer. Length: as long as needed for the reader to judge the episode,
roughly a tenth of the transcript, at most about 5,000 words.
The transcript was made by speech recognition: fix obvious mishearings, especially of names,
using the metadata, and don't quote garbled passages.
```
Prompt (user turn): the metadata block, then `Transkript:` and the text.
## The built-in instance (since 2026-11-09)
Whisper does not fit on the Strato server: with about 2 GB of memory free, large-v3-turbo needed
more than 2.2 GB and even `small` broke a 1.2 GB limit, and earlyoom then killed other services.
So the built-in instance runs on the user's PC (multi11, GTX 1660 Ti) and diora-prod reaches it over
the tailnet: `DIORA_WHISPER_URL=http://100.64.0.3:8766/v1` in `~/docker-compose/diora-prod/.env`.
It is `deploy/whisper/server.py` (a small OpenAI-compatible server around faster-whisper,
large-v3-turbo, int8_float16 on CUDA; one request at a time; the model is loaded on demand and freed
after 10 idle minutes), started at logon from `F:\diora-whisper` (start.cmd via start-hidden.vbs,
shortcut in the user's Startup folder; log `F:\diora-whisper\server.log`), listening only on the
tailnet address. One hour of speech takes about five minutes.
When the PC is off, a job does not fail: the server checks the instance before downloading, and a
connection error (there or while pieces are out) puts the job back into the queue with
`error = "Whisper-Instanz nicht erreichbar, neuer Versuch um HH:MM"`, tried again every ten
minutes (`WAIT_RETRY`) for up to two days after the button was pressed (`WAIT_MAX`). Only the
built-in instance waits like this; a custom instance's connection errors fail the job as before.

76
docs/youtube.md
View file

@ -1,76 +0,0 @@
# YouTube channels as podcasts
A YouTube channel (or playlist) can be subscribed to like a podcast. Only its
sound is played, and **the server never downloads or streams a video or its
sound**: each app finds the audio stream itself.
## Server (diora-web, `podcasts/youtube.py`)
- `POST /podcasts/feeds/add/` takes a YouTube address as `rss_url`: a channel
(`/@name`, `/channel/UC…`, `/c/…`, `/user/…`), a playlist (`?list=…`), a video
(subscribes its channel) or the feed itself. The server reads the channel page
once (feed address, name, picture) and stores the feed
`https://www.youtube.com/feeds/videos.xml?channel_id=UC…` (or `?playlist_id=…`).
`GET /podcasts/search/?q=<YouTube address>` answers with that one channel.
- From then on the feed is refreshed like any other — except that YouTube answers its
channel feeds with a 404 (or 5xx) now and then for no reason (October 2026: two
requests in three), so a YouTube feed is asked up to 12 times, 0.25 s apart, before
its refresh counts as failed (`YOUTUBE_FEED_TRIES` in `podcasts/views.py`). It carries the latest 15
videos. **Shorts are left out.** There is no duration (`duration_seconds` 0)
until a player knows it: `POST /podcasts/progress/save/` takes an optional
`duration_seconds` (number, 1 s … 7 days), stored only while the episode has none.
- An episode from such a feed has
- `audio_url` = `https://www.youtube.com/watch?v=<11-character id>`. This is
**the** marker: a client recognises a YouTube episode by this prefix;
- `guid` = `yt:video:<id>`, `artwork_url` = the video's thumbnail,
`description` = the video description as HTML (links, line breaks; timestamps
like `00:00 Intro` work as usual).
- Nothing on the server touches the sound of such an episode: no ID3 chapters,
no transcripts, and `/podcasts/episodes/<id>/silences/` answers
`{"ready": true, "spans": [], "error": "youtube"}` (skip silence is up to the
app, as on Android anyway).
- `POST /podcasts/feeds/<id>/move/ {"rss_url": …}` gives an existing subscription a
new address and keeps everything (settings, episodes, places, played and favourite
marks, queue). Moved to a YouTube address, every episode whose file is named after
its video (`…/<11-character id>.mp3`, as podsync-like servers name them) becomes
that video (`guid` `yt:video:<id>`, `audio_url` its `watch?v=` page), so the
channel's feed finds it again instead of adding it twice. Used to move channels
that came through such a server (it downloaded the sound on the server) to the
channel itself.
## Ad and outro: SponsorBlock
What goes out of the sound — the advertisement and the closing words — is other people's
work: SponsorBlock (`sponsor.ajay.app`) lets listeners mark a video's parts by category.
`podcasts/sponsor.py` asks it once per episode (only `sponsor` and `outro`, only parts a
listener skips, never a part marked `full`) and keeps the answer on the episode
(`PodcastEpisode.sponsor`); `GET /podcasts/episodes/<id>/sponsor/` answers with it,
`{"ready": true, "v": 1, "spans": [[from, to], …]}`, empty for an episode that isn't a
video and with an `error` beside empty spans when SponsorBlock couldn't be asked. Nothing
else is fetched — no intro, self-promotion, interaction reminder or non-music section.
An answer counts for a week, a failure for an hour (segments are added later; a failure was
probably the network). A player jumps over those parts and never runs them faster: the
setting is `skip_sponsor` of this device (on by default, in the play and speed menus), and
it does something only for YouTube episodes. Nothing of the sound leaves the device for
this — the video id does, to sponsor.ajay.app, when the server asks (never the sound, and
never for an episode that isn't a video).
## Apps
A client that sees a `watch?v=` address resolves it **itself, on the device,
to an audio-only stream** (no video track), and plays or downloads that.
| Client | How |
|---------|-----|
| Android | NewPipeExtractor: the original audio track, progressive, opus before m4a, ≤ 160 kbit/s; resolved when the stream is opened and once more on a 403; downloads in 10 MB ranges (in one go YouTube throttles to ~50 KB/s) |
| Desktop | yt-dlp in the main process (`-f bestaudio`), Electron itself as its JavaScript runtime; yt-dlp is fetched from its GitHub release on first use and kept up to date (`src/main/youtube.js`) |
| Browser | can't (YouTube doesn't let a web page read its pages, and a stream address only works for the device that asked for it): the episode is listed but says it plays in the app |
Stream addresses are bound to the device's IP address and run out after a few
hours: resolve shortly before playing, and again when the player reports an
error (podcasts-web.js reloads after a media error anyway).
Progress, queue, played state etc. work as for any episode: they belong to the
episode on the server, not to the stream address.

7
e2e/env.js
View file

@ -1,9 +1,6 @@
// Shared constants between playwright.config.js and e2e/global-setup.js.
// E2E_PORT / E2E_PYTHON: a port that no dev server holds (the suite must never
// run against a real database), and the Python with the requirements installed.
const PORT = Number(process.env.E2E_PORT || 8000);
const PORT = 8000;
const BASE_URL = `http://127.0.0.1:${PORT}`;
const PYTHON = process.env.E2E_PYTHON || 'python3';
const SERVER_ENV = {
...process.env,
@ -18,4 +15,4 @@ const TEST_USER = {
password: 'e2e-test-pw-12345',
};
module.exports = { PORT, BASE_URL, PYTHON, SERVER_ENV, TEST_USER };
module.exports = { PORT, BASE_URL, SERVER_ENV, TEST_USER };

129
e2e/fixtures/make_test_pdf.py
View file

@ -1,129 +0,0 @@
"""A small PDF with the cases the PDF reader has to get right (e2e and by hand).
python e2e/fixtures/make_test_pdf.py e2e/fixtures/test.pdf
12 pages: A4 portrait text pages, one A5 page and one A4 landscape page (mixed
sizes), page 7 nearly empty (only a page number: the smart crop must not zoom
it to infinity), an outline (table of contents) with three entries, internal
links from page 1 to pages 5 and 9 and one external link, a coloured picture on
page 2 (it must stay as it is in the reader's colours) and page labels like a
book's: i, ii for the first two pages, then 1, 2, 3 … Plain PDF 1.4, Helvetica,
no dependencies.
"""
import sys
A4 = (595, 842)
A5 = (420, 595)
A4L = (842, 595)
WORDS = ('Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. '
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor. ').split()
def esc(t):
return t.replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)')
def text_lines(n, words_per_line=11):
out, i = [], 0
for _ in range(n):
out.append(' '.join(WORDS[(i + k) % len(WORDS)] for k in range(words_per_line)))
i += words_per_line
return out
def page_content(no, size):
w, h = size
ops = ['BT', '/F1 22 Tf', f'50 {h - 70} Td', f'(Kapitel {no} - Seite {no}) Tj', 'ET']
if no == 7:
# Nearly empty: just the number at the bottom.
return '\n'.join(['BT', '/F1 10 Tf', f'{w / 2 - 10:.0f} 30 Td', f'({no}) Tj', 'ET'])
y = h - 110
lines = text_lines(28 if h > 700 else 18, 11 if w > 500 else 8)
ops += ['BT', '/F1 11 Tf', f'50 {y} Td', '14 TL']
for ln in lines:
ops.append(f'({esc(ln)}) Tj T*')
ops.append('ET')
if no == 2:
ops += ['q', f'200 0 0 120 {w - 260} 60 cm', '/Im1 Do', 'Q']
if no == 1:
ops += ['BT', '/F1 12 Tf', f'50 {y - 30 - 14 * len(lines)} Td',
'(Link zu Seite 5) Tj', '0 -20 Td', '(Link zu Seite 9) Tj', '0 -20 Td', '(Externer Link: example.org) Tj', 'ET']
ops += ['BT', '/F1 10 Tf', f'{w / 2 - 10:.0f} 30 Td', f'({no}) Tj', 'ET']
return '\n'.join(ops)
def build(path):
sizes = [A4] * 12
sizes[3] = A5 # page 4
sizes[8] = A4L # page 9
objs = [] # index = object number - 1
def add(body):
objs.append(body)
return len(objs)
catalog = add(None)
pages_obj = add(None)
font = add('<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>')
# A picture of four colours (2 x 2 pixels, drawn large).
pixels = b'FF0000 00A000 0000FF FFD700>'
image = add(b'<< /Type /XObject /Subtype /Image /Width 2 /Height 2 /ColorSpace /DeviceRGB '
b'/BitsPerComponent 8 /Filter /ASCIIHexDecode /Length %d >>\nstream\n' % len(pixels)
+ pixels + b'\nendstream')
page_ids = []
for i, size in enumerate(sizes):
content = page_content(i + 1, size).encode('cp1252')
cid = add(b'<< /Length %d >>\nstream\n' % len(content) + content + b'\nendstream')
pid = add(None)
page_ids.append((pid, cid, size))
# links on page 1 (placed where the text is: bottom part of the text block)
p1_h = sizes[0][1]
base_y = p1_h - 110 - 30 - 14 * 28
def link_to(page_no, y):
return add(f'<< /Type /Annot /Subtype /Link /Rect [48 {y - 4} 220 {y + 14}] /Border [0 0 0] '
f'/Dest [{page_ids[page_no - 1][0]} 0 R /XYZ 0 {sizes[page_no - 1][1]} null] >>')
annots = [link_to(5, base_y), link_to(9, base_y - 20),
add(f'<< /Type /Annot /Subtype /Link /Rect [48 {base_y - 44} 300 {base_y - 26}] /Border [0 0 0] '
f'/A << /S /URI /URI (https://example.org/) >> >>')]
for n, (pid, cid, (w, h)) in enumerate(page_ids):
extra = f' /Annots [{" ".join(f"{a} 0 R" for a in annots)}]' if n == 0 else ''
xobj = f' /XObject << /Im1 {image} 0 R >>' if n == 1 else ''
objs[pid - 1] = (f'<< /Type /Page /Parent {pages_obj} 0 R /MediaBox [0 0 {w} {h}] '
f'/Resources << /Font << /F1 {font} 0 R >>{xobj} >> /Contents {cid} 0 R{extra} >>')
objs[pages_obj - 1] = f'<< /Type /Pages /Kids [{" ".join(f"{p} 0 R" for p, _, _ in page_ids)}] /Count {len(page_ids)} >>'
# outline: three entries
outline = add(None)
entries = [('Anfang', 1), ('Querformat', 9), ('Fast leer', 7)]
ids = [add(None) for _ in entries]
for k, ((title, page), oid) in enumerate(zip(entries, ids)):
nav = ''
if k > 0:
nav += f' /Prev {ids[k - 1]} 0 R'
if k < len(ids) - 1:
nav += f' /Next {ids[k + 1]} 0 R'
objs[oid - 1] = (f'<< /Title ({esc(title)}) /Parent {outline} 0 R{nav} '
f'/Dest [{page_ids[page - 1][0]} 0 R /XYZ 0 {sizes[page - 1][1]} null] >>')
objs[outline - 1] = f'<< /Type /Outlines /First {ids[0]} 0 R /Last {ids[-1]} 0 R /Count {len(ids)} >>'
objs[catalog - 1] = (f'<< /Type /Catalog /Pages {pages_obj} 0 R /Outlines {outline} 0 R /PageMode /UseOutlines '
f'/PageLabels << /Nums [0 << /S /r >> 2 << /S /D >>] >> >>')
info = add('<< /Title (Testdokument PDF) /Author (diora Test) >>')
out = bytearray(b'%PDF-1.4\n%\xe2\xe3\xcf\xd3\n')
offsets = []
for i, body in enumerate(objs):
offsets.append(len(out))
data = body if isinstance(body, bytes) else body.encode('cp1252')
out += b'%d 0 obj\n' % (i + 1) + data + b'\nendobj\n'
xref = len(out)
out += b'xref\n0 %d\n0000000000 65535 f \n' % (len(objs) + 1)
for off in offsets:
out += b'%010d 00000 n \n' % off
out += (b'trailer\n<< /Size %d /Root %d 0 R /Info %d 0 R >>\nstartxref\n%d\n%%%%EOF\n'
% (len(objs) + 1, catalog, info, xref))
open(path, 'wb').write(out)
if __name__ == '__main__':
build(sys.argv[1] if len(sys.argv) > 1 else 'test.pdf')

652
e2e/fixtures/test.pdf
View file

@ -1,652 +0,0 @@
%PDF-1.4
%âãÏÓ
1 0 obj
<< /Type /Catalog /Pages 2 0 R /Outlines 32 0 R /PageMode /UseOutlines /PageLabels << /Nums [0 << /S /r >> 2 << /S /D >>] >> >>
endobj
2 0 obj
<< /Type /Pages /Kids [6 0 R 8 0 R 10 0 R 12 0 R 14 0 R 16 0 R 18 0 R 20 0 R 22 0 R 24 0 R 26 0 R 28 0 R] /Count 12 >>
endobj
3 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
endobj
4 0 obj
<< /Type /XObject /Subtype /Image /Width 2 /Height 2 /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /ASCIIHexDecode /Length 28 >>
stream
FF0000 00A000 0000FF FFD700>
endstream
endobj
5 0 obj
<< /Length 2463 >>
stream
BT
/F1 22 Tf
50 772 Td
(Kapitel 1 - Seite 1) Tj
ET
BT
/F1 11 Tf
50 732 Td
14 TL
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
(etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed) Tj T*
(do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und) Tj T*
(Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit,) Tj T*
(sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche) Tj T*
(und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing) Tj T*
(elit, sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl,) Tj T*
(Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur) Tj T*
(adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text, damit) Tj T*
(Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet,) Tj T*
(consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text,) Tj T*
(damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit) Tj T*
(amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt) Tj T*
(Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor) Tj T*
(sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite) Tj T*
(trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum) Tj T*
(dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die) Tj T*
(Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem) Tj T*
(ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor.) Tj T*
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
ET
BT
/F1 12 Tf
50 310 Td
(Link zu Seite 5) Tj
0 -20 Td
(Link zu Seite 9) Tj
0 -20 Td
(Externer Link: example.org) Tj
ET
BT
/F1 10 Tf
288 30 Td
(1) Tj
ET
endstream
endobj
6 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 3 0 R >> >> /Contents 5 0 R /Annots [29 0 R 30 0 R 31 0 R] >>
endobj
7 0 obj
<< /Length 2379 >>
stream
BT
/F1 22 Tf
50 772 Td
(Kapitel 2 - Seite 2) Tj
ET
BT
/F1 11 Tf
50 732 Td
14 TL
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
(etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed) Tj T*
(do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und) Tj T*
(Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit,) Tj T*
(sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche) Tj T*
(und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing) Tj T*
(elit, sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl,) Tj T*
(Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur) Tj T*
(adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text, damit) Tj T*
(Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet,) Tj T*
(consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text,) Tj T*
(damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit) Tj T*
(amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt) Tj T*
(Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor) Tj T*
(sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite) Tj T*
(trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum) Tj T*
(dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die) Tj T*
(Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem) Tj T*
(ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor.) Tj T*
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
ET
q
200 0 0 120 335 60 cm
/Im1 Do
Q
BT
/F1 10 Tf
288 30 Td
(2) Tj
ET
endstream
endobj
8 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents 7 0 R >>
endobj
9 0 obj
<< /Length 2345 >>
stream
BT
/F1 22 Tf
50 772 Td
(Kapitel 3 - Seite 3) Tj
ET
BT
/F1 11 Tf
50 732 Td
14 TL
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
(etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed) Tj T*
(do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und) Tj T*
(Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit,) Tj T*
(sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche) Tj T*
(und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing) Tj T*
(elit, sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl,) Tj T*
(Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur) Tj T*
(adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text, damit) Tj T*
(Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet,) Tj T*
(consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text,) Tj T*
(damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit) Tj T*
(amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt) Tj T*
(Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor) Tj T*
(sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite) Tj T*
(trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum) Tj T*
(dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die) Tj T*
(Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem) Tj T*
(ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor.) Tj T*
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
ET
BT
/F1 10 Tf
288 30 Td
(3) Tj
ET
endstream
endobj
10 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 3 0 R >> >> /Contents 9 0 R >>
endobj
11 0 obj
<< /Length 1196 >>
stream
BT
/F1 22 Tf
50 525 Td
(Kapitel 4 - Seite 4) Tj
ET
BT
/F1 11 Tf
50 485 Td
14 TL
(Die Seite trägt Text, damit Auswahl, Suche und) Tj T*
(Vorlesen etwas finden. Lorem ipsum dolor sit amet,) Tj T*
(consectetur adipiscing elit, sed do eiusmod tempor. Die) Tj T*
(Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
(etwas finden. Lorem ipsum dolor sit amet, consectetur) Tj T*
(adipiscing elit, sed do eiusmod tempor. Die Seite) Tj T*
(trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing) Tj T*
(elit, sed do eiusmod tempor. Die Seite trägt) Tj T*
(Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit,) Tj T*
(sed do eiusmod tempor. Die Seite trägt Text,) Tj T*
(damit Auswahl, Suche und Vorlesen etwas finden. Lorem) Tj T*
(ipsum dolor sit amet, consectetur adipiscing elit, sed) Tj T*
(do eiusmod tempor. Die Seite trägt Text, damit) Tj T*
(Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum) Tj T*
(dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl,) Tj T*
ET
BT
/F1 10 Tf
200 30 Td
(4) Tj
ET
endstream
endobj
12 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 420 595] /Resources << /Font << /F1 3 0 R >> >> /Contents 11 0 R >>
endobj
13 0 obj
<< /Length 2345 >>
stream
BT
/F1 22 Tf
50 772 Td
(Kapitel 5 - Seite 5) Tj
ET
BT
/F1 11 Tf
50 732 Td
14 TL
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
(etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed) Tj T*
(do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und) Tj T*
(Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit,) Tj T*
(sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche) Tj T*
(und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing) Tj T*
(elit, sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl,) Tj T*
(Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur) Tj T*
(adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text, damit) Tj T*
(Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet,) Tj T*
(consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text,) Tj T*
(damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit) Tj T*
(amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt) Tj T*
(Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor) Tj T*
(sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite) Tj T*
(trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum) Tj T*
(dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die) Tj T*
(Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem) Tj T*
(ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor.) Tj T*
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
ET
BT
/F1 10 Tf
288 30 Td
(5) Tj
ET
endstream
endobj
14 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 3 0 R >> >> /Contents 13 0 R >>
endobj
15 0 obj
<< /Length 2345 >>
stream
BT
/F1 22 Tf
50 772 Td
(Kapitel 6 - Seite 6) Tj
ET
BT
/F1 11 Tf
50 732 Td
14 TL
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
(etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed) Tj T*
(do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und) Tj T*
(Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit,) Tj T*
(sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche) Tj T*
(und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing) Tj T*
(elit, sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl,) Tj T*
(Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur) Tj T*
(adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text, damit) Tj T*
(Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet,) Tj T*
(consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text,) Tj T*
(damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit) Tj T*
(amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt) Tj T*
(Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor) Tj T*
(sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite) Tj T*
(trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum) Tj T*
(dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die) Tj T*
(Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem) Tj T*
(ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor.) Tj T*
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
ET
BT
/F1 10 Tf
288 30 Td
(6) Tj
ET
endstream
endobj
16 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 3 0 R >> >> /Contents 15 0 R >>
endobj
17 0 obj
<< /Length 32 >>
stream
BT
/F1 10 Tf
288 30 Td
(7) Tj
ET
endstream
endobj
18 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 3 0 R >> >> /Contents 17 0 R >>
endobj
19 0 obj
<< /Length 2345 >>
stream
BT
/F1 22 Tf
50 772 Td
(Kapitel 8 - Seite 8) Tj
ET
BT
/F1 11 Tf
50 732 Td
14 TL
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
(etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed) Tj T*
(do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und) Tj T*
(Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit,) Tj T*
(sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche) Tj T*
(und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing) Tj T*
(elit, sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl,) Tj T*
(Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur) Tj T*
(adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text, damit) Tj T*
(Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet,) Tj T*
(consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text,) Tj T*
(damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit) Tj T*
(amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt) Tj T*
(Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor) Tj T*
(sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite) Tj T*
(trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum) Tj T*
(dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die) Tj T*
(Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem) Tj T*
(ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor.) Tj T*
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
ET
BT
/F1 10 Tf
288 30 Td
(8) Tj
ET
endstream
endobj
20 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 3 0 R >> >> /Contents 19 0 R >>
endobj
21 0 obj
<< /Length 1547 >>
stream
BT
/F1 22 Tf
50 525 Td
(Kapitel 9 - Seite 9) Tj
ET
BT
/F1 11 Tf
50 485 Td
14 TL
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
(etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed) Tj T*
(do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und) Tj T*
(Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit,) Tj T*
(sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche) Tj T*
(und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing) Tj T*
(elit, sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl,) Tj T*
(Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur) Tj T*
(adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text, damit) Tj T*
(Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet,) Tj T*
(consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text,) Tj T*
(damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit) Tj T*
(amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt) Tj T*
(Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor) Tj T*
ET
BT
/F1 10 Tf
411 30 Td
(9) Tj
ET
endstream
endobj
22 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 842 595] /Resources << /Font << /F1 3 0 R >> >> /Contents 21 0 R >>
endobj
23 0 obj
<< /Length 2348 >>
stream
BT
/F1 22 Tf
50 772 Td
(Kapitel 10 - Seite 10) Tj
ET
BT
/F1 11 Tf
50 732 Td
14 TL
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
(etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed) Tj T*
(do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und) Tj T*
(Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit,) Tj T*
(sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche) Tj T*
(und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing) Tj T*
(elit, sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl,) Tj T*
(Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur) Tj T*
(adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text, damit) Tj T*
(Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet,) Tj T*
(consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text,) Tj T*
(damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit) Tj T*
(amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt) Tj T*
(Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor) Tj T*
(sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite) Tj T*
(trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum) Tj T*
(dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die) Tj T*
(Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem) Tj T*
(ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor.) Tj T*
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
ET
BT
/F1 10 Tf
288 30 Td
(10) Tj
ET
endstream
endobj
24 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 3 0 R >> >> /Contents 23 0 R >>
endobj
25 0 obj
<< /Length 2348 >>
stream
BT
/F1 22 Tf
50 772 Td
(Kapitel 11 - Seite 11) Tj
ET
BT
/F1 11 Tf
50 732 Td
14 TL
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
(etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed) Tj T*
(do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und) Tj T*
(Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit,) Tj T*
(sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche) Tj T*
(und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing) Tj T*
(elit, sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl,) Tj T*
(Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur) Tj T*
(adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text, damit) Tj T*
(Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet,) Tj T*
(consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text,) Tj T*
(damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit) Tj T*
(amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt) Tj T*
(Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor) Tj T*
(sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite) Tj T*
(trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum) Tj T*
(dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die) Tj T*
(Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem) Tj T*
(ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor.) Tj T*
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
ET
BT
/F1 10 Tf
288 30 Td
(11) Tj
ET
endstream
endobj
26 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 3 0 R >> >> /Contents 25 0 R >>
endobj
27 0 obj
<< /Length 2348 >>
stream
BT
/F1 22 Tf
50 772 Td
(Kapitel 12 - Seite 12) Tj
ET
BT
/F1 11 Tf
50 732 Td
14 TL
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
(etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed) Tj T*
(do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und) Tj T*
(Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit,) Tj T*
(sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche) Tj T*
(und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur adipiscing) Tj T*
(elit, sed do eiusmod tempor. Die Seite trägt Text, damit Auswahl,) Tj T*
(Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet, consectetur) Tj T*
(adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text, damit) Tj T*
(Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit amet,) Tj T*
(consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt Text,) Tj T*
(damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor sit) Tj T*
(amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite trägt) Tj T*
(Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum dolor) Tj T*
(sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die Seite) Tj T*
(trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem ipsum) Tj T*
(dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor. Die) Tj T*
(Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden. Lorem) Tj T*
(ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor.) Tj T*
(Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas finden.) Tj T*
(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod) Tj T*
(tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen etwas) Tj T*
(finden. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do) Tj T*
(eiusmod tempor. Die Seite trägt Text, damit Auswahl, Suche und Vorlesen) Tj T*
ET
BT
/F1 10 Tf
288 30 Td
(12) Tj
ET
endstream
endobj
28 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 3 0 R >> >> /Contents 27 0 R >>
endobj
29 0 obj
<< /Type /Annot /Subtype /Link /Rect [48 306 220 324] /Border [0 0 0] /Dest [14 0 R /XYZ 0 842 null] >>
endobj
30 0 obj
<< /Type /Annot /Subtype /Link /Rect [48 286 220 304] /Border [0 0 0] /Dest [22 0 R /XYZ 0 595 null] >>
endobj
31 0 obj
<< /Type /Annot /Subtype /Link /Rect [48 266 300 284] /Border [0 0 0] /A << /S /URI /URI (https://example.org/) >> >>
endobj
32 0 obj
<< /Type /Outlines /First 33 0 R /Last 35 0 R /Count 3 >>
endobj
33 0 obj
<< /Title (Anfang) /Parent 32 0 R /Next 34 0 R /Dest [6 0 R /XYZ 0 842 null] >>
endobj
34 0 obj
<< /Title (Querformat) /Parent 32 0 R /Prev 33 0 R /Next 35 0 R /Dest [22 0 R /XYZ 0 595 null] >>
endobj
35 0 obj
<< /Title (Fast leer) /Parent 32 0 R /Prev 34 0 R /Dest [18 0 R /XYZ 0 842 null] >>
endobj
36 0 obj
<< /Title (Testdokument PDF) /Author (diora Test) >>
endobj
xref
0 37
0000000000 65535 f
0000000015 00000 n
0000000158 00000 n
0000000292 00000 n
0000000389 00000 n
0000000584 00000 n
0000003099 00000 n
0000003256 00000 n
0000005687 00000 n
0000005839 00000 n
0000008236 00000 n
0000008363 00000 n
0000009612 00000 n
0000009740 00000 n
0000012138 00000 n
0000012266 00000 n
0000014664 00000 n
0000014792 00000 n
0000014875 00000 n
0000015003 00000 n
0000017401 00000 n
0000017529 00000 n
0000019129 00000 n
0000019257 00000 n
0000021658 00000 n
0000021786 00000 n
0000024187 00000 n
0000024315 00000 n
0000026716 00000 n
0000026844 00000 n
0000026964 00000 n
0000027084 00000 n
0000027218 00000 n
0000027292 00000 n
0000027388 00000 n
0000027502 00000 n
0000027602 00000 n
trailer
<< /Size 37 /Root 1 0 R /Info 36 0 R >>
startxref
27671
%%EOF

14
e2e/global-setup.js
View file

@ -11,7 +11,7 @@ const fs = require('fs');
const path = require('path');
const { execFileSync } = require('child_process');
const { chromium } = require('@playwright/test');
const { BASE_URL, PYTHON, SERVER_ENV, TEST_USER } = require('./env');
const { BASE_URL, SERVER_ENV, TEST_USER } = require('./env');
const ROOT = path.join(__dirname, '..');
const DB_PATH = path.join(ROOT, 'data', SERVER_ENV.DIORA_DB_NAME);
@ -22,14 +22,14 @@ module.exports = async function globalSetup() {
// Fresh DB every run.
fs.rmSync(DB_PATH, { force: true });
execFileSync(PYTHON, ['manage.py', 'migrate', '--noinput'], {
execFileSync('python3', ['manage.py', 'migrate', '--noinput'], {
cwd: ROOT,
env: SERVER_ENV,
stdio: 'inherit',
});
execFileSync(
PYTHON,
'python3',
[
'manage.py',
'shell',
@ -45,14 +45,6 @@ User.objects.create_user(${JSON.stringify(TEST_USER.username)}, "", ${JSON.strin
);
fs.mkdirSync(AUTH_DIR, { recursive: true });
// A big PDF (e2e/make_pdf.py): opening draws only the pages near the view.
execFileSync(PYTHON, [path.join(__dirname, 'make_pdf.py'), path.join(AUTH_DIR, 'big.pdf'), '400'], { stdio: 'inherit' });
// The book the reader tests read (e2e/make_epub.py).
execFileSync(PYTHON, [path.join(__dirname, 'make_epub.py'), path.join(AUTH_DIR, 'book.epub')], { stdio: 'inherit' });
// Two more for uploading several at once (navigation.spec.js).
for (const n of ['Eins', 'Zwei']) {
execFileSync(PYTHON, [path.join(__dirname, 'make_epub.py'), path.join(AUTH_DIR, `many-${n}.epub`), `Mehrfach ${n}`], { stdio: 'inherit' });
}
const browser = await chromium.launch();
const page = await browser.newPage({ baseURL: BASE_URL });

58
e2e/make_epub.py
View file

@ -1,58 +0,0 @@
"""A small EPUB for the e2e tests: 12 chapters of numbered paragraphs (long enough
to scroll and to have a reading position), one footnote. Written to the path in
argv[1]; e2e/global-setup.js calls it for every run."""
import sys
import zipfile
CONTAINER = """<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles>
</container>"""
def chapter(n):
paras = []
for p in range(1, 41):
text = f'Kapitel {n}, Absatz {p}. ' + ' '.join(
f'Wort{w} im Absatz {p} des Kapitels {n} steht hier, damit der Absatz einige Zeilen lang wird.'
for w in range(1, 6))
if n == 1 and p == 3:
text += ' Mit Fußnote<a href="ch12.xhtml#fn1" epub:type="noteref"><sup>1</sup></a>.'
paras.append(f'<p id="c{n}p{p}">{text}</p>')
notes = ''
if n == 12:
notes = '<aside epub:type="footnote" id="fn1"><p>1 Der Text der Fußnote.</p></aside>'
return f"""<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops">
<head><title>Kapitel {n}</title></head>
<body><h1>Kapitel {n}</h1>{''.join(paras)}{notes}</body></html>"""
def main(path, title='E2E Testbuch'):
chapters = range(1, 13)
manifest = ''.join(f'<item id="c{n}" href="ch{n}.xhtml" media-type="application/xhtml+xml"/>' for n in chapters)
spine = ''.join(f'<itemref idref="c{n}"/>' for n in chapters)
nav_items = ''.join(f'<li><a href="ch{n}.xhtml">Kapitel {n}</a></li>' for n in chapters)
opf = f"""<?xml version="1.0" encoding="utf-8"?>
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="id">
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
<dc:identifier id="id">diora-e2e-book</dc:identifier>
<dc:title>{title}</dc:title><dc:creator>diora</dc:creator><dc:language>de</dc:language>
</metadata>
<manifest><item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>{manifest}</manifest>
<spine>{spine}</spine>
</package>"""
nav = f"""<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops">
<head><title>Inhalt</title></head><body><nav epub:type="toc"><ol>{nav_items}</ol></nav></body></html>"""
with zipfile.ZipFile(path, 'w') as z:
z.writestr(zipfile.ZipInfo('mimetype'), 'application/epub+zip', compress_type=zipfile.ZIP_STORED)
z.writestr('META-INF/container.xml', CONTAINER, compress_type=zipfile.ZIP_DEFLATED)
z.writestr('OEBPS/content.opf', opf, compress_type=zipfile.ZIP_DEFLATED)
z.writestr('OEBPS/nav.xhtml', nav, compress_type=zipfile.ZIP_DEFLATED)
for n in chapters:
z.writestr(f'OEBPS/ch{n}.xhtml', chapter(n), compress_type=zipfile.ZIP_DEFLATED)
if __name__ == '__main__':
main(sys.argv[1], *sys.argv[2:3])

12
e2e/make_pdf.py
View file

@ -1,12 +0,0 @@
"""A PDF of many pages for the reader tests (big scanned books): make_pdf.py OUT [PAGES]."""
import sys
from pypdf import PdfWriter
out, pages = sys.argv[1], int(sys.argv[2]) if len(sys.argv) > 2 else 400
w = PdfWriter()
for _ in range(pages):
w.add_blank_page(width=420, height=595)
w.add_metadata({'/Title': 'Dicker Band', '/Author': 'diora'})
with open(out, 'wb') as f:
w.write(f)

287
e2e/navigation.spec.js
View file

@ -1,287 +0,0 @@
// The navigation (docs/navigation.md): Home, Lesen (Bücher, Zeitschriften, Nachrichten,
// Blogs), Hören (Hörbücher, Podcasts, Radio, Ambient), on the web page's own bar.
const { test, expect } = require('@playwright/test');
const path = require('path');
test.use({ storageState: path.join(__dirname, '.auth', 'user.json') });
const visiblePanel = page => page.evaluate(() =>
[...document.querySelectorAll('section.tab-panel')].filter(p => p.style.display !== 'none').map(p => p.id));
test('wide: every section in the bar under its group, each shows its panel and view', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 800 });
await page.goto('/#books');
const wide = page.locator('#tabs .nav-wide');
await expect(wide.locator('.nav-group-label')).toHaveText(['Lesen', 'Hören']);
await expect(wide.locator('[data-nav]')).toHaveText(
['Home', 'Bücher', 'Zeitschriften', 'Nachrichten', 'Blogs', 'Hörbücher', 'Podcasts', 'Radio', 'Ambient', 'Fokus', 'Projekte', 'Suche', 'Abos & Quellen']);
await wide.locator('[data-nav="blogs"]').click();
await expect(page).toHaveURL(/#blogs$/);
expect(await visiblePanel(page)).toEqual(['tab-feeds']);
expect(await page.evaluate(() => localStorage.getItem('diora_feeds_view'))).toBe('blogs');
await expect(page.locator('#tab-feeds .fd-seg')).toBeHidden(); // the bar chooses, not the tab
await expect(page.locator('#nav-segments')).toBeHidden();
await wide.locator('[data-nav="audiobooks"]').click();
expect(await visiblePanel(page)).toEqual(['tab-audiobooks']);
await expect(page.locator('#audiobook-shelf')).toBeVisible();
await expect(wide.locator('[data-nav="audiobooks"]')).toHaveClass(/active/);
});
test('narrow: three groups, the sections as a row on top, each group remembers its last', async ({ page }) => {
await page.setViewportSize({ width: 390, height: 800 });
await page.goto('/#news');
const narrow = page.locator('#tabs .nav-narrow');
await expect(narrow.locator('[data-nav]')).toHaveText(['Home', 'Lesen', 'Hören', 'Projekte', '', '']);
await expect(narrow.locator('[data-nav="suche"] svg.dnav-ic')).toBeVisible(); // the magnifier
await expect(narrow.locator('[data-nav="sources"] svg.dnav-ic')).toBeVisible();
await expect(narrow.locator('[data-nav="lesen"]')).toHaveClass(/active/);
const segs = page.locator('#nav-segments');
await expect(segs.locator('[data-nav]')).toHaveText(['Bücher', 'Zeitschriften', 'Nachrichten', 'Blogs']);
await expect(segs.locator('[data-nav="news"]')).toHaveClass(/active/);
await narrow.locator('[data-nav="hoeren"]').click();
await segs.locator('[data-nav="radio"]').click();
expect(await visiblePanel(page)).toEqual(['tab-radio']);
await narrow.locator('[data-nav="lesen"]').click();
await expect(page).toHaveURL(/#news$/); // back where Lesen was left
await narrow.locator('[data-nav="hoeren"]').click();
await expect(page).toHaveURL(/#radio/);
});
test('old links and direct panel switches land in their section', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 800 });
await page.goto('/#feeds');
expect(await visiblePanel(page)).toEqual(['tab-feeds']);
await expect(page.locator('#tabs .nav-wide [data-nav].active')).toHaveText(/Nachrichten|Blogs/);
await page.evaluate(() => showTab('podcasts'));
await expect(page.locator('#tabs [data-nav="podcasts"]')).toHaveClass(/active/);
await expect(page).toHaveURL(/#podcasts$/);
await page.reload();
expect(await visiblePanel(page)).toEqual(['tab-podcasts']); // remembered
});
test('Suche: books from this device, the rest from the server, a hit opens where it lives', async ({ page }) => {
await page.setViewportSize({ width: 390, height: 800 });
await page.route('**/search/?q=*', route => route.fulfill({json: {
q: 'kapital', podcasts: [{id: 7, title: 'Kapital-Lesekreis', author: 'Rote Hilfe', artwork_url: '', favorite: true}],
podcasts_total: 1, audiobooks: [], audiobooks_total: 0,
episodes: [{id: 70, feed__id: 7, feed__title: 'Kapital-Lesekreis', title: 'Folge 3: Das Kapital, Band 1', duration_seconds: 3600}],
sources: [], articles: [{id: 5, title: 'Kapital und Krise', source: 'Ein Blog', kind: 'blog', published: null, read: false}],
articles_total: 1, stations: []}}));
await page.goto('/#home');
await page.locator('#tabs .nav-narrow [data-nav="suche"]').click();
await expect(page).toHaveURL(/#suche$/);
expect(await visiblePanel(page)).toEqual(['tab-suche']);
await expect(page.locator('#tab-suche .sx-q')).toBeFocused();
await page.evaluate(() => {
_lastBookListData = [{id: 901, title: 'Das Kapital', author: 'Karl Marx', type: 'epub', keyOk: true},
{id: 902, title: 'Gartenbuch', author: 'Jemand', type: 'epub', keyOk: true}];
});
await page.locator('#tab-suche .sx-q').fill('kapital');
const groups = page.locator('#tab-suche .sx-h');
await expect(groups).toHaveText([/Bücher/, /Podcasts/, /Folgen/, /Artikel/]);
await expect(page.locator('#tab-suche .sx-group').first().locator('mark')).toHaveText('Kapital');
await page.locator('#tab-suche .sx-q').press('Enter');
await page.locator('#tab-suche .sx-q').fill('');
await expect(page.locator('#tab-suche .sx-recent .sx-chip')).toHaveText(['kapital']);
// Not where the app opens next time.
expect(await page.evaluate(() => localStorage.getItem('diora_nav_section'))).not.toBe('suche');
});
test('Projekte: a Forschung project, a paper by DOI, a linked book, a note citing them', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 900 });
await page.route('**/projects/doi/?q=*', route => route.fulfill({json: {doi: '10.1038/nn.1234', csl: {
type: 'article-journal', title: 'Neural correlates of political ideology', 'container-title': 'Nature Neuroscience',
author: [{family: 'Amodio', given: 'David'}, {family: 'Jost', given: 'John'}], issued: {'date-parts': [[2007, 9]]},
DOI: '10.1038/nn1979', volume: '10', page: '1246-1247'}}}));
await page.goto('/#projekte');
const tab = page.locator('#tab-projekte');
await tab.locator('[data-act="new"]').click();
await expect(tab.locator('.prj-preset.on')).toContainText('Forschung');
await tab.locator('input[name="name"]').fill('Neuropolitik');
await tab.locator('button[type="submit"]').click();
await expect(tab.locator('.prj-h1')).toHaveText('Neuropolitik');
await expect(page).toHaveURL(/#projekte\/\d+$/);
// The server keeps only ciphertext.
const raw = await page.evaluate(() => fetch('/projects/').then(r => r.json()));
expect(JSON.stringify(raw)).not.toContain('Neuropolitik');
await tab.locator('.prj-add input').fill('https://doi.org/10.1038/nn1979');
await tab.locator('.prj-add button').click();
await tab.locator('[data-add-doi]').click();
await expect(tab.locator('.prj-row .prj-title')).toHaveText(['Neural correlates of political ideology']);
await expect(tab.locator('.prj-key')).toHaveText(['amodio2007neural']);
// From a book's sheet elsewhere: DioraProjects.link.
await page.evaluate(() => DioraProjects.link('book', 4242, {title: 'Das politische Gehirn', author: 'Hans Muster'}));
await page.locator('.prj-picker [data-pick]').filter({hasText: 'Neuropolitik'}).click();
await expect(tab.locator('.prj-row')).toHaveCount(2);
expect(await page.evaluate(() => DioraProjects.of('book', 4242).map(p => p.name))).toEqual(['Neuropolitik']);
await tab.locator('[data-view="notes"]').click();
await tab.locator('[data-act="new-note"]').click();
await tab.locator('[data-note-title]').fill('These');
await tab.locator('[data-note-body]').fill('# Idee\nIdeologie zeigt sich im Gehirn [@amodio2007neural, S. 3] und bei [@muster0jdas].\n\n| Text | Zeit |\n|---|---|\n| Cobb | 2,5 h |\nDanach.');
await expect(tab.locator('.prj-saved')).toHaveText('gespeichert', {timeout: 5000});
await tab.locator('[data-act="preview"]').click();
await expect(tab.locator('.prj-md .prj-cite').first()).toHaveText('(Amodio & Jost 2007, S. 3)');
await expect(tab.locator('.prj-refs li').first()).toContainText('Nature Neuroscience');
await expect(tab.locator('.prj-md td')).toHaveText(['Cobb', '2,5 h']);
await tab.locator('[data-view="sources"]').click();
const [dl] = await Promise.all([page.waitForEvent('download'), tab.locator('[data-export="bib"]').click()]);
const bib = require('fs').readFileSync(await dl.path(), 'utf8');
expect(bib).toContain('@article{amodio2007neural,');
expect(bib).toContain('doi = {10.1038/nn1979}');
});
test('Radar: the week\'s digest, decisions kept encrypted, a paper into the sources', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 900 });
const digest = {id: 1, generated: '2026-10-19', bundle: {version: 1, generated: '2026-10-19',
auf_einen_blick: 'Ein Decoder kommt der Absicht zuvor.', grundlagenbegriff: {begriff: 'Decoder', erklaerung: 'Ein Modell …'},
news_markdown: '- **Paradromics:** Kooperation. [Quelle](https://example.org/n)',
stats: {records: 3, per_source: {PubMed: 2, arXiv: 1}, errors: [], cost_usd: 0.38},
items: [
{id: 'pmid:1', doi: '', source: 'PubMed', title: 'Intention decoded', journal: 'Nature', first_author: 'Verbaarschot',
n_authors: 5, date: '2026', url: 'https://example.org/1', gruppe: 'lesenswert', worum_es_geht: 'Ein Spiel.',
einordnung: 'Preprint.', politischer_haken: 'Libet.', kurz: 'Kurz.'},
{id: 'arxiv:2', source: 'arXiv', title: 'EEG artefacts', first_author: 'Adeika', n_authors: 2, url: 'https://example.org/2',
gruppe: 'technik', worum_es_geht: '', einordnung: '', politischer_haken: '', kurz: 'Muskeln statt Gedanken.'},
{id: 'pmid:3', source: 'PubMed', title: 'Off topic', gruppe: 'rauschen', kurz: ''}]}};
await page.route(/\/projects\/\d+\/radar\/$/, route => route.fulfill({json: {radar: {enabled: true, title: 'Neuropolitik-Digest',
groups: ['lesenswert', 'propriozeption', 'gesellschaft', 'technik', 'rauschen'], group_labels: {lesenswert: 'Lesenswert', technik: 'Technik im Überblick', rauschen: 'Rauschen'},
weekday: 6, hour: 18, days: 7, sources: ['PubMed', 'arXiv'], running: false, next_run: '2026-10-26T18:00:00+01:00',
last_error: '', key: true}, digests: [digest]}}));
await page.goto('/#projekte');
const tab = page.locator('#tab-projekte');
await tab.locator('[data-act="new"]').click();
await tab.locator('input[name="name"]').fill('Radarprojekt');
await tab.locator('button[type="submit"]').click();
await expect(tab.locator('.prj-tab.on')).toContainText('Radar');
await expect(tab.locator('.prj-radar-h')).toContainText('KW 43');
await expect(tab.locator('.prj-radar-group[open] > summary')).toHaveText([/Lesenswert/, /Technik/, /Branche/]);
await expect(tab.locator('.prj-paper').first()).toContainText('Politischer Haken: Libet.');
await tab.locator('[data-radar-decide="arxiv:2"][data-state="later"]').click();
await tab.locator('[data-radar-decide="pmid:1"][data-state="read"]').click();
await expect(tab.locator('[data-radar-later]')).toHaveText('Später 1');
// Kept, encrypted: after a reload still there.
await page.waitForTimeout(800);
await page.reload();
await expect(tab.locator('[data-radar-later]')).toHaveText('Später 1');
await expect(tab.locator('.prj-paper').first()).toContainText('gelesen');
const raw = await page.evaluate(() => fetch(location.pathname.replace(/\/$/, '') + '/projects/').then(r => r.json()));
expect(JSON.stringify(raw)).not.toContain('arxiv:2');
await tab.locator('[data-radar-source="arxiv:2"]').click();
await expect(tab.locator('.prj-paper').nth(1)).toContainText('✓ in Quellen');
await tab.locator('[data-view="sources"]').click();
await expect(tab.locator('.prj-row .prj-title')).toHaveText(['EEG artefacts']);
});
test('Abos & Quellen: one page for every subscription, reached from the feeds too', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 800 });
await page.goto('/#news');
await page.locator('#tab-feeds [data-act="manage"]').first().click();
await expect(page).toHaveURL(/#sources$/);
const tab = page.locator('#tab-sources');
await expect(tab.locator('.src-h1')).toHaveText('Abos & Quellen');
await expect(tab.locator('.src-add-q')).toBeVisible();
await expect(tab.locator('.src-chip.on')).toContainText('Nachrichten'); // came from the news
await tab.locator('[data-filter="all"]').click();
await expect(tab.locator('.src-chip.on')).toContainText('Alle');
});
test('the library: a header with search, sort, layout and +, the add menu', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 800 });
await page.goto('/#books');
const list = page.locator('#book-list');
await expect(list.locator('.lib-h')).toHaveText('Bücher');
await expect(list.locator('.lib-q')).toBeVisible();
await expect(list.locator('.lib-sort')).toBeVisible();
await expect(page.locator('#book-drop-zone')).toBeHidden(); // adding is behind +
await list.locator('[data-lib="add"]').click();
await expect(page.locator('.dfo-sheet')).toContainText('E-Book-Datei');
await page.locator('.dfo-sheet [data-x]').click();
await expect(page.locator('.dfo-sheet')).toHaveCount(0);
await page.goto('/#magazines');
await expect(list.locator('.lib-h')).toHaveText('Zeitschriften');
});
test('several books at once; one already there is left out', async ({ page }) => {
const files = ['Eins', 'Zwei'].map(n => path.join(__dirname, '.auth', `many-${n}.epub`));
await page.goto('/#books');
await page.waitForFunction(() => typeof _importMany === 'function' && Array.isArray(_lastBookListData));
const status = page.locator('#book-upload-status');
await page.locator('#book-file-input').setInputFiles(files);
await expect(status).toContainText('2 Bücher hinzugefügt', { timeout: 30000 });
await page.waitForFunction(() => _lastBookListData.filter(b => /^Mehrfach/.test(b.title)).length === 2);
await page.locator('#book-file-input').setInputFiles(files);
await expect(status).toContainText('2 schon da', { timeout: 30000 });
// Out again, so the other tests see the library as before.
const books = await page.evaluate(() => _lastBookListData.filter(b => /^Mehrfach/.test(b.title)).map(b => b.id));
for (const id of books) await page.request.post(`/books/${id}/delete/`);
});
test('editions: two books joined show once with their formats, set apart show twice', async ({ page }) => {
await page.setViewportSize({ width: 1200, height: 800 });
const files = ['Eins', 'Zwei'].map(n => path.join(__dirname, '.auth', `many-${n}.epub`));
await page.goto('/#books');
await page.waitForFunction(() => typeof _importMany === 'function' && Array.isArray(_lastBookListData));
await page.locator('#book-file-input').setInputFiles(files);
await page.waitForFunction(() => _lastBookListData.filter(b => /^Mehrfach/.test(b.title) && b.id).length === 2, null, { timeout: 30000 });
const ids = await page.evaluate(() => _lastBookListData.filter(b => /^Mehrfach/.test(b.title)).map(b => b.id));
const list = page.locator('#book-list');
// New books: as cards under "Ungelesen" (not in the "Zuletzt gelesen" area on top).
const unread = () => page.evaluate(() => { _lib.status = 'unread'; renderBookList(_lastBookListData); });
await unread();
const cards = () => list.locator('.lib-grid [data-lib-book]').filter({ hasText: 'Mehrfach' });
await expect(cards()).toHaveCount(2);
// Choose both, join them.
await page.evaluate(ids => { _lib.selecting = new Set(ids); renderBookList(_lastBookListData); }, ids);
await list.locator('[data-lib="sel-editions"]').click();
await page.waitForFunction(ids => {
const w = ids.map(id => _lastBookListData.find(b => b.id === id).work);
return w[0] && w[0] === w[1];
}, ids);
await expect(cards()).toHaveCount(1);
await expect(cards().locator('.lib-ed')).toHaveText('EPUB');
// The work survives a reload (it is in the encrypted meta).
await page.reload();
await page.waitForFunction(() => Array.isArray(_lastBookListData) && _lastBookListData.length);
await unread();
await expect(cards()).toHaveCount(1);
// Ausgaben: both listed; one set apart, and the other is on its own again.
await cards().locator('.lib-more').click({ force: true });
await page.locator('.dfo-sheet .lib-sheet-item', { hasText: 'Ausgaben (2)' }).click();
await expect(page.locator('.dfo-sheet .lib-ed-row')).toHaveCount(2);
await page.locator('.dfo-sheet [data-unlink]').first().click();
await page.waitForFunction(ids => ids.every(id => !_lastBookListData.find(b => b.id === id).work), ids);
await expect(cards()).toHaveCount(2);
await page.evaluate(() => { _lib.status = 'all'; });
for (const id of ids) await page.request.post(`/books/${id}/delete/`);
});
test('a meta change from an old list goes onto the current meta (409, then again)', async ({ page }) => {
const files = [path.join(__dirname, '.auth', 'many-Eins.epub')];
await page.goto('/#books');
await page.waitForFunction(() => typeof _importMany === 'function' && Array.isArray(_lastBookListData));
await page.locator('#book-file-input').setInputFiles(files);
await page.waitForFunction(() => _lastBookListData.some(b => b.title === 'Mehrfach Eins' && b.id && b.meta_iv), null, { timeout: 30000 });
const id = await page.evaluate(() => _lastBookListData.find(b => b.title === 'Mehrfach Eins').id);
// Elsewhere the title changes; this page still has the old list.
await page.evaluate(id => _updateBookMeta(id, {title: 'Anderswo umbenannt'}), id);
await page.evaluate(id => {
const b = _lastBookListData.find(x => x.id === id);
b.title = 'Mehrfach Eins'; b.meta_iv = '000000000000000000000000'; // the old list
bookMetaCache[id] = {...bookMetaCache[id], title: 'Mehrfach Eins'};
}, id);
await page.evaluate(id => _updateBookMeta(id, {folders: ['Probe']}), id);
const meta = await page.evaluate(async id => {
const row = (await (await fetch('/books/', {cache: 'no-store'})).json()).find(b => b.id === id);
return JSON.parse(new TextDecoder().decode(await decryptBytes(await getOrCreateEncKey(), row.meta_iv, row.meta_ct)));
}, id);
expect(meta.title).toBe('Anderswo umbenannt');
expect(meta.folders).toEqual(['Probe']);
await page.request.post(`/books/${id}/delete/`);
});

71
e2e/pdf.spec.js
View file

@ -1,71 +0,0 @@
// A big PDF (a scanned volume of 400 pages, e2e/make_pdf.py): it opens at once, only the
// pages near the view are drawn, the end can be reached, and opening can be cancelled.
const path = require('path');
const { test, expect } = require('@playwright/test');
test.use({ storageState: path.join(__dirname, '.auth', 'user.json') });
test.describe.configure({ mode: 'serial' });
const PDF = path.join(__dirname, '.auth', 'big.pdf');
let bookId = null;
let pageErrors = [];
test.beforeEach(({ page }) => {
pageErrors = [];
page.on('pageerror', e => pageErrors.push(String(e)));
});
test.afterEach(() => {
expect(pageErrors, 'errors in the page').toEqual([]);
});
async function ready(page) {
await page.goto('/#books');
await page.waitForLoadState('networkidle');
await page.waitForFunction(() => typeof openBook === 'function' && Array.isArray(_lastBookListData));
}
const drawn = page => page.evaluate(() =>
[...document.querySelectorAll('#reader-content canvas.pdf-page')].filter(c => c.width > 0).length);
test('the big PDF goes in', async ({ page }) => {
await ready(page);
await page.locator('#book-file-input').setInputFiles(PDF);
await page.waitForFunction(() => _lastBookListData.some(b => b.title === 'Dicker Band'), null, { timeout: 30000 });
bookId = await page.evaluate(() => _lastBookListData.find(b => b.title === 'Dicker Band').id);
});
test('it opens with every page placed, only those near the view drawn', async ({ page }) => {
await ready(page);
await page.evaluate(id => openBook(id), bookId);
await page.waitForFunction(() => _readPos && _readPos.ready, null, { timeout: 20000 });
await expect(page.locator('#reader-content .pdf-page-wrapper')).toHaveCount(400);
await expect(page.locator('.pdf-loading-overlay')).toHaveCount(0);
await expect.poll(() => drawn(page)).toBeGreaterThan(0);
expect(await drawn(page)).toBeLessThan(30);
// To the end: the last page gets drawn, the first ones are let go.
await page.evaluate(() => { const c = document.getElementById('reader-content'); c.scrollTop = c.scrollHeight; });
await expect.poll(() => page.evaluate(() => document.querySelector('#pdf-page-400 canvas').width > 0)).toBe(true);
await expect.poll(() => page.evaluate(() => document.querySelector('#pdf-page-1 canvas').width)).toBe(0);
expect(await drawn(page)).toBeLessThan(30);
await page.evaluate(() => closeReader());
});
test('opening can be cancelled, and another book opens after it', async ({ page }) => {
await ready(page);
// Slow the file down so the opening is still under way.
await page.route(`**/books/${bookId}/data/`, async route => {
await new Promise(r => setTimeout(r, 1500));
await route.continue().catch(() => {});
});
const p = page.evaluate(id => openBook(id).then(() => 'done', e => 'error ' + e), bookId);
await expect(page.locator('.reader-opening-cancel')).toBeVisible({ timeout: 5000 });
await page.locator('.reader-opening-cancel').click();
await expect(page.locator('#reader-overlay')).toBeHidden();
expect(await p).toBe('done');
await page.unroute(`**/books/${bookId}/data/`);
// Opening it again works, all the way.
await page.evaluate(id => openBook(id), bookId);
await page.waitForFunction(() => _readPos && _readPos.ready, null, { timeout: 20000 });
await page.evaluate(() => closeReader());
await page.request.post(`/books/${bookId}/delete/`);
});

169
e2e/reader.spec.js
View file

@ -1,169 +0,0 @@
// The reader end to end: a book goes in (encrypted in the browser), opens, is
// read, and its place, a highlight and the theme survive a reload. The safety
// net for refactoring app.js (docs/refactoring.md). One book for the whole file:
// the tests build on each other and run in order.
const path = require('path');
const { test, expect } = require('@playwright/test');
test.use({ storageState: path.join(__dirname, '.auth', 'user.json') });
test.describe.configure({ mode: 'serial' });
const BOOK = path.join(__dirname, '.auth', 'book.epub');
let bookId = null;
// Errors in the page fail the test that caused them.
let pageErrors = [];
test.beforeEach(({ page }) => {
pageErrors = [];
page.on('pageerror', e => pageErrors.push(String(e)));
});
test.afterEach(() => {
expect(pageErrors, 'errors in the page').toEqual([]);
});
async function openTestBook(page) {
await page.goto('/#books');
// A new service worker taking over reloads the page once: let that settle first.
await page.waitForLoadState('networkidle');
await page.waitForFunction(() => typeof openBook === 'function' && typeof _lastBookListData !== 'undefined' && Array.isArray(_lastBookListData));
await page.evaluate(id => openBook(id), bookId);
await page.waitForFunction(() => document.querySelectorAll('#reader-content [data-epub-src]').length === 12
&& typeof _readPos !== 'undefined' && _readPos && _readPos.ready);
}
async function serverBook(page) {
const books = await (await page.request.get('/books/')).json();
return books.find(b => b.id === bookId);
}
test('a book goes in and shows in the list', async ({ page }) => {
await page.goto('/#books');
await page.setInputFiles('#book-file-input', BOOK);
const found = await page.waitForFunction(() => {
const list = typeof _lastBookListData !== 'undefined' ? (_lastBookListData || []) : [];
const b = list.find(x => x.title === 'E2E Testbuch');
return b ? b.id : null;
}, null, { timeout: 20000 });
bookId = await found.jsonValue();
expect(bookId).toBeTruthy();
});
test('opening without reading sends no position', async ({ page }) => {
const posts = [];
page.on('request', r => { if (r.method() === 'POST' && /\/progress\/$/.test(r.url())) posts.push(r.url()); });
await openTestBook(page);
await page.waitForTimeout(3000);
await page.evaluate(() => saveReaderProgress());
await page.evaluate(() => closeReader());
await page.waitForTimeout(500);
expect(posts).toEqual([]);
});
test('reading moves the position, and it comes back after a reload', async ({ page }) => {
await openTestBook(page);
const content = page.locator('#reader-content');
await content.hover();
for (let i = 0; i < 12; i++) await page.mouse.wheel(0, 600);
const saved = page.waitForRequest(r => r.method() === 'POST' && /\/progress\/$/.test(r.url()));
const body = JSON.parse((await saved).postData());
expect(body.device).toBe('Browser');
expect(body.read_at).toBeGreaterThan(1e12);
const anchor = await page.evaluate(() => getPositionAnchor(document.getElementById('reader-content')));
await page.waitForTimeout(500);
const b = await serverBook(page);
expect(b.position_anchor).not.toBe('');
expect(b.device).toBe('Browser');
await page.reload();
await openTestBook(page);
const restored = await page.evaluate(() => getPositionAnchor(document.getElementById('reader-content')));
const block = a => Number(a.split(':')[0]);
expect(Math.abs(block(restored) - block(anchor))).toBeLessThanOrEqual(1);
});
test('a footnote shows its text', async ({ page }) => {
await openTestBook(page);
const ref = page.locator('#reader-content [data-footnote-ref]').first();
await ref.scrollIntoViewIfNeeded();
await ref.click();
await expect(page.locator('.footnote-popover')).toContainText('Der Text der Fußnote');
});
test('a highlight is kept, and shows in the margin', async ({ page }) => {
await openTestBook(page);
// Marker mode on (the ✎ button), select the start of a paragraph on screen
// the way a mouse would, then let go.
await page.evaluate(() => {
if (!markerModeActive) toggleMarkerMode();
const p = document.querySelector('#reader-content #c1p2') || document.querySelector('#reader-content p');
p.scrollIntoView({ block: 'center' });
const text = p.firstChild;
const range = document.createRange();
range.setStart(text, 0);
range.setEnd(text, 20);
const sel = window.getSelection();
sel.removeAllRanges();
sel.addRange(range);
const r = range.getBoundingClientRect();
p.dispatchEvent(new MouseEvent('mouseup', { bubbles: true, clientX: r.right, clientY: r.bottom }));
});
const saved = page.waitForRequest(r => r.method() === 'POST' && /\/highlights\/$/.test(r.url()), { timeout: 15000 });
await page.locator('.hl-color-btn[data-hl-color="yellow"]').click();
await page.evaluate(() => saveHighlights && saveHighlights());
await saved;
await page.reload();
await openTestBook(page);
await page.waitForFunction(() => document.querySelectorAll('#reader-content mark.epub-highlight, #reader-content .epub-highlight').length > 0);
// The margin remembers being open; open it if it isn't.
await page.evaluate(() => { if (!readerAnnotationsMarginOpen) toggleAnnotationsMargin(); });
await expect(page.locator('#reader-margin-markers .margin-note')).toHaveCount(1);
});
test('an open margin keeps the text column in the middle when there is room', async ({ page }) => {
const column = () => page.evaluate(() => {
const c = document.getElementById('reader-content');
const p = c.querySelector('p');
const r = p.getBoundingClientRect();
const m = document.getElementById('reader-margin').getBoundingClientRect();
return {left: r.left, right: r.right, width: r.width, marginRight: m.right, window: window.innerWidth};
});
await page.setViewportSize({ width: 1600, height: 900 });
await openTestBook(page);
const closed = await column();
await page.evaluate(() => { if (!readerAnnotationsMarginOpen) toggleAnnotationsMargin(); });
await page.waitForTimeout(400); // the margin and the padding slide together
const open = await column();
expect(Math.abs(open.left - closed.left)).toBeLessThanOrEqual(2);
expect(open.width).toBeCloseTo(closed.width, 0);
// Not enough room: the column moves aside, but keeps its width and clears the margin.
await page.setViewportSize({ width: 1000, height: 900 });
await page.waitForTimeout(400);
const narrow = await column();
expect(narrow.left).toBeGreaterThanOrEqual(narrow.marginRight);
await page.evaluate(() => { toggleAnnotationsMargin(); });
});
test('the theme switch turns the page and the reader light, and stays', async ({ page }) => {
await page.goto('/');
await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark');
await page.locator('#theme-toggle').click();
await expect(page.locator('html')).toHaveAttribute('data-theme', 'light');
await page.reload();
await expect(page.locator('html')).toHaveAttribute('data-theme', 'light');
await openTestBook(page);
await expect(page.locator('#reader-overlay')).toHaveClass(/reader-theme-bright/);
await page.evaluate(() => { closeReader(); __dioraSetTheme('dark'); });
await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark');
});
test('every tab opens without errors', async ({ page }) => {
await page.goto('/');
await page.waitForLoadState('networkidle');
await page.waitForFunction(() => typeof showTab === 'function' && typeof loadBookList === 'function');
for (const tab of ['home', 'radio', 'podcasts', 'books', 'feeds', 'focus', 'ambient']) {
await page.evaluate(t => showTab(t), tab);
await expect(page.locator(`#tab-${tab}`)).toBeVisible();
}
});

178
e2e/sponsor.spec.js
View file

@ -1,178 +0,0 @@
// Skipping the ad and the outro of a YouTube episode (docs/youtube.md, podcasts/sponsor.py).
//
// The sound of a YouTube episode plays only in an app, so the desktop is imitated here:
// window.dioraDesktop.youtubeAudio answers with a stream of our own (a silent WAV, served
// with byte ranges so the player can jump in it). The browser host (podcasts-web.js) is the
// desktop's podcast host, so this is the code that runs there.
const { test, expect } = require('@playwright/test');
const { execFileSync } = require('child_process');
const path = require('path');
const { PYTHON, SERVER_ENV, TEST_USER } = require('./env');
const ROOT = path.join(__dirname, '..');
const VID = 'aqz-KE-bpKQ';
const WATCH = `https://www.youtube.com/watch?v=${VID}`;
const SPANS = [[1, 3], [6, 7.5]]; // ad, outro
const SECONDS = 9;
const STREAM = '/e2e-silence.wav';
// A silent 8 kHz mono WAV: long enough to reach both parts and past them.
function wav(seconds, rate = 8000) {
const n = seconds * rate;
const buf = Buffer.alloc(44 + n * 2);
buf.write('RIFF', 0);
buf.writeUInt32LE(36 + n * 2, 4);
buf.write('WAVE', 8);
buf.write('fmt ', 12);
buf.writeUInt32LE(16, 16);
buf.writeUInt16LE(1, 20); // PCM
buf.writeUInt16LE(1, 22); // mono
buf.writeUInt32LE(rate, 24);
buf.writeUInt32LE(rate * 2, 28); // byte rate
buf.writeUInt16LE(2, 32); // block align
buf.writeUInt16LE(16, 34); // bits
buf.write('data', 36);
buf.writeUInt32LE(n * 2, 40);
return buf;
}
const BODY = wav(SECONDS);
function seed() {
const py = `
from django.contrib.auth import get_user_model
from django.utils import timezone
from podcasts.models import PodcastEpisode, PodcastFeed, PodcastQueue
import json
u = get_user_model().objects.get(username=${JSON.stringify(TEST_USER.username)})
PodcastQueue.objects.filter(user=u).delete()
PodcastEpisode.objects.filter(feed__user=u).delete()
PodcastFeed.objects.filter(user=u).delete()
f = PodcastFeed.objects.create(user=u, rss_url='https://www.youtube.com/feeds/videos.xml?channel_id=UCe2etest',
title='Kanal')
ep = PodcastEpisode.objects.create(feed=f, guid='yt:video:${VID}', title='Eine Folge',
audio_url=${JSON.stringify(WATCH)}, pub_date=timezone.now(),
duration_seconds=${SECONDS},
sponsor={'v': 1, 'spans': ${JSON.stringify(SPANS)},
'at': timezone.now().isoformat()})
PodcastQueue.objects.create(user=u, episode=ep, position=0)
print('episode', ep.id)
`;
execFileSync(PYTHON, ['manage.py', 'shell', '-c', py], { cwd: ROOT, env: SERVER_ENV, stdio: 'inherit' });
}
test.beforeAll(seed);
test.use({ storageState: 'e2e/.auth/user.json' });
// The stream, with byte ranges: without them Chromium may refuse to jump in a file.
async function serveStream(page) {
await page.route(`**${STREAM}`, route => {
const range = route.request().headers()['range'];
const head = { 'Content-Type': 'audio/wav', 'Accept-Ranges': 'bytes' };
if (!range) return route.fulfill({ status: 200, headers: head, body: BODY });
const m = /bytes=(\d*)-(\d*)/.exec(range);
const from = m && m[1] ? Number(m[1]) : 0;
const to = m && m[2] ? Number(m[2]) : BODY.length - 1;
return route.fulfill({
status: 206,
headers: { ...head, 'Content-Range': `bytes ${from}-${to}/${BODY.length}`, 'Content-Length': String(to - from + 1) },
body: BODY.subarray(from, to + 1),
});
});
}
// The desktop the host talks to, and a look at the player's own position over time.
async function imitateDesktop(page) {
await page.addInitScript(url => {
window.dioraDesktop = {
youtubeAudio: () => Promise.resolve({ url }),
deviceName: 'diora Desktop',
};
window.__times = [];
const Orig = window.Audio;
window.Audio = function () {
const el = new Orig(...arguments);
window.__el = el;
el.addEventListener('timeupdate', () => window.__times.push(el.currentTime));
el.addEventListener('playing', () => window.__times.push(el.currentTime));
return el;
};
window.Audio.prototype = Orig.prototype;
}, STREAM);
}
// The episode as the host itself holds it (the queue from /podcasts/state/).
async function episode(page) {
return page.evaluate(() => {
const s = JSON.parse(window.DioraPodcastHost.podcastState());
return (s.queue || []).find(q => /watch\?v=/.test(q.episode__audio_url || '')) || null;
});
}
function inPart(t) {
return SPANS.find(([from, to]) => t > from + 0.05 && t < to - 0.05) || null;
}
test('the ad and the outro are jumped over', async ({ page }) => {
const asked = [];
page.on('request', r => { if (/\/podcasts\/episodes\/\d+\/sponsor\//.test(r.url())) asked.push(r.url()); });
await serveStream(page);
await imitateDesktop(page);
await page.goto('/');
await page.waitForFunction(() => window.DioraPodcastHost, null, { timeout: 15000 });
await page.waitForFunction(() => {
const s = JSON.parse(window.DioraPodcastHost.podcastState());
return (s.queue || []).some(q => /watch\?v=/.test(q.episode__audio_url || ''));
}, null, { timeout: 15000 });
const ep = await episode(page);
expect(ep, 'the seeded episode is in the queue').not.toBeNull();
await page.evaluate(json => window.DioraPodcastHost.podcastPlayEpisode(json, 0), JSON.stringify(ep));
// Where the sound is over time, from here (a `timeupdate` comes only every 250 ms).
const times = await sample(page, 4.5);
expect(asked.length, 'the server was asked for the parts').toBeGreaterThan(0);
const inside = times.filter(inPart);
expect(inside, `no part was played: ${inside.join(', ')} of ${times.length} positions`).toEqual([]);
expect(Math.max(...times), 'the sound kept going past the ad').toBeGreaterThan(3.2);
});
// The player's position, read every 100 ms until it passed `until` (in seconds).
async function sample(page, until, limit = 250) {
const times = [];
for (let i = 0; i < limit; i++) {
const t = await page.evaluate(() => (window.__el && window.__el.readyState > 0
? window.__el.currentTime : null));
if (t !== null) times.push(t);
if (t !== null && t > until) break;
await page.waitForTimeout(100);
}
return times;
}
test('with the setting off nothing is asked and nothing is jumped over', async ({ page }) => {
seed(); // the first test left the episode played (and out of the queue)
let asked = 0;
page.on('request', r => { if (/\/podcasts\/episodes\/\d+\/sponsor\//.test(r.url())) asked++; });
await serveStream(page);
await imitateDesktop(page);
await page.goto('/');
// This device's setting, before the host reads it (localStorage, PlaybackPrefs.kt).
await page.evaluate(() => localStorage.setItem('dpw_prefs_' + window.USER_ID, JSON.stringify({skip_sponsor: false})));
await page.reload();
await page.waitForFunction(() => window.DioraPodcastHost, null, { timeout: 15000 });
await page.waitForFunction(() => {
const s = JSON.parse(window.DioraPodcastHost.podcastState());
return (s.queue || []).some(q => /watch\?v=/.test(q.episode__audio_url || ''));
}, null, { timeout: 15000 });
expect(await page.evaluate(() => JSON.parse(window.DioraPodcastHost.podcastPrefs()).skip_sponsor)).toBe(false);
const ep = await episode(page);
await page.evaluate(json => window.DioraPodcastHost.podcastPlayEpisode(json, 0), JSON.stringify(ep));
const times = await sample(page, 4.5);
expect(asked, 'nothing was asked for').toBe(0);
const inside = times.filter(inPart);
expect(inside.length, `the ad was played (${inside.join(', ')})`).toBeGreaterThan(0);
});

0
feeds/__init__.py
View file

6
feeds/apps.py
View file

@ -1,6 +0,0 @@
from django.apps import AppConfig
class FeedsConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'feeds'

149
feeds/cluster.py
View file

@ -1,149 +0,0 @@
"""Bundling news items about the same event into stories.
Each item is described by the words of its title (counted twice) and summary,
without stop words and with a crude German/English suffix cut. A new item joins
the most similar story of the last WINDOW hours (cosine of tf-idf vectors,
document frequencies over those stories) if that similarity reaches THRESHOLD
and they share at least MIN_SHARED words; otherwise it starts a story of its own.
A story many sources report on is a top story.
"""
import math
import re
from collections import Counter
from datetime import timedelta
from .models import Item, Story
WINDOW = timedelta(hours=36)
THRESHOLD = 0.25
MIN_SHARED = 2
STOP = set('''
aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher
da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese
diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für
gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer
in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen
nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind
so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter
weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier
heute gestern morgen laut sagt sagte erst mal neu alt video live ticker liveblog news meldung bericht
the and for with from that this have has are was were will would been into about after over their they
his her its our you your not but all can new says said more than when what who how why off out
tot tote toten getötet sterben stirbt gestorben menschen mensch leute millionen milliarden prozent euro
deutschland deutsche deutschen land länder stadt welt woche wochen monat monate tag tage tagen uhr
frau frauen mann männer kinder erste ersten zweite neuer neues großen große groß kleine wegen gibt geben
'''.split())
_WORD = re.compile(r'[a-zäöüß][a-zäöüß0-9\-]+')
_SUFFIXES = ('ungen', 'ung', 'en', 'er', 'es', 'e', 's', 'n')
def _stem(w):
if len(w) > 5:
for s in _SUFFIXES:
if w.endswith(s) and len(w) - len(s) >= 4:
return w[:-len(s)]
return w
def words(text):
out = []
for w in _WORD.findall((text or '').lower()):
w = w.strip('-')
if len(w) < 3 or w in STOP:
continue
out.append(_stem(w))
return out
def item_terms(title, summary):
return Counter(words(title) * 2 + words(summary[:300]))
def _cosine(a, b, idf):
dot = sum(a[w] * b[w] * idf.get(w, 1.0) ** 2 for w in a.keys() & b.keys())
if not dot:
return 0.0
na = math.sqrt(sum((n * idf.get(w, 1.0)) ** 2 for w, n in a.items()))
nb = math.sqrt(sum((n * idf.get(w, 1.0)) ** 2 for w, n in b.items()))
return dot / (na * nb)
class Clusterer:
"""Assigns one user's new news items to stories (load once, add items, done).
An item is compared with every item already in a story: the story's score
is the mean of the best match and the average match, so one stray shared
word can't pull an item in, and a story doesn't grow into a magnet the way
a pooled word list would. Items of the same source join only when very
close (a source rarely reports one event twice, but often two similar ones)."""
SAME_SOURCE = 0.5
def __init__(self, user, now):
self.user = user
self.stories = list(Story.objects.filter(user=user, last_at__gte=now - WINDOW))
self.members = {s.pk: [] for s in self.stories} # story id -> [(source id, terms)]
for i in Item.objects.filter(story__in=self.stories, gone=False).only('story_id', 'source_id', 'title', 'summary'):
self.members[i.story_id].append((i.source_id, item_terms(i.title, i.summary)))
self._df = Counter()
for ms in self.members.values():
for _, t in ms:
self._df.update(t.keys())
self._n = sum(len(ms) for ms in self.members.values())
def _idf(self):
n = self._n + 1
return {w: math.log((n + 1) / (d + 1)) + 1 for w, d in self._df.items()}
def score(self, t, source_id, members, idf):
sims = []
for sid, mt in members:
if len(t.keys() & mt.keys()) < MIN_SHARED:
sims.append(0.0)
continue
sim = _cosine(t, mt, idf)
if sid == source_id and sim < self.SAME_SOURCE:
sim = 0.0
sims.append(sim)
if not sims or max(sims) == 0:
return 0.0
return (max(sims) + sum(sims) / len(sims)) / 2
def add(self, item):
t = item_terms(item.title, item.summary)
best, best_sim = None, 0.0
if t:
idf = self._idf()
for s in self.stories:
if abs((s.last_at - item.published).total_seconds()) > WINDOW.total_seconds():
continue
sim = self.score(t, item.source_id, self.members[s.pk], idf)
if sim > best_sim:
best, best_sim = s, sim
if best is not None and best_sim >= THRESHOLD:
self._join(best, item)
else:
best = Story.objects.create(user=self.user, lead=item, first_at=item.published,
last_at=item.published, size=1, sources=1)
self.stories.append(best)
self.members[best.pk] = []
item.story = best
item.save(update_fields=['story'])
self.members[best.pk].append((item.source_id, t))
self._df.update(t.keys())
self._n += 1
return best
def _join(self, s, item):
item.story = s
item.save(update_fields=['story'])
s.size = Item.objects.filter(story=s).count()
s.sources = Item.objects.filter(story=s).values('source').distinct().count()
s.first_at = min(s.first_at, item.published)
s.last_at = max(s.last_at, item.published)
if s.lead is None or (not s.lead.image_url and item.image_url):
s.lead = item
s.save(update_fields=['size', 'sources', 'first_at', 'last_at', 'lead'])

253
feeds/fetch.py
View file

@ -1,253 +0,0 @@
"""Reading feeds: entries into items, the whole article from its page, finding a
site's feed. Downloading the feed itself is podcasts.views._fetch_feed (ETag,
body hash, redirects), shared with the podcast refresh."""
import calendar
import hashlib
import html
import logging
import re
from datetime import datetime, timezone as dt_timezone
from html.parser import HTMLParser
from urllib.parse import urljoin, urlparse
import requests
from django.utils import timezone
from podcasts.shownotes import _Sanitizer, html_to_text
log = logging.getLogger(__name__)
USER_AGENT = 'Mozilla/5.0 (compatible; diora feed reader; +https://diora.creamfresh.xyz)'
PAGE_TIMEOUT = 15
# Shorter text than this in the feed counts as a teaser: the article is fetched ("auto").
TEASER_CHARS = 700
SUMMARY_CHARS = 400
class _AbsoluteSanitizer(_Sanitizer):
"""The podcast sanitizer, with links and images made absolute first
(article HTML is full of relative ones, which the allowlist would drop)."""
def __init__(self, base):
super().__init__()
self.base = base
def _start(self, tag, attrs, closed):
if tag == 'img':
# Lazy-loaded images keep the real address in data-src, src is a placeholder.
a = dict(attrs)
real = a.get('data-src') or a.get('data-original')
if real:
attrs = [(n, v) for n, v in attrs if n not in ('src', 'data-src', 'data-original')] + [('src', real)]
fixed = []
for name, value in attrs:
if name in ('href', 'src') and value and not value.strip().startswith(('data:', 'javascript:', '#')):
value = urljoin(self.base, value.strip())
fixed.append((name, value))
super()._start(tag, fixed, closed)
def clean_html(text, base=''):
if not text:
return ''
s = _AbsoluteSanitizer(base)
s.feed(text)
return s.result().strip()
def plain(text):
return re.sub(r'\s+', ' ', html_to_text(text or '')).strip()
def _short(text, n=SUMMARY_CHARS):
if len(text) <= n:
return text
cut = text[:n].rsplit(' ', 1)[0]
return cut + ' …'
def _when(entry):
"""The entry's time (published, else updated), never in the future, or None."""
now = timezone.now()
for key in ('published_parsed', 'updated_parsed', 'created_parsed'):
t = entry.get(key)
if t:
try:
when = datetime.fromtimestamp(calendar.timegm(t), dt_timezone.utc)
except (OverflowError, ValueError, TypeError):
continue
return min(when, now)
return None
class _FirstImage(HTMLParser):
def __init__(self):
super().__init__()
self.src = ''
def handle_starttag(self, tag, attrs):
if tag == 'img' and not self.src:
a = dict(attrs)
w, h = a.get('width', ''), a.get('height', '')
if w in ('1', '0') or h in ('1', '0'):
return # tracking pixel
self.src = a.get('src') or ''
def _image(entry, content_html, base):
for m in entry.get('media_thumbnail') or []:
if m.get('url'):
return m['url']
for m in entry.get('media_content') or []:
if m.get('url') and (m.get('medium') == 'image' or (m.get('type') or '').startswith('image/')):
return m['url']
for enc in entry.get('enclosures') or []:
if (enc.get('type') or '').startswith('image/') and enc.get('href'):
return enc['href']
p = _FirstImage()
try:
p.feed(content_html or '')
except Exception:
pass
return urljoin(base, p.src) if p.src else ''
def guid_hash(entry):
key = entry.get('id') or entry.get('link') or f"{entry.get('title', '')}|{entry.get('published', '')}"
return hashlib.sha256(key.encode('utf-8', 'replace')).hexdigest()
def parse_entry(entry, feed_url):
"""A feedparser entry -> the fields of an Item (without source/user)."""
link = entry.get('link') or ''
base = link or feed_url
contents = [c.get('value') or '' for c in entry.get('content') or []]
content_html = max(contents, key=len) if contents else ''
summary_html = entry.get('summary') or ''
if len(plain(summary_html)) > len(plain(content_html)):
content_html = summary_html
content = clean_html(content_html, base)
text = plain(content)
summary = plain(summary_html) or text
title = plain(html.unescape(entry.get('title') or '')) or _short(text, 80) or 'Ohne Titel'
when = _when(entry)
return {
'guid_hash': guid_hash(entry),
'url': link[:2000],
'title': title[:500],
'author': plain(entry.get('author') or '')[:300],
'published': when or timezone.now(),
'dated': when is not None, # not an Item field: refresh._apply uses and drops it
'summary': _short(summary),
'content': content,
'words': word_count(content),
'image_url': _image(entry, content_html, base)[:2000],
}
def word_count(content_html):
return len(plain(content_html).split())
def is_teaser(content_html):
return len(plain(content_html)) < TEASER_CHARS
def _drop_title(html_text):
"""trafilatura repeats the headline as the first <h1>; the reader shows it already."""
return re.sub(r'^\s*(<img[^>]*>\s*)?<h1>.*?</h1>', lambda m: m.group(1) or '', html_text, count=1, flags=re.S)
def fetch_article(url, http=None):
"""The article on `url` as (clean HTML, image URL, paywalled), or None.
trafilatura finds the main text and leaves out navigation, ads, comments.
`http`: a logged-in session for a paywalled site (logins.py); paywalled
says the page showed only the free part anyway."""
import trafilatura # heavy import, only where articles are fetched
from .logins import adapter_for
try:
r = (http or requests).get(url, timeout=PAGE_TIMEOUT, headers={'User-Agent': USER_AGENT})
except requests.RequestException as e:
log.info('article %s: %s', url, type(e).__name__)
return None
if r.status_code >= 400 or 'html' not in r.headers.get('Content-Type', 'text/html'):
return None
page = r.text
adapter = adapter_for(r.url)
paywalled = bool(adapter and adapter.paywalled(page))
try:
body = trafilatura.extract(page, url=r.url, output_format='html', include_images=True,
include_links=True, include_tables=True, favor_precision=True)
except Exception:
log.exception('article %s: extraction failed', url)
return None
if not body:
return None
body = re.sub(r'^\s*<html>\s*<body>|</body>\s*</html>\s*$', '', body.strip())
content = clean_html(_drop_title(body), r.url)
if len(plain(content)) < 200 and not paywalled:
return None
image = ''
try:
meta = trafilatura.extract_metadata(page, default_url=r.url)
image = (meta.image if meta else '') or ''
except Exception:
pass
return content, image, paywalled
# ---- finding a site's feed
FEED_TYPES = ('application/rss+xml', 'application/atom+xml', 'application/feed+json', 'application/xml', 'text/xml')
class _FeedLinks(HTMLParser):
def __init__(self):
super().__init__()
self.feeds = []
self.icon = ''
def handle_starttag(self, tag, attrs):
if tag != 'link':
return
a = {k.lower(): (v or '') for k, v in attrs}
rel = a.get('rel', '').lower().split()
if 'alternate' in rel and a.get('type', '').lower() in FEED_TYPES and a.get('href'):
self.feeds.append((a['href'], a.get('title', '')))
if ('icon' in rel or 'apple-touch-icon' in rel) and a.get('href') and not self.icon:
self.icon = a['href']
def discover(page_html, base):
"""[(feed url, title)] a page links to, and its icon URL."""
p = _FeedLinks()
try:
p.feed(page_html)
except Exception:
pass
seen, out = set(), []
for href, title in p.feeds:
u = urljoin(base, href)
if u not in seen:
seen.add(u)
out.append((u, title))
return out, (urljoin(base, p.icon) if p.icon else '')
def short_title(title):
"""A feed's name without the slogan: "tagesschau.de - die erste Adresse für …"
-> "tagesschau.de", "Aktuell - FAZ.NET" -> "FAZ.NET", "DIE ZEIT | Nachrichten, …" -> "DIE ZEIT"."""
parts = [p.strip() for p in re.split(r'\s+[-–—|:]\s+', title or '') if p.strip()]
if len(parts) < 2:
return (title or '').strip()
branded = [p for p in parts if re.search(r'\w\.[a-z]{2,}$', p, re.I)]
return (branded or parts)[0]
def site_icon(site_url):
if not site_url:
return ''
u = urlparse(site_url)
return f'{u.scheme}://{u.netloc}/favicon.ico' if u.scheme and u.netloc else ''

167
feeds/logins.py
View file

@ -1,167 +0,0 @@
"""Subscriber logins for sites behind a paywall (junge Welt …).
The account holder enters the site's username and password in the feed's
settings; the server logs in with them and fetches the articles as a
subscriber. The password is stored encrypted (Fernet, key derived from
SECRET_KEY — a copy of the database alone doesn't reveal it) and never
handed out again. Each supported site is an adapter: how to log in, how a
paywalled page looks.
"""
import base64
import hashlib
import logging
import threading
import time
from urllib.parse import urlparse
import requests
from cryptography.fernet import Fernet, InvalidToken
from django.conf import settings
from django.utils import timezone
log = logging.getLogger(__name__)
TIMEOUT = 15
SESSION_SECONDS = 6 * 3600 # a logged-in session is reused this long, then logs in again
class LoginFailed(Exception):
pass
class JungeWelt:
domain = 'jungewelt.de'
name = 'junge Welt'
def login(self, http, username, password):
r = http.post('https://www.jungewelt.de/login.php', timeout=TIMEOUT, allow_redirects=False,
data={'username': username, 'password': password, 'ref': '/', 'submit': 'Login'})
where = r.headers.get('Location', '')
if 'loginFailed' in where:
raise LoginFailed('Benutzername oder Passwort falsch')
if r.status_code not in (301, 302, 303) or not http.cookies.get('jW'):
raise LoginFailed(f'Anmeldung fehlgeschlagen ({r.status_code})')
def paywalled(self, page):
return 'mit einem Abonnement und erhalten Sie Zugang' in page
ADAPTERS = {a.domain: a for a in (JungeWelt(),)}
def adapter_for(url):
host = (urlparse(url or '').hostname or '').lower()
for domain, adapter in ADAPTERS.items():
if host == domain or host.endswith('.' + domain):
return adapter
return None
def _fernet():
key = hashlib.sha256(b'diora feeds site login\0' + settings.SECRET_KEY.encode()).digest()
return Fernet(base64.urlsafe_b64encode(key))
def encrypt(text):
return _fernet().encrypt(text.encode()).decode()
def decrypt(token):
"""The plain text, or None when it can't be read (SECRET_KEY changed)."""
try:
return _fernet().decrypt(token.encode()).decode()
except (InvalidToken, ValueError):
return None
def new_http():
from .fetch import USER_AGENT
s = requests.Session()
s.headers['User-Agent'] = USER_AGENT
return s
# Logged-in sessions in this process: (user id, domain) -> (session, logged in at).
_sessions = {}
_lock = threading.Lock()
def forget(user_id, domain):
with _lock:
_sessions.pop((user_id, domain), None)
def check_and_store(user, url, username, password):
"""Log in once; on success store the login (encrypted). Raises LoginFailed."""
from .models import SiteLogin
adapter = adapter_for(url)
if adapter is None:
raise LoginFailed('Für diese Seite gibt es keine Anmeldung.')
http = new_http()
try:
adapter.login(http, username, password)
except requests.RequestException:
raise LoginFailed(f'{adapter.name} ist nicht erreichbar.')
SiteLogin.objects.update_or_create(user=user, domain=adapter.domain, defaults={
'username': username, 'secret': encrypt(password), 'ok': True, 'last_error': '',
'checked_at': timezone.now()})
with _lock:
_sessions[(user.id, adapter.domain)] = (http, time.monotonic())
return adapter
def session_for(user_id, url, fresh=False):
"""A logged-in requests session for the article's site, or None (no login
stored, or it no longer works — then the login is marked as failing)."""
from .models import SiteLogin
adapter = adapter_for(url)
if adapter is None:
return None
key = (user_id, adapter.domain)
with _lock:
cached = _sessions.get(key)
if cached and not fresh and time.monotonic() - cached[1] < SESSION_SECONDS:
return cached[0]
login = SiteLogin.objects.filter(user_id=user_id, domain=adapter.domain).first()
if login is None:
return None
password = decrypt(login.secret)
if password is None:
login.ok, login.last_error = False, 'Bitte die Zugangsdaten neu eingeben.'
login.save(update_fields=['ok', 'last_error'])
return None
http = new_http()
try:
adapter.login(http, login.username, password)
except (LoginFailed, requests.RequestException) as e:
login.ok = False
login.last_error = str(e) if isinstance(e, LoginFailed) else f'{adapter.name} nicht erreichbar'
login.checked_at = timezone.now()
login.save(update_fields=['ok', 'last_error', 'checked_at'])
log.info('site login %s for user %s failed: %s', adapter.domain, user_id, e)
return None
if not login.ok:
login.ok, login.last_error = True, ''
login.save(update_fields=['ok', 'last_error'])
with _lock:
_sessions[key] = (http, time.monotonic())
return http
def has_login(user_id, url):
from .models import SiteLogin
adapter = adapter_for(url)
return bool(adapter) and SiteLogin.objects.filter(user_id=user_id, domain=adapter.domain, ok=True).exists()
def info(user_id, url):
"""What the feed settings show about the login for this source's site."""
from .models import SiteLogin
adapter = adapter_for(url)
if adapter is None:
return None
login = SiteLogin.objects.filter(user_id=user_id, domain=adapter.domain).first()
return {'site': adapter.name, 'domain': adapter.domain,
'username': login.username if login else '', 'ok': bool(login and login.ok),
'error': login.last_error if login else ''}

0
feeds/management/__init__.py
View file

0
feeds/management/commands/__init__.py
View file

25
feeds/management/commands/refresh_newsfeeds.py
View file

@ -1,25 +0,0 @@
from django.core.management.base import BaseCommand
from django.utils import timezone
from feeds.models import Source
from feeds.refresh import refresh
from podcasts.views import _in_backoff_q
class Command(BaseCommand):
help = 'Refresh the feed reader\'s sources (news and blogs)'
def add_arguments(self, parser):
parser.add_argument('--source-id', type=int, help='Refresh only this source (even when backing off)')
def handle(self, *args, **options):
qs = Source.objects.filter(synthetic=False).select_related('user').order_by('last_refreshed_at')
if options['source_id']:
qs = qs.filter(pk=options['source_id'])
else:
qs = qs.exclude(_in_backoff_q(timezone.now()))
if not qs.exists():
self.stdout.write('No sources to refresh.')
return
refreshed, failed, new = refresh(qs)
self.stdout.write(f'{refreshed} sources refreshed, {failed} failed, +{new} items')

111
feeds/migrations/0001_initial.py
View file

@ -1,111 +0,0 @@
# Generated by Django 6.1 on 2026-10-04 17:54
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='FeedSettings',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('news_seen_at', models.DateTimeField(blank=True, null=True)),
('news_keep_days', models.PositiveIntegerField(default=2)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='feed_settings', to=settings.AUTH_USER_MODEL)),
],
),
migrations.CreateModel(
name='Source',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.URLField(max_length=1000)),
('title', models.CharField(max_length=300)),
('site_url', models.URLField(blank=True, max_length=1000)),
('icon_url', models.URLField(blank=True, max_length=1000)),
('kind', models.CharField(choices=[('news', 'Nachrichten'), ('blog', 'Blog')], default='news', max_length=10)),
('category', models.CharField(blank=True, max_length=100)),
('fulltext', models.CharField(choices=[('auto', 'Automatisch'), ('always', 'Immer'), ('never', 'Nie')], default='auto', max_length=10)),
('added_at', models.DateTimeField(auto_now_add=True)),
('last_refreshed_at', models.DateTimeField(blank=True, null=True)),
('http_etag', models.CharField(blank=True, max_length=500)),
('http_last_modified', models.CharField(blank=True, max_length=100)),
('http_body_hash', models.CharField(blank=True, max_length=64)),
('last_error', models.CharField(blank=True, max_length=500)),
('last_error_at', models.DateTimeField(blank=True, null=True)),
('error_count', models.PositiveIntegerField(default=0)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_sources', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['title'],
'unique_together': {('user', 'url')},
},
),
migrations.CreateModel(
name='Item',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('kind', models.CharField(choices=[('news', 'Nachrichten'), ('blog', 'Blog')], max_length=10)),
('guid_hash', models.CharField(max_length=64)),
('url', models.URLField(blank=True, max_length=2000)),
('title', models.CharField(max_length=500)),
('author', models.CharField(blank=True, max_length=300)),
('published', models.DateTimeField(db_index=True)),
('fetched_at', models.DateTimeField(auto_now_add=True)),
('summary', models.TextField(blank=True)),
('content', models.TextField(blank=True)),
('content_full', models.BooleanField(default=False)),
('fulltext_failed', models.BooleanField(default=False)),
('image_url', models.URLField(blank=True, max_length=2000)),
('read_at', models.DateTimeField(blank=True, null=True)),
('starred', models.BooleanField(default=False)),
('gone', models.BooleanField(default=False)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_items', to=settings.AUTH_USER_MODEL)),
('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='feeds.source')),
],
options={
'ordering': ['-published'],
},
),
migrations.CreateModel(
name='Story',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('first_at', models.DateTimeField()),
('last_at', models.DateTimeField(db_index=True)),
('size', models.PositiveIntegerField(default=1)),
('sources', models.PositiveIntegerField(default=1)),
('terms', models.TextField(blank=True)),
('lead', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='feeds.item')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_stories', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['-last_at'],
},
),
migrations.AddField(
model_name='item',
name='story',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='items', to='feeds.story'),
),
migrations.AddIndex(
model_name='item',
index=models.Index(fields=['user', 'kind', 'published'], name='feeds_item_user_id_1a8bce_idx'),
),
migrations.AddIndex(
model_name='item',
index=models.Index(fields=['user', 'kind', 'read_at'], name='feeds_item_user_id_0007dd_idx'),
),
migrations.AlterUniqueTogether(
name='item',
unique_together={('source', 'guid_hash')},
),
]

37
feeds/migrations/0002_item_paywalled_sitelogin.py
View file

@ -1,37 +0,0 @@
# Generated by Django 6.1 on 2026-10-04 19:22
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feeds', '0001_initial'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddField(
model_name='item',
name='paywalled',
field=models.BooleanField(default=False),
),
migrations.CreateModel(
name='SiteLogin',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('domain', models.CharField(max_length=200)),
('username', models.CharField(max_length=300)),
('secret', models.TextField()),
('ok', models.BooleanField(default=True)),
('last_error', models.CharField(blank=True, max_length=300)),
('checked_at', models.DateTimeField(blank=True, null=True)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_site_logins', to=settings.AUTH_USER_MODEL)),
],
options={
'unique_together': {('user', 'domain')},
},
),
]

42
feeds/migrations/0003_blog_flags_words.py
View file

@ -1,42 +0,0 @@
# Generated by Django 6.1 on 2026-10-04 19:28
from django.db import migrations, models
def count_words(apps, schema_editor):
"""The reading time of the articles already there."""
from feeds.fetch import word_count
Item = apps.get_model('feeds', 'Item')
for item in Item.objects.exclude(content='').only('id', 'content').iterator():
Item.objects.filter(pk=item.pk).update(words=word_count(item.content))
class Migration(migrations.Migration):
dependencies = [
('feeds', '0002_item_paywalled_sitelogin'),
]
operations = [
migrations.AddField(
model_name='item',
name='words',
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name='source',
name='fav',
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name='source',
name='hidden',
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name='source',
name='notify',
field=models.BooleanField(default=False),
),
migrations.RunPython(count_words, migrations.RunPython.noop),
]

18
feeds/migrations/0004_item_notify.py
View file

@ -1,18 +0,0 @@
# Generated by Django 6.1 on 2026-10-04 19:33
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feeds', '0003_blog_flags_words'),
]
operations = [
migrations.AddField(
model_name='item',
name='notify',
field=models.BooleanField(db_index=True, default=False),
),
]

46
feeds/migrations/0005_blog_folders_progress.py
View file

@ -1,46 +0,0 @@
# Generated by Django 6.1 on 2026-10-04 19:50
from django.db import migrations, models
def category_to_folders(apps, schema_editor):
"""A blog's folder was its category: now its first (only) folder."""
Source = apps.get_model('feeds', 'Source')
for s in Source.objects.filter(kind='blog').exclude(category=''):
s.folders = [s.category[:40]]
s.category = ''
s.save(update_fields=['folders', 'category'])
def folders_to_category(apps, schema_editor):
Source = apps.get_model('feeds', 'Source')
for s in Source.objects.filter(kind='blog'):
if s.folders:
s.category = s.folders[0]
s.save(update_fields=['category'])
class Migration(migrations.Migration):
dependencies = [
('feeds', '0004_item_notify'),
]
operations = [
migrations.AddField(
model_name='item',
name='progress',
field=models.FloatField(default=0),
),
migrations.AddField(
model_name='item',
name='progress_at',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name='source',
name='folders',
field=models.JSONField(blank=True, default=list),
),
migrations.RunPython(category_to_folders, folders_to_category),
]

18
feeds/migrations/0006_folder_order.py
View file

@ -1,18 +0,0 @@
# Generated by Django 6.1 on 2026-10-16 18:21
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feeds', '0005_blog_folders_progress'),
]
operations = [
migrations.AddField(
model_name='feedsettings',
name='folder_order',
field=models.JSONField(blank=True, default=list),
),
]

18
feeds/migrations/0007_transcription.py
View file

@ -1,18 +0,0 @@
# Generated by Django 6.1 on 2026-11-09 08:20
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feeds', '0006_folder_order'),
]
operations = [
migrations.AddField(
model_name='source',
name='synthetic',
field=models.BooleanField(default=False),
),
]

0
feeds/migrations/__init__.py
View file

143
feeds/models.py
View file

@ -1,143 +0,0 @@
from django.contrib.auth.models import User
from django.db import models
class Source(models.Model):
"""A subscribed feed. Every source is either news or a blog, and that shapes
everything after it: news gets bundled into stories, shows up in the
"Seit letztem Mal" briefing and expires after a few days; blog articles stay
until read and are listed per blog."""
NEWS = 'news'
BLOG = 'blog'
KINDS = [(NEWS, 'Nachrichten'), (BLOG, 'Blog')]
# Whether the whole article is fetched from its page: "auto" when the feed
# only carries a teaser.
FULLTEXT_AUTO = 'auto'
FULLTEXT_ALWAYS = 'always'
FULLTEXT_NEVER = 'never'
FULLTEXTS = [(FULLTEXT_AUTO, 'Automatisch'), (FULLTEXT_ALWAYS, 'Immer'), (FULLTEXT_NEVER, 'Nie')]
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_sources')
url = models.URLField(max_length=1000)
title = models.CharField(max_length=300)
site_url = models.URLField(max_length=1000, blank=True)
icon_url = models.URLField(max_length=1000, blank=True)
kind = models.CharField(max_length=10, choices=KINDS, default=NEWS)
# News: the section it belongs to (Politik, Tech …), groups the briefing.
category = models.CharField(max_length=100, blank=True)
# Blogs: the folders it's in (labels, any number; a folder exists while a blog carries it).
folders = models.JSONField(default=list, blank=True)
fulltext = models.CharField(max_length=10, choices=FULLTEXTS, default=FULLTEXT_AUTO)
added_at = models.DateTimeField(auto_now_add=True)
# Blogs: favorites come first; a hidden blog stays out of "Alle" (and its
# counts) but shows in its folder; notify = a notification for new posts.
fav = models.BooleanField(default=False)
hidden = models.BooleanField(default=False)
notify = models.BooleanField(default=False)
# Filled by diora itself (the blog "Podcast-Artikel"), not by a feed: never
# fetched, pruned, given another URL or turned into news.
synthetic = models.BooleanField(default=False)
# Refresh bookkeeping, as for podcast feeds (see podcasts.views._fetch_feed).
last_refreshed_at = models.DateTimeField(null=True, blank=True)
http_etag = models.CharField(max_length=500, blank=True)
http_last_modified = models.CharField(max_length=100, blank=True)
http_body_hash = models.CharField(max_length=64, blank=True)
last_error = models.CharField(max_length=500, blank=True)
last_error_at = models.DateTimeField(null=True, blank=True)
error_count = models.PositiveIntegerField(default=0)
class Meta:
unique_together = ('user', 'url')
ordering = ['title']
def __str__(self):
return f'{self.title} ({self.kind}, user={self.user_id})'
class Story(models.Model):
"""News items about the same event from one or more sources (see cluster.py)."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_stories')
# The item the story is shown with (the first one, or the longest text).
lead = models.ForeignKey('Item', null=True, blank=True, on_delete=models.SET_NULL, related_name='+')
first_at = models.DateTimeField()
last_at = models.DateTimeField(db_index=True)
size = models.PositiveIntegerField(default=1) # items in it
sources = models.PositiveIntegerField(default=1) # distinct sources among them
# Words that describe it (space-separated, for matching new items).
terms = models.TextField(blank=True)
class Meta:
ordering = ['-last_at']
class Item(models.Model):
source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name='items')
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_items')
kind = models.CharField(max_length=10, choices=Source.KINDS) # the source's, at the time
guid_hash = models.CharField(max_length=64) # sha256 of the entry's id (or link)
url = models.URLField(max_length=2000, blank=True)
title = models.CharField(max_length=500)
author = models.CharField(max_length=300, blank=True)
published = models.DateTimeField(db_index=True)
fetched_at = models.DateTimeField(auto_now_add=True)
summary = models.TextField(blank=True) # plain text, short
content = models.TextField(blank=True) # sanitized HTML
# The content came from the article's page, not the feed.
content_full = models.BooleanField(default=False)
# Fetching the page failed or gave nothing: don't try on every open.
fulltext_failed = models.BooleanField(default=False)
# The page showed only the free part (paywall): fetched again once there's
# a subscriber login for the site (logins.py).
paywalled = models.BooleanField(default=False)
image_url = models.URLField(max_length=2000, blank=True)
words = models.PositiveIntegerField(default=0) # of the content: the reading time
# To be announced (a blog with the bell on): set when the item came in, not on a
# source's first fetch and not for posts older than refresh.NOTIFY_MAX_AGE.
notify = models.BooleanField(default=False, db_index=True)
read_at = models.DateTimeField(null=True, blank=True)
# How far it was read (0-1, the article's scroll position), for every device;
# from READ_AT on it counts as read (views.item_progress).
progress = models.FloatField(default=0)
progress_at = models.DateTimeField(null=True, blank=True)
starred = models.BooleanField(default=False)
story = models.ForeignKey(Story, null=True, blank=True, on_delete=models.SET_NULL, related_name='items')
# Expired news: the row stays a while without its text, so the entry isn't
# taken in again as new while the feed still carries it (refresh.expire).
gone = models.BooleanField(default=False)
class Meta:
unique_together = ('source', 'guid_hash')
indexes = [
models.Index(fields=['user', 'kind', 'published']),
models.Index(fields=['user', 'kind', 'read_at']),
]
ordering = ['-published']
def __str__(self):
return self.title
class SiteLogin(models.Model):
"""A subscriber login for a paywalled site (logins.py); the password encrypted."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_site_logins')
domain = models.CharField(max_length=200)
username = models.CharField(max_length=300)
secret = models.TextField() # the password, Fernet-encrypted
ok = models.BooleanField(default=True)
last_error = models.CharField(max_length=300, blank=True)
checked_at = models.DateTimeField(null=True, blank=True)
class Meta:
unique_together = ('user', 'domain')
class FeedSettings(models.Model):
user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='feed_settings')
# The last time the news briefing was seen; what came after is "Seit letztem Mal".
news_seen_at = models.DateTimeField(null=True, blank=True)
# News older than this goes (unless starred).
news_keep_days = models.PositiveIntegerField(default=2)
# The order of the blog folders (their chips), names (docs/library.md).
folder_order = models.JSONField(default=list, blank=True)

278
feeds/refresh.py
View file

@ -1,278 +0,0 @@
"""Refreshing sources: new entries become items, teasers get their article,
news is bundled into stories, old news expires."""
import logging
import time
from collections import Counter
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
from datetime import timedelta
from django.db import IntegrityError, transaction
from django.utils import timezone
from podcasts.views import FeedFetchError, _clear_error, _error_text, _fetch_feed
from . import fetch, logins
from .cluster import Clusterer
from .models import FeedSettings, Item, Source, Story
log = logging.getLogger(__name__)
FEED_WORKERS = 12
ARTICLE_WORKERS = 8
ARTICLES_PER_RUN = 80 # articles fetched per refresh at most; the rest when opened
BLOG_KEEP = 1000 # items kept per blog
BLOG_FIRST_UNREAD_DAYS = 7 # a new blog's older articles start out read
UNDATED_FIRST_UNREAD = 3 # of a new feed without dates, only the first few
GONE_KEEP_DAYS = 30 # expired news rows kept (without text) against re-adding
NOTIFY_MAX_AGE = timedelta(days=2) # older posts are never announced (feeds dig up old ones)
def settings_for(user):
s, _ = FeedSettings.objects.get_or_create(user=user)
return s
def _apply(source, result, now, keep_days):
"""Write one download; returns the new items."""
if result.status == 304:
source.last_refreshed_at = now
_clear_error(source)
source.save(update_fields=['last_refreshed_at', 'last_error', 'last_error_at', 'error_count',
'http_etag', 'http_last_modified'])
return []
parsed = result.parsed
feed = parsed.feed
if feed.get('title') and (not source.title or source.title == source.url):
source.title = fetch.short_title(fetch.plain(feed.title))[:300]
if feed.get('link'):
source.site_url = feed.link[:1000]
if not source.icon_url:
icon = (feed.get('image') or {}).get('href') or feed.get('icon') or fetch.site_icon(source.site_url)
source.icon_url = (icon or '')[:1000]
if result.permanent_url and result.permanent_url != source.url and \
not Source.objects.filter(user=source.user, url=result.permanent_url).exists():
source.url = result.permanent_url[:1000]
source.http_etag = result.etag
source.http_last_modified = result.last_modified
source.http_body_hash = result.body_hash
source.last_refreshed_at = now
_clear_error(source)
first = not source.items.exists()
source.save()
known = set(source.items.values_list('guid_hash', flat=True))
oldest_kept = None
if source.kind == Source.BLOG and len(known) >= BLOG_KEEP:
oldest_kept = source.items.order_by('published').values_list('published', flat=True).first()
news_cutoff = now - timedelta(days=keep_days)
new = []
entries = [fetch.parse_entry(e, source.url) for e in parsed.entries]
# Some feeds (Deutschlandfunk) give every entry the time the feed was made:
# then the dates say nothing, and the time an entry first shows up counts.
stamps = Counter(f['published'] for f in entries if f['dated'])
bogus_dates = len(entries) >= 5 and bool(stamps) and stamps.most_common(1)[0][1] >= 0.8 * len(entries)
for n, fields in enumerate(entries):
dated = fields.pop('dated')
undated = bogus_dates or not dated
if undated:
# First time: the feed's order stands for the time (newest first), so the
# list doesn't fill up with "just now"; afterwards, when it showed up.
fields['published'] = now - timedelta(minutes=n) if first else now
fields['undated'] = first and n >= UNDATED_FIRST_UNREAD
if fields['guid_hash'] in known:
continue
known.add(fields['guid_hash'])
if source.kind == Source.NEWS and fields['published'] < news_cutoff:
continue
if oldest_kept and fields['published'] < oldest_kept:
continue
old_undated = fields.pop('undated', False)
item = Item(source=source, user=source.user, kind=source.kind, **fields)
if source.kind == Source.BLOG and first and (
old_undated or fields['published'] < now - timedelta(days=BLOG_FIRST_UNREAD_DAYS)):
item.read_at = now
# The bell: never for what a first fetch brings, never for old posts.
item.notify = source.notify and not first and not undated and fields['published'] >= now - NOTIFY_MAX_AGE
new.append(item)
if new:
try:
with transaction.atomic():
Item.objects.bulk_create(new)
except IntegrityError:
# Two entries of one feed with the same id: one by one, skipping the doubles.
kept = []
for item in new:
try:
with transaction.atomic():
item.save()
kept.append(item)
except IntegrityError:
pass
new = kept
return new
def needs_article(item, source):
if not item.url or item.content_full or item.fulltext_failed:
return False
if source.fulltext == Source.FULLTEXT_NEVER:
return False
if item.paywalled:
# Only the free part came: again once there's a working subscriber login.
return logins.has_login(item.user_id, item.url)
return source.fulltext == Source.FULLTEXT_ALWAYS or fetch.is_teaser(item.content)
def load_article(item):
"""Fetch the article for one item now (when opened); saves it."""
http = logins.session_for(item.user_id, item.url)
got = fetch.fetch_article(item.url, http)
if got and got[2] and http:
# Logged in, still the paywall: the session ran out, log in anew once.
http = logins.session_for(item.user_id, item.url, fresh=True)
if http:
got = fetch.fetch_article(item.url, http)
_store_article(item, got)
return item
def _store_article(item, got):
if got is None:
item.fulltext_failed = True
item.save(update_fields=['fulltext_failed'])
return
content, image, paywalled = got
longer = len(fetch.plain(content)) > len(fetch.plain(item.content))
if paywalled:
# The free part, if it's more than the feed's teaser; the rest waits for a login.
item.paywalled = True
if longer:
item.content = content
item.words = fetch.word_count(content)
if not item.image_url and image:
item.image_url = image[:2000]
item.save(update_fields=['paywalled', 'content', 'words', 'image_url'])
return
if not longer:
item.fulltext_failed = True # the feed had more
item.save(update_fields=['fulltext_failed'])
return
item.content = content
item.words = fetch.word_count(content)
item.content_full = True
item.paywalled = False
if not item.image_url and image:
item.image_url = image[:2000]
item.save(update_fields=['content', 'words', 'content_full', 'paywalled', 'image_url'])
def _articles(items, deadline):
todo = [i for i in items if needs_article(i, i.source)][:ARTICLES_PER_RUN]
if not todo:
return
# Logged-in sessions for paywalled sites, one per account and site (made here, not in the threads).
sessions = {}
for i in todo:
adapter = logins.adapter_for(i.url)
if adapter and (i.user_id, adapter.domain) not in sessions:
sessions[(i.user_id, adapter.domain)] = logins.session_for(i.user_id, i.url)
def http_for(i):
adapter = logins.adapter_for(i.url)
return sessions.get((i.user_id, adapter.domain)) if adapter else None
pool = ThreadPoolExecutor(ARTICLE_WORKERS)
futures = {pool.submit(fetch.fetch_article, i.url, http_for(i)): i for i in todo}
pending = set(futures)
try:
while pending:
left = None if deadline is None else deadline - time.monotonic()
if left is not None and left <= 0:
break
done, pending = wait(pending, timeout=left, return_when=FIRST_COMPLETED)
for f in done:
i = futures[f]
try:
got = f.result()
if got and got[2] and http_for(i) is not None:
# Logged in and still the paywall: next time with a fresh login.
logins.forget(i.user_id, logins.adapter_for(i.url).domain)
_store_article(i, got)
except Exception:
log.exception('article for item %s', i.pk)
finally:
pool.shutdown(wait=False, cancel_futures=True)
def expire(user, now=None):
"""News past the user's keep time loses its text (rows stay GONE_KEEP_DAYS
as markers), stories without visible items go."""
now = now or timezone.now()
keep = settings_for(user).news_keep_days
Item.objects.filter(user=user, kind=Source.NEWS, gone=False, starred=False,
published__lt=now - timedelta(days=keep)).update(
gone=True, content='', summary='', story=None)
Item.objects.filter(user=user, gone=True, published__lt=now - timedelta(days=GONE_KEEP_DAYS)).delete()
Story.objects.filter(user=user).exclude(items__gone=False).delete()
# Blogs: the oldest read articles beyond BLOG_KEEP per blog.
for source in Source.objects.filter(user=user, kind=Source.BLOG, synthetic=False):
ids = list(source.items.order_by('-published').values_list('id', flat=True)[BLOG_KEEP:])
if ids:
Item.objects.filter(id__in=ids, starred=False).exclude(read_at=None).delete()
def refresh(sources, deadline=None):
"""Refresh `sources` (downloads in parallel, writes one by one).
Returns (refreshed, failed, new items)."""
sources = [s for s in sources if not s.synthetic] # synthetic sources have no feed to fetch
now = timezone.now()
keep = {}
refreshed = failed = 0
new_items = []
pool = ThreadPoolExecutor(FEED_WORKERS)
try:
futures = {
pool.submit(_fetch_feed, s.url, s.http_etag, s.http_last_modified, s.http_body_hash,
user_agent=fetch.USER_AGENT): s
for s in sources
}
try:
timeout = None if deadline is None else max(0.0, deadline - time.monotonic())
for f in as_completed(futures, timeout=timeout):
s = futures[f]
try:
result = f.result()
if s.user_id not in keep:
keep[s.user_id] = settings_for(s.user).news_keep_days
new_items += _apply(s, result, now, keep[s.user_id])
refreshed += 1
except Exception as e:
if not isinstance(e, FeedFetchError):
log.exception('feed source %s', s.pk)
s.last_error = _error_text(e)
s.last_error_at = now
s.error_count += 1
s.save(update_fields=['last_error', 'last_error_at', 'error_count'])
failed += 1
except TimeoutError:
pass
finally:
# Past the deadline: don't wait for the slow ones (they stay as they are).
pool.shutdown(wait=False, cancel_futures=True)
for item in new_items:
item.source = next(s for s in sources if s.pk == item.source_id)
_articles(new_items, deadline)
by_user = {}
for item in new_items:
if item.kind == Source.NEWS:
by_user.setdefault(item.user_id, []).append(item)
for user_id, items in by_user.items():
c = Clusterer(items[0].user, now)
for item in sorted(items, key=lambda i: i.published):
c.add(item)
for user_id in {s.user_id for s in sources}:
expire(next(s.user for s in sources if s.user_id == user_id), now)
return refreshed, failed, len(new_items)

594
feeds/tests.py
View file

@ -1,594 +0,0 @@
import json
from datetime import timedelta
from unittest import mock
import feedparser
from django.contrib.auth.models import User
from django.core.cache import cache
from django.test import TestCase
from django.utils import timezone
from podcasts.views import FeedFetch
from . import cluster, fetch, logins, refresh, textproc
from .models import FeedSettings, Item, SiteLogin, Source, Story
def rss(items, title='Testfeed', link='https://example.org/'):
body = ''.join(
f'<item><guid>{i["guid"]}</guid><title>{i["title"]}</title><link>{i.get("link", "https://example.org/" + i["guid"])}</link>'
f'<pubDate>{i.get("date", timezone.now().strftime("%a, %d %b %Y %H:%M:%S +0000"))}</pubDate>'
f'<description><![CDATA[{i.get("text", "")}]]></description></item>'
for i in items)
return f'<?xml version="1.0"?><rss version="2.0"><channel><title>{title}</title><link>{link}</link>{body}</channel></rss>'.encode()
def fetched(content, url='https://example.org/feed'):
return FeedFetch(200, feedparser.parse(content), 'etag', '', url, url, 'hash')
LONG = '<p>' + 'Ein langer Absatz mit sehr viel Text. ' * 40 + '</p>'
class FetchTest(TestCase):
def test_relative_links_become_absolute_and_scripts_go(self):
out = fetch.clean_html('<p>Hi <a href="/a">x</a><img src="i.jpg"><script>bad()</script>'
'<img data-src="/real.jpg" src="placeholder.gif"></p>', 'https://ex.org/post/1')
self.assertIn('href="https://ex.org/a"', out)
self.assertIn('src="https://ex.org/post/i.jpg"', out)
self.assertIn('src="https://ex.org/real.jpg"', out)
self.assertNotIn('placeholder', out)
self.assertNotIn('bad()', out)
def test_parse_entry(self):
e = feedparser.parse(rss([{'guid': 'g1', 'title': 'A &amp; B', 'text': '<p>Kurz <img src="/p.jpg"></p>'}])).entries[0]
f = fetch.parse_entry(e, 'https://example.org/feed')
self.assertEqual(f['title'], 'A & B')
self.assertEqual(f['image_url'], 'https://example.org/p.jpg')
self.assertTrue(fetch.is_teaser(f['content']))
def test_drop_repeated_headline(self):
self.assertEqual(fetch._drop_title('<img src="x"><h1>Titel</h1><p>Text</p>'), '<img src="x"><p>Text</p>')
def test_short_title(self):
self.assertEqual(fetch.short_title('tagesschau.de - die erste Adresse für Nachrichten'), 'tagesschau.de')
self.assertEqual(fetch.short_title('Aktuell - FAZ.NET'), 'FAZ.NET')
self.assertEqual(fetch.short_title('DIE ZEIT | Nachrichten, News, Hintergründe'), 'DIE ZEIT')
self.assertEqual(fetch.short_title('Golem.de'), 'Golem.de')
self.assertEqual(fetch.short_title('Kotlin-Blog'), 'Kotlin-Blog')
def test_discover(self):
feeds, icon = fetch.discover('<link rel="alternate" type="application/rss+xml" href="/feed" title="F">'
'<link rel="icon" href="/fav.png">', 'https://ex.org/')
self.assertEqual(feeds, [('https://ex.org/feed', 'F')])
self.assertEqual(icon, 'https://ex.org/fav.png')
class ClusterTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('u', password='pw12345678')
self.a = Source.objects.create(user=self.u, url='https://a/', title='A')
self.b = Source.objects.create(user=self.u, url='https://b/', title='B')
self.c = Source.objects.create(user=self.u, url='https://c/', title='C')
def item(self, source, title, summary=''):
return Item.objects.create(source=source, user=self.u, kind='news', guid_hash=title[:60],
title=title, summary=summary, published=timezone.now())
def test_same_event_bundled_other_apart(self):
c = cluster.Clusterer(self.u, timezone.now())
s1 = c.add(self.item(self.a, 'Oktoberfest endet mit neuem Besucherrekord in München'))
s2 = c.add(self.item(self.b, 'Rekord-Wiesn: 7,4 Millionen Besucher auf dem Oktoberfest in München'))
s3 = c.add(self.item(self.c, 'Bahn führt Familienticket dauerhaft ein'))
self.assertEqual(s1, s2)
self.assertNotEqual(s1, s3)
s1.refresh_from_db()
self.assertEqual((s1.size, s1.sources), (2, 2))
def test_same_source_needs_to_be_very_close(self):
c = cluster.Clusterer(self.u, timezone.now())
s1 = c.add(self.item(self.a, 'Nations League: Pressekonferenz vor Irland gegen Israel abgebrochen'))
s2 = c.add(self.item(self.a, 'Nations League: England schlägt Kroatien deutlich'))
self.assertNotEqual(s1, s2)
class ApiTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('tester', password='pw12345678')
self.client.force_login(self.u)
def post(self, url, data=None):
return self.client.post(url, json.dumps(data or {}), content_type='application/json')
def add(self, kind, feed, url='https://example.org/feed'):
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed, url)), \
mock.patch('feeds.fetch.fetch_article', return_value=(LONG, 'https://example.org/img.jpg', False)):
r = self.post('/feeds/sources/add/', {'url': url, 'kind': kind, 'category': 'Politik'})
self.assertEqual(r.status_code, 200)
return r.json()['source']
def test_requires_login(self):
self.client.logout()
self.assertEqual(self.client.get('/feeds/state/').status_code, 401)
def test_add_news_fetches_articles_and_bundles(self):
s = self.add('news', rss([{'guid': 'a', 'title': 'Merz besucht Kiew und sagt Hilfe zu', 'text': 'Kurz.'},
{'guid': 'b', 'title': 'Bahn Familienticket dauerhaft', 'text': 'Kurz.'}]))
self.assertEqual(s['title'], 'Testfeed')
self.assertEqual(s['unread'], 2)
items = Item.objects.filter(user=self.u)
self.assertTrue(all(i.content_full for i in items))
self.assertEqual(Story.objects.filter(user=self.u).count(), 2)
news = self.client.get('/feeds/news/').json()
self.assertEqual(len(news['stories']), 2)
self.assertEqual(news['stories'][0]['category'], 'Politik')
b = self.client.get('/feeds/briefing/').json()
self.assertEqual(b['total'], 2)
self.assertEqual(b['sections'][0]['category'], 'Politik')
self.post('/feeds/briefing/seen/')
self.assertFalse(Item.objects.filter(user=self.u, read_at=None).exists())
self.assertEqual(self.client.get('/feeds/briefing/').json()['total'], 0)
def test_blog_keeps_full_feed_text_and_lists(self):
self.add('blog', rss([{'guid': 'p1', 'title': 'Ein Artikel', 'text': LONG}]))
item = Item.objects.get(user=self.u)
self.assertFalse(item.content_full) # the feed had the whole text
self.assertIsNone(item.story)
blogs = self.client.get('/feeds/blogs/?unread=1').json()
self.assertEqual([i['title'] for i in blogs['items']], ['Ein Artikel'])
detail = self.client.get(f'/feeds/items/{item.id}/').json()['item']
self.assertIn('langer Absatz', detail['content'])
self.post('/feeds/items/read/', {'ids': [item.id]})
self.assertEqual(self.client.get('/feeds/blogs/?unread=1').json()['items'], [])
self.post(f'/feeds/items/{item.id}/star/', {'starred': True})
self.assertEqual(len(self.client.get('/feeds/starred/').json()['items']), 1)
def test_blog_folders(self):
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'x', 'title': 'Essay', 'text': LONG}]))):
self.post('/feeds/sources/add/', {'url': 'https://a.example/feed', 'kind': 'blog', 'category': 'Philosophie'})
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'y', 'title': 'Code', 'text': LONG}]))):
self.post('/feeds/sources/add/', {'url': 'https://b.example/feed', 'kind': 'blog', 'category': ''})
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Philosophie'])
self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=Philosophie').json()['items']], ['Essay'])
self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=').json()['items']], ['Code'])
self.assertIn('text="Blogs – Philosophie"', self.client.get('/feeds/export.opml').content.decode())
def test_old_news_expires_and_is_not_taken_in_again(self):
old = (timezone.now() - timedelta(days=3)).strftime('%a, %d %b %Y %H:%M:%S +0000')
feed = rss([{'guid': 'new', 'title': 'Heute passiert', 'text': LONG},
{'guid': 'old', 'title': 'Vorgestern passiert', 'text': LONG, 'date': old}])
src = self.add('news', feed)
self.assertEqual(Item.objects.filter(user=self.u).count(), 1) # too old to take in
Item.objects.filter(user=self.u).update(published=timezone.now() - timedelta(days=3))
refresh.expire(self.u)
self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0)
self.assertEqual(Story.objects.filter(user=self.u).count(), 0)
source = Source.objects.get(pk=src['id'])
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed)):
refresh.refresh([source])
self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0)
def test_feed_wide_timestamp_means_first_seen(self):
stamp = (timezone.now() - timedelta(hours=5)).strftime('%a, %d %b %Y %H:%M:%S +0000')
self.add('news', rss([{'guid': f'g{n}', 'title': f'Meldung {n}', 'text': LONG, 'date': stamp} for n in range(6)]))
for i in Item.objects.filter(user=self.u):
self.assertLess((timezone.now() - i.published).total_seconds(), 600) # first fetch: feed order, minutes apart
def test_new_blog_without_dates_keeps_order_and_few_unread(self):
feed = (b'<?xml version="1.0"?><rss version="2.0"><channel><title>Undatiert</title>'
+ b''.join(b'<item><guid>u%d</guid><title>Post %d</title><description>x</description></item>' % (n, n)
for n in range(6)) + b'</channel></rss>')
self.add('blog', feed)
items = list(Item.objects.filter(user=self.u).order_by('-published'))
self.assertEqual([i.title for i in items], [f'Post {n}' for n in range(6)])
self.assertEqual(sum(1 for i in items if i.read_at is None), 3)
def test_switching_kind_moves_items(self):
src = self.add('news', rss([{'guid': 'a', 'title': 'Etwas', 'text': LONG}]))
self.post(f'/feeds/sources/{src["id"]}/', {'kind': 'blog'})
item = Item.objects.get(user=self.u)
self.assertEqual((item.kind, item.story), ('blog', None))
self.assertEqual(Story.objects.filter(user=self.u).count(), 0)
def test_other_users_items_invisible(self):
self.add('blog', rss([{'guid': 'p1', 'title': 'Privat', 'text': LONG}]))
item = Item.objects.get(user=self.u)
other = User.objects.create_user('o', password='pw12345678')
self.client.force_login(other)
self.assertEqual(self.client.get(f'/feeds/items/{item.id}/').status_code, 404)
self.assertEqual(self.client.get('/feeds/blogs/').json()['items'], [])
self.post('/feeds/items/read/', {'ids': [item.id]})
item.refresh_from_db()
self.assertIsNone(item.read_at)
def test_opml_export(self):
self.add('blog', rss([]), url='https://blog.example/feed')
r = self.client.get('/feeds/export.opml')
self.assertIn(b'xmlUrl="https://blog.example/feed"', r.content)
self.assertIn('text="Blogs – Politik"', r.content.decode())
def test_settings(self):
self.post('/feeds/settings/', {'news_keep_days': 5})
self.assertEqual(FeedSettings.objects.get(user=self.u).news_keep_days, 5)
class FakeResponse:
def __init__(self, status=302, location='/', text='', url='https://www.jungewelt.de/x', ctype='text/html'):
self.status_code = status
self.headers = {'Location': location, 'Content-Type': ctype}
self.text = text
self.url = url
PAYWALL = ('<html><body><article><h1>T</h1><p>' + 'Freier Anfang des Artikels. ' * 12 + '</p></article>'
'<p>Unterstützen Sie die junge Welt mit einem Abonnement und erhalten Sie Zugang zu allen Analysen</p></body></html>')
FULL = '<html><body><article><h1>T</h1>' + ('<p>' + 'Der ganze Artikel für Abonnenten. ' * 15 + '</p>') * 4 + '</article></body></html>'
class SiteLoginTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('tester', password='pw12345678')
self.client.force_login(self.u)
self.src = Source.objects.create(user=self.u, url='https://www.jungewelt.de/feeds/newsticker.rss',
title='junge Welt', kind='news')
self.item = Item.objects.create(source=self.src, user=self.u, kind='news', guid_hash='a', title='Artikel',
url='https://www.jungewelt.de/artikel/1.html', summary='Anriss',
content='<p>Anriss</p>', published=timezone.now())
logins._sessions.clear()
def post(self, url, data):
return self.client.post(url, json.dumps(data), content_type='application/json')
def test_encryption_round_trip_and_not_plain(self):
token = logins.encrypt('geheim')
self.assertNotIn('geheim', token)
self.assertEqual(logins.decrypt(token), 'geheim')
self.assertIsNone(logins.decrypt('kaputt'))
def test_paywalled_page_keeps_free_part_and_waits(self):
with mock.patch('requests.get', return_value=FakeResponse(200, text=PAYWALL)):
refresh.load_article(self.item)
self.item.refresh_from_db()
self.assertTrue(self.item.paywalled)
self.assertFalse(self.item.content_full)
self.assertIn('Freier Anfang', self.item.content)
self.assertFalse(refresh.needs_article(self.item, self.src)) # no login: don't try again
def test_wrong_login_is_refused_and_not_stored(self):
with mock.patch('requests.Session.post', return_value=FakeResponse(302, 'https://www.jungewelt.de/loginFailed.php?x')):
r = self.post('/feeds/logins/', {'source': self.src.id, 'username': 'u', 'password': 'falsch'})
self.assertEqual(r.status_code, 400)
self.assertIn('falsch', r.json()['error'])
self.assertFalse(SiteLogin.objects.exists())
def test_login_stores_encrypted_and_refetches_full_articles(self):
Item.objects.filter(pk=self.item.pk).update(paywalled=True)
def login_post(session, url, **kw):
session.cookies.set('jW', 'sitzung')
return FakeResponse(302, 'https://www.jungewelt.de/')
with mock.patch('requests.Session.post', autospec=True, side_effect=login_post), \
mock.patch('requests.Session.get', return_value=FakeResponse(200, text=FULL)):
r = self.post('/feeds/logins/', {'source': self.src.id, 'username': 'leser', 'password': 'abo-passwort'})
self.assertEqual(r.status_code, 200)
self.assertEqual(r.json()['refetched'], 1)
login = SiteLogin.objects.get(user=self.u)
self.assertNotIn('abo-passwort', login.secret)
self.item.refresh_from_db()
self.assertTrue(self.item.content_full)
self.assertFalse(self.item.paywalled)
self.assertIn('ganze Artikel', self.item.content)
state = self.client.get('/feeds/state/').json()
info = state['sources'][0]['login']
self.assertEqual((info['username'], info['ok'], info['site']), ('leser', True, 'junge Welt'))
self.assertNotIn('abo-passwort', json.dumps(state))
self.post('/feeds/logins/delete/', {'source': self.src.id})
self.assertFalse(SiteLogin.objects.exists())
def test_sources_of_other_sites_have_no_login(self):
other = Source.objects.create(user=self.u, url='https://blog.example/feed', title='B', kind='blog')
state = self.client.get('/feeds/state/').json()
by_id = {s['id']: s for s in state['sources']}
self.assertIsNone(by_id[other.id]['login'])
self.assertIsNotNone(by_id[self.src.id]['login'])
class BlogFeedTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('tester', password='pw12345678')
self.client.force_login(self.u)
now = timezone.now()
self.busy = Source.objects.create(user=self.u, url='https://busy/', title='Busy', kind='blog', folders=['Politik'])
self.rare = Source.objects.create(user=self.u, url='https://rare/', title='Rare', kind='blog', folders=['Politik'])
for n in range(10): # a post every two hours
Item.objects.create(source=self.busy, user=self.u, kind='blog', guid_hash=f'b{n}', title=f'Busy {n}',
content='<p>' + 'wort ' * 460 + '</p>', words=460, published=now - timedelta(hours=2 * n))
Item.objects.create(source=self.rare, user=self.u, kind='blog', guid_hash='r', title='Rare essay',
published=now - timedelta(hours=5))
def post(self, url, data):
return self.client.post(url, json.dumps(data), content_type='application/json')
def by_id(self):
return {s['id']: s for s in self.client.get('/feeds/state/').json()['sources']}
def test_interval_new_and_words(self):
st = self.by_id()
self.assertLess(st[self.busy.id]['interval_days'], 1)
self.assertEqual(st[self.rare.id]['interval_days'], 30.0) # one post: counts as rare
self.assertEqual(st[self.busy.id]['new'], 10)
items = self.client.get('/feeds/blogs/?unread=1').json()['items']
self.assertEqual(next(i for i in items if i['title'] == 'Busy 0')['words'], 460)
def test_hidden_blog_only_in_its_folder_and_not_counted(self):
self.post(f'/feeds/sources/{self.busy.id}/', {'hidden': True})
state = self.client.get('/feeds/state/').json()
self.assertEqual((state['blogs_unread'], state['blogs_new']), (1, 1))
main = [i['title'] for i in self.client.get('/feeds/blogs/?unread=1').json()['items']]
self.assertEqual(main, ['Rare essay'])
folder = self.client.get('/feeds/blogs/?unread=1&folder=Politik').json()['items']
self.assertEqual(len(folder), 11)
def test_favorites_filter(self):
self.post(f'/feeds/sources/{self.rare.id}/', {'fav': True})
self.assertTrue(self.by_id()[self.rare.id]['fav'])
favs = [i['title'] for i in self.client.get('/feeds/blogs/?fav=1').json()['items']]
self.assertEqual(favs, ['Rare essay'])
class NotifyTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('tester', password='pw12345678')
self.client.force_login(self.u)
self.src = Source.objects.create(user=self.u, url='https://example.org/feed', title='Blog', kind='blog', notify=True)
def fetch(self, items):
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss(items))):
refresh.refresh([self.src])
def test_rules(self):
old = (timezone.now() - timedelta(days=5)).strftime('%a, %d %b %Y %H:%M:%S +0000')
self.fetch([{'guid': 'a', 'title': 'Schon da', 'text': LONG}]) # first fetch: never
start = self.client.get('/feeds/notifications/').json()
self.assertEqual(start['items'], [])
self.fetch([{'guid': 'a', 'title': 'Schon da', 'text': LONG},
{'guid': 'b', 'title': 'Neuer Post', 'text': LONG},
{'guid': 'c', 'title': 'Alter Post', 'text': LONG, 'date': old}])
got = self.client.get(f"/feeds/notifications/?after={start['last']}").json()
self.assertEqual([i['title'] for i in got['items']], ['Neuer Post'])
# Seen up to there: nothing again; read or bell off: nothing either.
self.assertEqual(self.client.get(f"/feeds/notifications/?after={got['last']}").json()['items'], [])
Source.objects.filter(pk=self.src.pk).update(notify=False)
self.assertEqual(self.client.get(f"/feeds/notifications/?after={start['last']}").json()['items'], [])
class FolderAndProgressTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('tester', password='pw12345678')
self.client.force_login(self.u)
self.a = Source.objects.create(user=self.u, url='https://a/', title='A', kind='blog', folders=['Philosophie'])
self.b = Source.objects.create(user=self.u, url='https://b/', title='B', kind='blog', folders=['philosophie', 'Rust'])
self.post_a = Item.objects.create(source=self.a, user=self.u, kind='blog', guid_hash='a', title='Essay',
published=timezone.now())
def post(self, url, data):
return self.client.post(url, json.dumps(data), content_type='application/json')
def test_folders_case_insensitive_rename_dissolve(self):
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Philosophie', 'Rust'])
self.post(f'/feeds/sources/{self.a.id}/', {'folders': [' rust ', 'Neu', 'neu']})
self.a.refresh_from_db()
self.assertEqual(self.a.folders, ['Rust', 'Neu']) # spelled as the existing one, no doubles
self.post('/feeds/folders/rename/', {'name': 'rust', 'new': 'Rust & Linux'})
self.a.refresh_from_db(); self.b.refresh_from_db()
self.assertEqual(self.a.folders, ['Neu', 'Rust & Linux'])
self.assertEqual(self.b.folders, ['philosophie', 'Rust & Linux'])
self.post('/feeds/folders/dissolve/', {'name': 'Rust & Linux'})
self.b.refresh_from_db()
self.assertEqual(self.b.folders, ['philosophie'])
self.assertTrue(Source.objects.filter(pk=self.b.pk).exists())
titles = [i['title'] for i in self.client.get('/feeds/blogs/?folder=Neu').json()['items']]
self.assertEqual(titles, ['Essay'])
def test_progress_marks_read_at_ninety_percent(self):
r = self.post(f'/feeds/items/{self.post_a.id}/progress/', {'progress': 0.4}).json()
self.assertEqual(r, {'progress': 0.4, 'read': False})
self.assertEqual(self.client.get('/feeds/blogs/').json()['items'][0]['progress'], 0.4)
self.assertTrue(self.post(f'/feeds/items/{self.post_a.id}/progress/', {'progress': 0.95}).json()['read'])
class DiscoverKindTest(TestCase):
"""Abos & Quellen: a feed whose entries carry sound is offered as a podcast."""
def setUp(self):
self.u = User.objects.create_user('u', password='pw12345678')
self.client.force_login(self.u)
def discover(self, items):
xml = ('<?xml version="1.0"?><rss version="2.0"><channel><title>X</title>' + ''.join(
f'<item><title>Folge {i}</title><guid>g{i}</guid>{extra}</item>' for i, extra in enumerate(items)) +
'</channel></rss>')
with mock.patch('feeds.views._probe', lambda url: (('https://ex.org/feed', 'X', feedparser.parse(xml)), None)):
r = self.client.post('/feeds/discover/', json.dumps({'url': 'https://ex.org/feed'}),
content_type='application/json')
return r.json()['feeds'][0]
def test_sound_files_make_a_podcast(self):
enc = '<enclosure url="https://ex.org/a.mp3" type="audio/mpeg" length="1"/>'
f = self.discover([enc] * 4)
self.assertEqual(f['suggested_kind'], 'podcast')
self.assertFalse(f['podcast_subscribed'])
def test_text_entries_stay_news_or_a_blog(self):
self.assertIn(self.discover([''] * 4)['suggested_kind'], ('news', 'blog'))
class FolderOrderTest(TestCase):
def test_blog_folders_come_in_the_users_order(self):
u = User.objects.create_user('u', password='pw12345678')
self.client.force_login(u)
Source.objects.create(user=u, url='https://a/', title='A', kind='blog', folders=['Kunst', 'Code'])
Source.objects.create(user=u, url='https://b/', title='B', kind='blog', folders=['Politik'])
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Code', 'Kunst', 'Politik'])
r = self.client.post('/feeds/folders/order/', json.dumps({'order': ['Politik', 'kunst']}),
content_type='application/json')
self.assertEqual(r.json()['folders'], ['Politik', 'Kunst', 'Code'])
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Politik', 'Kunst', 'Code'])
DONATE = 'Unterstützen Sie unabhängigen Journalismus mit einer Spende an unsere Redaktion.'
FRANCE = [
('Proteste in Frankreich: Gewalt bei Demonstrationen gegen Macron', 'Die Proteste in Frankreich gegen die Regierung Macron nehmen zu.'),
('Frankreich: Proteste gegen Sparpläne weiten sich aus', 'In Frankreich wachsen die Proteste gegen die Sparpläne von Macron.'),
('Macron verurteilt Gewalt bei Protesten in Frankreich', 'Präsident Macron verurteilt die Gewalt bei den Protesten in Frankreich.'),
]
class TextprocTest(TestCase):
def test_furniture_is_what_a_source_prints_under_many_articles(self):
texts = [f'Artikel Nummer {n} über etwas ganz anderes als die anderen. {DONATE}' for n in 'abcd']
keys = textproc.furniture(texts)
self.assertEqual(textproc.without_furniture(texts[0], keys), 'Artikel Nummer a über etwas ganz anderes als die anderen.')
html = f'<p>Der eigentliche Text des Artikels steht hier.</p><p>{DONATE}</p>'
self.assertEqual(textproc.html_without_furniture(html, keys), '<p>Der eigentliche Text des Artikels steht hier.</p>')
self.assertEqual(textproc.furniture(texts[:2]), set()) # twice is no pattern
def test_overview_of_two_reports(self):
shared = 'Die Regierung in Paris hat am Montag ein neues Sparpaket von vierzig Milliarden Euro vorgestellt. '
a = shared + 'Gewerkschaften kündigten daraufhin landesweite Streiks für die kommende Woche an und riefen zu Demonstrationen auf.'
b = shared + 'Die Börse in Paris reagierte mit deutlichen Kursgewinnen bei den großen Banken des Landes am Nachmittag.'
docs = [('taz', a + ' Gewerkschaften Gewerkschaften'), ('FAZ', b + ' Börse Börse')]
self.assertTrue(textproc.summarize(docs, k=2))
novel = textproc.novel(docs)
self.assertIn('Gewerkschaften', novel['taz'][0])
self.assertIn('Börse', novel['FAZ'][0])
base = ' Die Regierung stellte das Paket vor und die Opposition kritisierte es scharf.' * 3
words = textproc.word_choice([('taz', base + ' Gewerkschaften streiken.' * 5), ('FAZ', base + ' Börse steigt.' * 5)])
self.assertEqual(words, {'taz': ['Gewerkschaften'], 'FAZ': ['Börse']})
def test_topics_group_one_matter_from_several_sources(self):
stories = [(n, f'{t}. {s}', {src}) for n, ((t, s), src) in enumerate(zip(FRANCE, ['taz', 'nd', 'tagesschau']))]
stories.append((9, 'Bahn erhöht die Preise für das Deutschlandticket. Fahrgäste zahlen mehr.', {'taz'}))
got = textproc.topics(stories)
self.assertEqual(len(got), 1)
self.assertEqual(sorted(got[0]['ids']), [0, 1, 2])
self.assertIn('Frankreich', got[0]['label'])
self.assertEqual(textproc.topics([(n, t, {'taz'}) for n, t, _ in stories[:3]]), []) # one source: no topic
def test_related_story_in_the_other_language(self):
de = 'Der Papst Leo traf in Beirut den libanesischen Präsidenten Aoun und Ministerpräsident Salam.'
en = 'Pope Leo met Lebanese President Aoun and Prime Minister Salam in Beirut on Sunday.'
other = 'Die Bahn erhöht in Berlin die Preise für das Deutschlandticket deutlich.'
sl = [(1, textproc.language(de), textproc.names(de)), (2, textproc.language(en), textproc.names(en)),
(3, textproc.language(other), textproc.names(other))]
self.assertEqual((sl[0][1], sl[1][1]), ('de', 'en'))
got = textproc.related(sl, min_weight=1.0)
self.assertEqual([b for b, _ in got[1]], [2])
self.assertNotIn(3, got)
class NewsTextApiTest(TestCase):
def setUp(self):
self.u = User.objects.create_user('reader', password='pw12345678')
self.client.force_login(self.u)
cache.clear()
now = timezone.now()
self.items = []
for n, ((title, text), name) in enumerate(zip(FRANCE, ['taz', 'nd', 'tagesschau'])):
src = Source.objects.create(user=self.u, url=f'https://{name}/feed', title=name, kind='news', category='Welt')
story = Story.objects.create(user=self.u, first_at=now, last_at=now - timedelta(minutes=n))
for k in range(4): # older articles of the source, each with the donation box
Item.objects.create(source=src, user=self.u, kind='news', guid_hash=f'{name}{k}', title=f'Alt {k}',
published=now - timedelta(days=1, minutes=k),
content=f'<p>Ein älterer Artikel Nummer {k} von {name}.</p><p>{DONATE}</p>')
i = Item.objects.create(source=src, user=self.u, kind='news', guid_hash=f'{name}x', title=title,
summary=text, published=now - timedelta(minutes=n), story=story,
content=f'<p>{text} Mehr dazu im Laufe des Tages von {name}.</p><p>{DONATE}</p>')
story.lead = i
story.save()
self.items.append(i)
def test_first_page_has_topics_and_ids_filter(self):
r = self.client.get('/feeds/news/').json()
self.assertEqual(len(r['topics']), 1)
ids = r['topics'][0]['ids']
self.assertEqual(len(ids), 3)
got = self.client.get('/feeds/news/?ids=' + ','.join(map(str, ids[:2]))).json()
self.assertEqual(sorted(s['id'] for s in got['stories']), sorted(ids[:2]))
self.assertNotIn('topics', got)
def test_unencoded_plus_in_before_is_no_error(self):
when = timezone.now().isoformat() # +00:00, sent unencoded the '+' becomes a space
r = self.client.get('/feeds/news/?before=' + when.replace('+', ' '))
self.assertEqual(r.status_code, 200)
self.assertEqual(self.client.get('/feeds/news/?before=garbled').status_code, 200)
def test_article_comes_without_furniture(self):
got = self.client.get(f'/feeds/items/{self.items[0].id}/').json()['item']
self.assertNotIn('Spende', got['content'])
self.assertIn('Proteste', got['content'])
def test_overview_of_a_story_with_several_sources(self):
a, b = self.items[0], self.items[1]
b.story = a.story
b.save()
got = self.client.get(f'/feeds/stories/{a.story_id}/overview/').json()
self.assertTrue(got['summary'])
self.assertFalse(any('Spende' in x['text'] for x in got['summary']))
self.assertEqual(got['related'], [])
single = self.client.get(f'/feeds/stories/{self.items[2].story_id}/overview/').json()
self.assertNotIn('summary', single)
class SyntheticSourceTest(TestCase):
"""A source diora fills itself ("Podcast-Artikel") is never fetched or pruned."""
def setUp(self):
self.u = User.objects.create_user('tester', password='pw12345678')
self.client.force_login(self.u)
self.synth = Source.objects.create(user=self.u, url='diora://podcast-artikel', title='Podcast-Artikel',
kind='blog', fav=True, synthetic=True)
self.real = Source.objects.create(user=self.u, url='https://real.example/feed', title='Real', kind='blog')
def post(self, url, data):
return self.client.post(url, json.dumps(data), content_type='application/json')
def test_refresh_skips_it(self):
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([]), 'https://real.example/feed')) as get:
refresh.refresh([self.synth, self.real])
self.assertEqual([c.args[0] for c in get.call_args_list], ['https://real.example/feed'])
def test_refresh_now_and_command_skip_it(self):
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([]), 'https://real.example/feed')) as get:
r = self.post('/feeds/refresh/', {})
from django.core.management import call_command
call_command('refresh_newsfeeds', stdout=mock.MagicMock())
self.assertEqual(r.status_code, 200)
self.assertTrue(get.call_args_list)
self.assertTrue(all(c.args[0] == 'https://real.example/feed' for c in get.call_args_list))
call_command('refresh_newsfeeds', '--source-id', str(self.synth.id), stdout=mock.MagicMock())
def test_old_items_are_not_pruned(self):
old = timezone.now() - timedelta(days=400)
for source in (self.synth, self.real):
for n in range(refresh.BLOG_KEEP + 5):
Item.objects.create(source=source, user=self.u, kind='blog', guid_hash=f'{source.pk}-{n}',
title='x', published=old - timedelta(minutes=n), read_at=old)
refresh.expire(self.u)
self.assertEqual(self.synth.items.count(), refresh.BLOG_KEEP + 5)
self.assertEqual(self.real.items.count(), refresh.BLOG_KEEP)
def test_cannot_become_news_but_can_be_hidden(self):
r = self.post(f'/feeds/sources/{self.synth.id}/', {'kind': 'news'})
self.assertEqual(r.status_code, 400)
r = self.post(f'/feeds/sources/{self.synth.id}/', {'hidden': True, 'notify': True})
self.assertEqual(r.status_code, 200)
self.synth.refresh_from_db()
self.assertEqual((self.synth.kind, self.synth.hidden, self.synth.notify), ('blog', True, True))
self.assertTrue(r.json()['source']['synthetic'])
def test_not_in_opml(self):
body = self.client.get('/feeds/export.opml').content.decode()
self.assertIn('real.example', body)
self.assertNotIn('diora://', body)

353
feeds/textproc.py
View file

@ -1,353 +0,0 @@
"""Reading help for news, with classic text methods (no language model):
- furniture: sentences a source prints under many of its articles (donation boxes,
"Diese Nachricht wurde … gesendet") are no part of an article; they are left out
when an article is shown and when a story is looked at;
- a story's overview: a summary drawn from all its reports (LexRank: the sentences
most like the others, without repeating each other — MMR), what only one report
says (its sentences unlike any other source's), and each source's own words for
the event (log-odds of a word in that report against the story's other reports);
- related stories in the other language (German/English): stories of the same days
that share names (capitalised words that aren't at a sentence's start), weighted by
how rare a name is among the day's stories;
- topics: stories grouped by their words when they're about the same matter, not
only the same event, labelled with the words they share most.
Pure Python: these run on the server with the feeds, for one user's stories at a time.
"""
import math
import re
from collections import Counter, defaultdict
from . import fetch
_SENT_END = re.compile(r'(?<=[.!?…])\s+(?=[„"»A-ZÄÖÜ0-9])')
_TOKEN = re.compile(r"[A-Za-zÄÖÜäöüßÀ-ÿ][A-Za-zÄÖÜäöüßÀ-ÿ0-9'’\-]+")
STOP = set('''
aber alle allem allen aller alles also als am an andere anderen auch auf aus bei beim bereits bis bisher
da dabei damit dann darauf darf darum das dass dem den denen der deren des deshalb dessen die dies diese
diesem diesen dieser dieses doch dort durch ein eine einem einen einer eines einige er es etwa euch für
gegen geht gibt hab habe haben hat hatte hatten hier hin hinter ich ihr ihre ihrem ihren ihrer im immer
in ins ist ja jahr jahre jahren jetzt kann kein keine können könnte man mehr mit muss müssen nach neue neuen
nicht nichts noch nun nur ob oder ohne schon sei seid sein seine seinem seinen seiner seit sich sie sind
so soll sollen sollte sondern sowie über um und uns unter viel viele vom von vor war waren was weil weiter
weitere wenn wer werde werden wie wieder will wir wird wo worden wurde wurden zu zum zur zwei drei vier
heute gestern morgen laut sagt sagte sagten erst mal neu video live ticker meldung bericht hatte hätte
außerdem diejenigen derjenige inzwischen zuvor derzeit dennoch trotzdem deswegen ebenfalls
wäre würde würden zudem sowohl jedoch allerdings bereits rund etwa demnach zufolge angaben unter
the and for with from that this have has are was were will would been into about after over their they
his her its our you your not but all can new says said more than when what who how why off out also had
one two which there been being some other could should would may might them then just only very
'''.split())
# Capitalised words that are no names (German nouns are capitalised too: those don't matter,
# they never match an English text; these would match both).
_NOT_NAMES = set('''
Der Die Das Den Dem Des Ein Eine Einen Einem Einer Er Sie Es Wir Ihr Ich Und Oder Aber Auch Doch Mit Von
Nach Bei Für Auf Aus Im In Am Zum Zur Vor Seit Als Wie Wenn Dass Nicht Noch Nur So Da Dann Hier Jetzt
The A An And Or But With From For After Before In On At By To Of As It He She They We I This That These
Those Mr Mrs Ms Dr Monday Tuesday Wednesday Thursday Friday Saturday Sunday Montag Dienstag Mittwoch
Donnerstag Freitag Samstag Sonntag Sonnabend January February March April May June July August September
October November December Januar Februar März Juni Juli Oktober Dezember Reuters AFP AP dpa BBC CNN
'''.split())
def plain(html):
return fetch.plain(html or '')
def sentences(text):
text = re.sub(r'\s+', ' ', text or '').strip()
return [s.strip() for s in _SENT_END.split(text) if 40 <= len(s.strip()) <= 400]
def _key(s):
"""A sentence, the same however its numbers and spaces differ."""
return re.sub(r'\d+', '0', re.sub(r'\s+', ' ', s.lower())).strip()
def words(text):
return [w.lower() for w in _TOKEN.findall(text or '') if len(w) > 2 and w.lower() not in STOP]
# ---- furniture ------------------------------------------------------------------------------------
FURNITURE_MIN = 3 # in this many of a source's articles: furniture
def furniture(texts):
"""The sentences (keys) that stand in at least FURNITURE_MIN of these texts (one source's)."""
seen = Counter()
for t in texts:
seen.update({_key(s) for s in _units(t)})
return {k for k, n in seen.items() if n >= FURNITURE_MIN}
def _units(text):
return [u.strip() for u in re.split(r'\n|(?<=[.!?])\s+', text or '') if len(u.strip()) > 15]
def without_furniture(text, keys):
if not keys:
return text
return ' '.join(u for u in _units(text) if _key(u) not in keys)
_BLOCK = re.compile(r'<(p|li|h[1-6]|blockquote|figcaption|div)\b[^>]*>(.*?)</\1>', re.S | re.I)
def html_without_furniture(html, keys):
"""The article's HTML without the blocks whose text is (only) furniture."""
if not keys or not html:
return html
def keep(m):
inner = plain(m.group(2))
units = _units(inner)
if units and all(_key(u) in keys for u in units):
return ''
return m.group(0)
return _BLOCK.sub(keep, html)
# ---- tf-idf over sentences -------------------------------------------------------------------------
def _vectors(texts):
toks = [Counter(words(t)) for t in texts]
df = Counter()
for t in toks:
df.update(t.keys())
n = len(texts)
idf = {w: math.log((n + 1) / (d + 1)) + 1 for w, d in df.items()}
vecs = []
for t in toks:
v = {w: (1 + math.log(c)) * idf[w] for w, c in t.items()}
norm = math.sqrt(sum(x * x for x in v.values())) or 1.0
vecs.append({w: x / norm for w, x in v.items()})
return vecs
def _cos(a, b):
if len(a) > len(b):
a, b = b, a
return sum(x * b.get(w, 0.0) for w, x in a.items())
def summarize(docs, k=3, lam=0.7):
"""docs: [(source, text)]. The k sentences that say most of what all say, in their order
of appearance: [(source, sentence)]."""
ss = [(src, s) for src, t in docs for s in sentences(t)[:30]]
if len(ss) <= k:
return ss
vecs = _vectors([s for _, s in ss])
n = len(ss)
sim = [[_cos(vecs[a], vecs[b]) if a != b else 0.0 for b in range(n)] for a in range(n)]
links = [[1.0 if sim[a][b] > 0.1 else 0.0 for b in range(n)] for a in range(n)]
deg = [sum(r) or 1.0 for r in links]
rank = [1.0 / n] * n
for _ in range(40):
rank = [0.15 / n + 0.85 * sum(links[b][a] * rank[b] / deg[b] for b in range(n)) for a in range(n)]
top = max(rank) or 1.0
chosen = []
for _ in range(k):
best, score = None, -1e9
for a in range(n):
if a in chosen:
continue
s = lam * rank[a] / top - (1 - lam) * max((sim[a][c] for c in chosen), default=0.0)
if s > score:
best, score = a, s
chosen.append(best)
return [ss[c] for c in sorted(chosen)]
def novel(docs, per_source=2, limit=0.2):
"""What only one report says: per source, up to per_source of its sentences that no other
source has anything like (cosine < limit), the most central of its own first."""
ss = [(src, s) for src, t in docs for s in sentences(t)[:40]]
if len({src for src, _ in ss}) < 2:
return {}
vecs = _vectors([s for _, s in ss])
out = defaultdict(list)
for a, (src, s) in enumerate(ss):
others = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 != src]
if not others or max(others) >= limit:
continue
own = [_cos(vecs[a], vecs[b]) for b, (s2, _) in enumerate(ss) if s2 == src and b != a]
out[src].append((sum(own) / len(own) if own else 0.0, s))
return {src: [s for _, s in sorted(v, key=lambda x: -x[0])[:per_source]] for src, v in out.items()}
def word_choice(docs, per_source=6):
"""Each source's own words for the event: log-odds of a word in its report against the
other reports (with a prior from all of them), at least twice in its report."""
forms = defaultdict(Counter)
counts = {}
for src, t in docs:
c = Counter()
for w in _TOKEN.findall(t or ''):
lw = w.lower()
if len(lw) > 3 and lw not in STOP:
c[lw] += 1
forms[lw][w] += 1
counts[src] = counts.get(src, Counter()) + c
if len(counts) < 2:
return {}
total = sum(counts.values(), Counter())
a0 = sum(total.values()) or 1
out = {}
for src, c in counts.items():
n_i = sum(c.values()) or 1
rest = total - c
n_j = sum(rest.values()) or 1
scored = []
for w, f in c.items():
if f < 2:
continue
aw = total[w] / a0 * 100
d = math.log((f + aw) / (n_i + 100 - f - aw)) - math.log((rest[w] + aw) / (n_j + 100 - rest[w] - aw))
scored.append((d / math.sqrt(1 / (f + aw) + 1 / (rest[w] + aw)), w))
# Nouns and names (as written: capitalised), not the verbs and particles around them.
best = [(z, forms[w].most_common(1)[0][0]) for z, w in sorted(scored, reverse=True) if z > 1.0]
out[src] = [w for z, w in best if w[:1].isupper()][:per_source]
return {k: v for k, v in out.items() if v}
# ---- names, languages, related stories -------------------------------------------------------------
_EN = set('the and of to in is that for with was said on as by at from'.split())
_DE = set('der die das und ist nicht mit den von zu im sich auf für dem des eine sagte'.split())
def language(text):
w = [x.lower() for x in _TOKEN.findall(text[:3000])]
en = sum(1 for x in w if x in _EN)
de = sum(1 for x in w if x in _DE)
return 'en' if en > de else 'de'
def names(text):
"""Capitalised words not at a sentence's start (in German: names and nouns; in English:
names), as a set of lowercased forms."""
out = set()
for s in sentences(text) or [text or '']:
toks = _TOKEN.findall(s)
for t in toks[1:]:
if t[0].isupper() and t not in _NOT_NAMES and len(t) > 2:
out.add(t.lower().rstrip('s’\''))
return out
def related(stories, min_weight=17.0, per_story=3):
"""stories: [(id, language, set of names)] of the same days. Pairs of stories in different
languages sharing rare names: {id: [(other id, weight)]}, weight = the shared names' idf."""
df = Counter()
for _, _, ns in stories:
df.update(ns)
n = len(stories)
idf = {w: math.log((n + 1) / (d + 1)) for w, d in df.items()}
out = defaultdict(list)
for a in range(n):
ia, la, na = stories[a]
for b in range(a + 1, n):
ib, lb, nb = stories[b]
if la == lb:
continue
shared = na & nb
if len(shared) < 3:
continue
w = sum(idf[x] for x in shared)
if w >= min_weight:
out[ia].append((ib, round(w, 1)))
out[ib].append((ia, round(w, 1)))
return {k: sorted(v, key=lambda x: -x[1])[:per_story] for k, v in out.items()}
# ---- topics ---------------------------------------------------------------------------------------------
NOT_TOPICS = set('breaking news free live watch video sekunden gebärdensprache podcast newsletter abo online'.split())
TOPIC_MIN = 3
def topics(stories, threshold=0.12, min_size=TOPIC_MIN, max_topics=8, rounds=3):
"""stories: [(id, text, sources)] (titles and summaries, the set of source names). Groups of
stories about one matter, the largest first, each with a label (the words most of its stories
share): [{'label': str, 'ids': [ids]}]. Leader clustering against the groups' centroids, a
few rounds (each story to its nearest centroid, then the centroids anew); a group counts
when it has min_size stories from at least two sources."""
if len(stories) < min_size:
return []
vecs = _vectors([s[1] for s in stories])
order = sorted(range(len(stories)), key=lambda i: -len(vecs[i]))
cents = []
assign = [-1] * len(stories)
for _ in range(rounds):
changed = False
for i in order:
best, sim = -1, threshold
for g, c in enumerate(cents):
x = _cos(vecs[i], c)
if x > sim:
best, sim = g, x
if best < 0:
cents.append(dict(vecs[i]))
best = len(cents) - 1
if assign[i] != best:
assign[i] = best
changed = True
# centroids anew from their members
acc = [Counter() for _ in cents]
for i, g in enumerate(assign):
for w, v in vecs[i].items():
acc[g][w] += v
cents = []
remap = {}
for g, a in enumerate(acc):
if a:
norm = math.sqrt(sum(v * v for v in a.values())) or 1.0
remap[g] = len(cents)
cents.append({w: v / norm for w, v in a.items()})
assign = [remap.get(g, -1) for g in assign]
if not changed:
break
groups = defaultdict(list)
for i, g in enumerate(assign):
groups[g].append(i)
out = []
source_words = {w.lower() for s in stories for src in s[2] for w in _TOKEN.findall(src)}
for g in sorted(groups.values(), key=len, reverse=True):
if len(g) < min_size or len({src for i in g for src in stories[i][2]}) < 2:
continue
# Close enough together: the stories' mean likeness to their group's centre.
acc = Counter()
for i in g:
for w, v in vecs[i].items():
acc[w] += v
norm = math.sqrt(sum(v * v for v in acc.values())) or 1.0
centre = {w: v / norm for w, v in acc.items()}
if sum(_cos(vecs[i], centre) for i in g) / len(g) < 0.3:
continue
df = Counter()
for i in g:
df.update({w for w in words(stories[i][1]) if w not in NOT_TOPICS and w not in source_words})
nice = []
for w, c in df.most_common(8):
if c < max(2, len(g) // 2):
break
forms = Counter(t for i in g for t in _TOKEN.findall(stories[i][1]) if t.lower() == w)
form = forms.most_common(1)[0][0] if forms else w
# Nouns and names make a label, not "rückt" or "näher"; "Linke" once, not "Linken" too.
if form[:1].isupper() and not any(x.lower()[:5] == form.lower()[:5] for x in nice):
nice.append(form)
if len(nice) == 3:
break
if len(nice) < 2: # one shared word is a coincidence, not a topic
continue
out.append({'label': ' · '.join(nice), 'ids': [stories[i][0] for i in g]})
if len(out) >= max_topics:
break
return out

32
feeds/urls.py
View file

@ -1,32 +0,0 @@
from django.urls import path
from . import views
urlpatterns = [
path('state/', views.state, name='feeds_state'),
path('discover/', views.discover, name='feeds_discover'),
path('sources/add/', views.add_source, name='feeds_add_source'),
path('sources/bulk/', views.add_sources, name='feeds_add_sources'),
path('sources/<int:pk>/', views.edit_source, name='feeds_edit_source'),
path('sources/<int:pk>/delete/', views.delete_source, name='feeds_delete_source'),
path('export.opml', views.export_opml, name='feeds_export_opml'),
path('refresh/', views.refresh_now, name='feeds_refresh'),
path('briefing/', views.briefing, name='feeds_briefing'),
path('briefing/seen/', views.briefing_seen, name='feeds_briefing_seen'),
path('news/', views.news, name='feeds_news'),
path('stories/<int:pk>/', views.story, name='feeds_story'),
path('stories/<int:pk>/overview/', views.story_overview, name='feeds_story_overview'),
path('blogs/', views.blogs, name='feeds_blogs'),
path('starred/', views.starred, name='feeds_starred'),
path('items/read/', views.mark_read, name='feeds_mark_read'),
path('items/<int:pk>/', views.item, name='feeds_item'),
path('items/<int:pk>/star/', views.star, name='feeds_star'),
path('items/<int:pk>/progress/', views.item_progress, name='feeds_item_progress'),
path('folders/rename/', views.rename_folder, name='feeds_rename_folder'),
path('folders/order/', views.folder_order, name='feeds_folder_order'),
path('folders/dissolve/', views.dissolve_folder, name='feeds_dissolve_folder'),
path('settings/', views.settings, name='feeds_settings'),
path('notifications/', views.notifications, name='feeds_notifications'),
path('logins/', views.save_login, name='feeds_save_login'),
path('logins/delete/', views.delete_login, name='feeds_delete_login'),
]

836
feeds/views.py
View file

@ -1,836 +0,0 @@
"""The feed reader's JSON API (session or Bearer token, like the podcast API).
Sources: GET /feeds/state/ sources, unread counts, settings
POST /feeds/discover/ {url} -> the feeds a page or URL offers
POST /feeds/sources/add/ {url, kind, category, title?}
POST /feeds/sources/bulk/ {sources: [{url, title, kind, category}]} (OPML import)
POST /feeds/sources/<id>/ {title, kind, category (news), folders (blogs), fulltext, fav, hidden, notify}
POST /feeds/folders/rename/ {name, new} (all blogs in it)
POST /feeds/folders/dissolve/ {name} (the blogs stay)
POST /feeds/sources/<id>/delete/
GET /feeds/export.opml
POST /feeds/refresh/ refresh the account's sources now
News: GET /feeds/briefing/ what's new since the briefing was last seen
POST /feeds/briefing/seen/ seen: everything up to now counts as read
GET /feeds/news/?before=&category=&ids= stories, newest first (ids: just these, a topic's);
the first page also has the day's topics
GET /feeds/stories/<id>/ one story with all its items
GET /feeds/stories/<id>/overview/ summary, what only one report says, each source's
words, the story in the other language (feeds/textproc.py)
Blogs: GET /feeds/blogs/?folder=&source=&unread=1&fav=1&before= (folder= alone: blogs without one;
without folder/source the hidden blogs stay out)
Items: GET /feeds/items/<id>/ the article (fetches the page if the feed had a teaser)
POST /feeds/items/read/ {ids, read}
POST /feeds/items/<id>/star/ {starred}
POST /feeds/items/<id>/progress/ {progress} 0-1; from 0.9 on it's read
GET /feeds/starred/
Settings: POST /feeds/settings/ {news_keep_days}
Notices: GET /feeds/notifications/?after=<item id> new posts of blogs with the bell on (devices
remember the last id; without after: only that id)
Logins: POST /feeds/logins/ {source, username, password} subscriber login for its site
POST /feeds/logins/delete/ {source}
"""
import json
import time
from datetime import timedelta
from xml.sax.saxutils import quoteattr
import feedparser
import requests
from django.core.cache import cache
from django.db.models import Count
from django.http import HttpResponse, JsonResponse
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from . import fetch, logins, refresh, textproc
from .models import Item, SiteLogin, Source, Story
PAGE = 40
BLOG_PAGE = 150 # blog posts per page (all unread ones come at once, up to BLOG_UNREAD_MAX)
BLOG_UNREAD_MAX = 600
NEW_HOURS = 24 # a blog post this young and unread is "new"
RARE_POSTS = 10 # how often a blog posts: the mean gap over its last this many posts
FOLDER_MAX = 40 # characters of a folder name
READ_AT = 0.9 # read this far, a post counts as read
BRIEFING_TOP = 8
BRIEFING_PER_CATEGORY = 5
REFRESH_WAIT = 40 # seconds an explicit refresh may take
CATEGORY_NONE = 'Allgemein'
def _auth(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
return None
def _body(request):
try:
return json.loads(request.body or b'{}')
except (ValueError, UnicodeDecodeError):
return None
def _when(value):
"""A datetime parameter; None when there's none or it's garbled (an unencoded '+' arrives as
a space)."""
try:
return parse_datetime((value or '').strip().replace(' ', '+'))
except ValueError:
return None
def _iso(dt):
return dt.isoformat() if dt else None
def _source_data(s, unread=0, new=0, interval=None):
return {
'id': s.id, 'url': s.url, 'title': s.title, 'site_url': s.site_url, 'icon_url': s.icon_url,
'kind': s.kind, 'category': s.category, 'folders': s.folders, 'fulltext': s.fulltext,
'unread': unread, 'new': new,
'fav': s.fav, 'hidden': s.hidden, 'notify': s.notify, 'synthetic': s.synthetic,
# Days between posts (blogs, for "Seltene zuerst"); a blog with one post counts as rare.
'interval_days': interval,
'last_refreshed_at': _iso(s.last_refreshed_at), 'last_error': s.last_error, 'error_count': s.error_count,
'login': logins.info(s.user_id, s.url),
}
def _item_data(i, content=False):
d = {
'id': i.id, 'source': i.source_id, 'source_title': i.source.title, 'kind': i.kind,
'url': i.url, 'title': i.title, 'author': i.author, 'published': _iso(i.published), 'words': i.words,
'summary': i.summary, 'image_url': i.image_url, 'read': i.read_at is not None, 'progress': i.progress,
'starred': i.starred, 'story': i.story_id,
}
if i.source.synthetic:
# The article of a podcast episode (docs/transcription.md); the lists fetch it with the item.
art = getattr(i, 'episode_article', None)
if art is not None:
d['episode_id'] = art.episode_id
if content:
d['content'] = i.content
d['content_full'] = i.content_full
d['paywalled'] = i.paywalled
return d
def _story_data(s, items):
items = sorted(items, key=lambda i: i.published)
lead = next((i for i in items if i.id == s.lead_id), items[0] if items else None)
return {
'id': s.id, 'first_at': _iso(s.first_at), 'last_at': _iso(s.last_at),
'size': len(items), 'sources': len({i.source_id for i in items}),
'category': (lead.source.category if lead else '') or CATEGORY_NONE,
'unread': sum(1 for i in items if i.read_at is None),
'lead': _item_data(lead) if lead else None,
'items': [_item_data(i) for i in items],
}
def _stories(user, qs):
"""Story dicts for a Story queryset (items fetched in one go)."""
stories = list(qs)
items = {}
for i in Item.objects.filter(story__in=stories, gone=False).select_related('source'):
items.setdefault(i.story_id, []).append(i)
return [_story_data(s, items[s.id]) for s in stories if s.id in items]
# ---- sources
@csrf_exempt
@require_http_methods(['GET'])
def state(request):
if (err := _auth(request)):
return err
unread = dict(Item.objects.filter(user=request.user, gone=False, read_at=None)
.values_list('source').annotate(n=Count('id')))
now = timezone.now()
new = dict(Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False, read_at=None,
published__gte=now - timedelta(hours=NEW_HOURS))
.values_list('source').annotate(n=Count('id')))
sources = list(Source.objects.filter(user=request.user))
intervals = _intervals(request.user, [x for x in sources if x.kind == Source.BLOG], now)
s = refresh.settings_for(request.user)
since = s.news_seen_at or (timezone.now() - timedelta(days=1))
new_news = Story.objects.filter(user=request.user, last_at__gt=since,
items__gone=False, items__read_at=None).distinct().count()
return JsonResponse({
'sources': [_source_data(x, unread.get(x.id, 0), new.get(x.id, 0), intervals.get(x.id)) for x in sources],
'categories': sorted({x.category for x in sources if x.kind == Source.NEWS and x.category}),
'folders': _ordered(_all_folders(sources), s.folder_order),
'news_new': new_news,
'blogs_unread': sum(unread.get(x.id, 0) for x in sources if x.kind == Source.BLOG and not x.hidden),
'blogs_new': sum(new.get(x.id, 0) for x in sources if x.kind == Source.BLOG and not x.hidden),
'settings': {'news_keep_days': s.news_keep_days, 'news_seen_at': _iso(s.news_seen_at)},
})
def _clean_folder(name):
return ' '.join(str(name or '').split())[:FOLDER_MAX]
def _all_folders(sources):
"""The folders the blogs carry, each once (case doesn't matter), sorted."""
seen = {}
for s in sources:
if s.kind == Source.BLOG:
for f in s.folders or []:
seen.setdefault(f.casefold(), f)
return sorted(seen.values(), key=str.casefold)
def _ordered(names, order):
"""The folders in the order the user gave them (docs/library.md), new ones after."""
rank = {n.casefold(): i for i, n in enumerate(order or [])}
return sorted(names, key=lambda n: (rank.get(n.casefold(), len(rank)), n.casefold()))
@csrf_exempt
@require_http_methods(['POST'])
def folder_order(request):
"""{order: [names]}: the blog folders' chip order."""
if (err := _auth(request)):
return err
order = (_body(request) or {}).get('order')
if not isinstance(order, list) or not all(isinstance(n, str) for n in order):
return JsonResponse({'error': 'order: eine Liste von Namen'}, status=400)
s = refresh.settings_for(request.user)
s.folder_order = [_clean_folder(n) for n in order if _clean_folder(n)][:100]
s.save(update_fields=['folder_order'])
sources = Source.objects.filter(user=request.user, kind=Source.BLOG)
return JsonResponse({'folders': _ordered(_all_folders(sources), s.folder_order)})
def _folders_for(user, names):
"""Folder names cleaned, without doubles, spelled as an existing folder is."""
existing = {f.casefold(): f for f in _all_folders(Source.objects.filter(user=user, kind=Source.BLOG))}
out = []
for n in names or []:
n = _clean_folder(n)
if n and n.casefold() not in {o.casefold() for o in out}:
out.append(existing.get(n.casefold(), n))
return out
def _in_folder(source, name):
if name == '':
return not source.folders
return name.casefold() in {f.casefold() for f in source.folders or []}
def _intervals(user, blogs, now):
"""Mean days between the last RARE_POSTS posts of each blog, up to now:
(now - the oldest of them) / their number; one post counts as 30 days."""
out = {}
for s in blogs:
dates = list(s.items.filter(gone=False).order_by('-published').values_list('published', flat=True)[:RARE_POSTS])
if not dates:
continue
days = (now - dates[-1]).total_seconds() / 86400 / len(dates)
out[s.id] = round(max(days, 30.0) if len(dates) == 1 else days, 2)
return out
def _probe(url):
"""(feed url, title, entries) if `url` is a feed, else ([candidate feeds], icon)."""
r = requests.get(url, timeout=fetch.PAGE_TIMEOUT, headers={'User-Agent': fetch.USER_AGENT})
r.raise_for_status()
ctype = r.headers.get('Content-Type', '')
if 'html' in ctype and b'<rss' not in r.content[:2000] and b'<feed' not in r.content[:2000]:
return None, fetch.discover(r.text, r.url)
parsed = feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
if parsed.get('bozo') and not parsed.entries and not parsed.feed.get('title'):
return None, ([], '')
return (r.url, fetch.short_title(fetch.plain(parsed.feed.get('title') or '')) or r.url, parsed), None
@csrf_exempt
@require_http_methods(['POST'])
def discover(request):
"""The feeds behind a URL: itself if it's a feed, else the ones the page links to."""
if (err := _auth(request)):
return err
body = _body(request) or {}
url = (body.get('url') or '').strip()
if not url:
return JsonResponse({'error': 'Adresse fehlt'}, status=400)
if not url.startswith(('http://', 'https://')):
url = 'https://' + url
try:
feed, page = _probe(url)
if feed is None:
candidates, _icon = page
found = []
for u, title in candidates[:6]:
try:
f, _ = _probe(u)
except requests.RequestException:
continue
if f:
found.append(f)
if not found:
return JsonResponse({'error': 'Dort ist kein Feed zu finden.'}, status=404)
else:
found = [feed]
except requests.RequestException:
return JsonResponse({'error': 'Die Adresse ist nicht erreichbar.'}, status=502)
from podcasts.models import PodcastFeed
out = []
for u, title, parsed in found:
entries = parsed.entries[:5]
teaser = sum(1 for e in entries if fetch.is_teaser(fetch.parse_entry(e, u)['content']))
# Most entries with a sound or video file: a podcast (Abos & Quellen offers it as one).
media = sum(1 for e in parsed.entries[:10] if _has_media(e))
podcast = media and media * 2 >= min(len(parsed.entries), 10)
out.append({
'url': u, 'title': title,
'subscribed': Source.objects.filter(user=request.user, url=u).exists(),
'podcast_subscribed': PodcastFeed.objects.filter(user=request.user, rss_url=u).exists(),
'samples': [fetch.plain(e.get('title') or '') for e in entries],
# Many entries a day suggests news, few long ones a blog.
'suggested_kind': 'podcast' if podcast
else Source.NEWS if len(parsed.entries) >= 25 or teaser >= 3 else Source.BLOG,
})
return JsonResponse({'feeds': out})
def _has_media(entry):
for enc in entry.get('enclosures') or []:
if (enc.get('type') or '').lower().startswith(('audio/', 'video/')):
return True
return False
def _place(user, kind, category, folders, **more):
"""Where a new source goes: news into a section (category), a blog into
folders (the category of an OPML folder or the add dialog is its folder)."""
category = (category or '').strip()[:100]
if kind == Source.NEWS:
return dict(kind=kind, category=category, **more)
names = folders if isinstance(folders, list) else ([category] if category else [])
return dict(kind=kind, category='', folders=_folders_for(user, names), **more)
def _kind(v):
return v if v in (Source.NEWS, Source.BLOG) else None
@csrf_exempt
@require_http_methods(['POST'])
def add_source(request):
if (err := _auth(request)):
return err
body = _body(request) or {}
url = (body.get('url') or '').strip()
kind = _kind(body.get('kind'))
if not url.startswith(('http://', 'https://')) or not kind:
return JsonResponse({'error': 'Adresse und Art (Nachrichten/Blog) nötig'}, status=400)
source, created = Source.objects.get_or_create(
user=request.user, url=url[:1000],
defaults=_place(request.user, kind, body.get('category'), body.get('folders'),
title=(body.get('title') or url)[:300]))
if created:
refresh.refresh([source], deadline=time.monotonic() + 25)
source.refresh_from_db()
return JsonResponse({'source': _source_data(source, source.items.filter(read_at=None, gone=False).count()),
'created': created})
@csrf_exempt
@require_http_methods(['POST'])
def add_sources(request):
"""Many at once (an OPML file the page read); refreshed by the next run."""
if (err := _auth(request)):
return err
body = _body(request) or {}
added = 0
for x in body.get('sources') or []:
url = (x.get('url') or '').strip()
kind = _kind(x.get('kind'))
if not url.startswith(('http://', 'https://')) or not kind:
continue
_, created = Source.objects.get_or_create(
user=request.user, url=url[:1000],
defaults=_place(request.user, kind, x.get('category'), x.get('folders'),
title=(x.get('title') or url)[:300]))
added += created
sources = Source.objects.filter(user=request.user, last_refreshed_at=None, synthetic=False)
refresh.refresh(sources, deadline=time.monotonic() + REFRESH_WAIT)
return JsonResponse({'added': added})
@csrf_exempt
@require_http_methods(['POST'])
def edit_source(request, pk):
if (err := _auth(request)):
return err
source = Source.objects.filter(user=request.user, pk=pk).first()
body = _body(request)
if source is None or body is None:
return JsonResponse({'error': 'not found'}, status=404)
if 'title' in body and (body['title'] or '').strip():
source.title = body['title'].strip()[:300]
for flag in ('fav', 'hidden', 'notify'):
if isinstance(body.get(flag), bool):
setattr(source, flag, body[flag])
if isinstance(body.get('folders'), list):
source.folders = _folders_for(request.user, body['folders'])
elif 'category' in body and source.kind == Source.BLOG and _kind(body.get('kind')) != Source.NEWS:
# Older clients: one folder in the category field.
cat = _clean_folder(body['category'])
source.folders = _folders_for(request.user, [cat]) if cat else []
if 'category' in body and (source.kind == Source.NEWS or _kind(body.get('kind')) == Source.NEWS):
source.category = (body['category'] or '').strip()[:100]
if body.get('fulltext') in dict(Source.FULLTEXTS):
source.fulltext = body['fulltext']
new_kind = _kind(body.get('kind'))
if source.synthetic and new_kind == Source.NEWS:
return JsonResponse({'error': 'Diese Quelle bleibt ein Blog'}, status=400)
if new_kind and new_kind != source.kind:
# A section becomes a folder and back.
if new_kind == Source.BLOG and source.category and not source.folders:
source.folders, source.category = _folders_for(request.user, [source.category]), ''
elif new_kind == Source.NEWS and source.folders and not source.category:
source.category = source.folders[0][:100]
source.kind = new_kind
# Its items follow; as blog articles they leave their stories.
source.items.update(kind=new_kind, **({'story': None} if new_kind == Source.BLOG else {}))
Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
source.save()
return JsonResponse({'source': _source_data(source)})
@csrf_exempt
@require_http_methods(['POST'])
def delete_source(request, pk):
if (err := _auth(request)):
return err
Source.objects.filter(user=request.user, pk=pk).delete()
Story.objects.filter(user=request.user).exclude(items__gone=False).delete()
return JsonResponse({'ok': True})
@csrf_exempt
@require_http_methods(['GET'])
def export_opml(request):
if (err := _auth(request)):
return err
groups = {}
for s in Source.objects.filter(user=request.user, synthetic=False).order_by('kind', 'category', 'title'):
kind = 'Nachrichten' if s.kind == Source.NEWS else 'Blogs'
place = s.category if s.kind == Source.NEWS else ((s.folders or [''])[0])
label = kind + (f' – {place}' if place else '')
groups.setdefault(label, []).append(s)
lines = ['<?xml version="1.0" encoding="UTF-8"?>', '<opml version="2.0">',
'<head><title>diora Feeds</title></head>', '<body>']
for label, sources in groups.items():
lines.append(f'<outline text={quoteattr(label)}>')
for s in sources:
lines.append(f'<outline type="rss" text={quoteattr(s.title)} xmlUrl={quoteattr(s.url)}'
+ (f' htmlUrl={quoteattr(s.site_url)}' if s.site_url else '') + '/>')
lines.append('</outline>')
lines += ['</body>', '</opml>']
resp = HttpResponse('\n'.join(lines), content_type='text/x-opml; charset=utf-8')
resp['Content-Disposition'] = 'attachment; filename="diora-feeds.opml"'
return resp
@csrf_exempt
@require_http_methods(['POST'])
def refresh_now(request):
if (err := _auth(request)):
return err
refreshed, failed, new = refresh.refresh(Source.objects.filter(user=request.user, synthetic=False),
deadline=time.monotonic() + REFRESH_WAIT)
return JsonResponse({'refreshed': refreshed, 'failed': failed, 'new': new})
# ---- news
@csrf_exempt
@require_http_methods(['GET'])
def briefing(request):
"""Since the briefing was last seen (else the last day): the top stories
(reported by several sources), then the rest by section, newest first."""
if (err := _auth(request)):
return err
s = refresh.settings_for(request.user)
since = s.news_seen_at or (timezone.now() - timedelta(days=1))
qs = Story.objects.filter(user=request.user, last_at__gt=since,
items__gone=False, items__read_at=None).distinct()
stories = _stories(request.user, qs)
stories.sort(key=lambda x: (-x['sources'], -x['size'], x['last_at']), reverse=False)
top = [x for x in stories if x['sources'] >= 2][:BRIEFING_TOP]
top_ids = {x['id'] for x in top}
sections = {}
for x in sorted(stories, key=lambda x: x['last_at'], reverse=True):
if x['id'] not in top_ids:
sections.setdefault(x['category'], []).append(x)
return JsonResponse({
'since': _iso(since), 'total': len(stories), 'top': top,
'sections': [{'category': c, 'count': len(v), 'stories': v[:BRIEFING_PER_CATEGORY]}
for c, v in sorted(sections.items(), key=lambda kv: (kv[0] == CATEGORY_NONE, kv[0]))],
})
@csrf_exempt
@require_http_methods(['POST'])
def briefing_seen(request):
if (err := _auth(request)):
return err
now = timezone.now()
s = refresh.settings_for(request.user)
Item.objects.filter(user=request.user, kind=Source.NEWS, read_at=None, fetched_at__lte=now).update(read_at=now)
s.news_seen_at = now
s.save(update_fields=['news_seen_at'])
return JsonResponse({'news_seen_at': _iso(now)})
@csrf_exempt
@require_http_methods(['GET'])
def news(request):
if (err := _auth(request)):
return err
qs = Story.objects.filter(user=request.user, items__gone=False).distinct().order_by('-last_at')
ids = [int(x) for x in request.GET.get('ids', '').split(',') if x.strip().isdigit()]
if ids:
stories = _stories(request.user, qs.filter(pk__in=ids[:PAGE]))
return JsonResponse({'stories': stories, 'more': False})
before = _when(request.GET.get('before'))
if before:
qs = qs.filter(last_at__lt=before)
category = request.GET.get('category')
if category:
qs = qs.filter(lead__source__category='' if category == CATEGORY_NONE else category)
stories = _stories(request.user, qs[:PAGE])
out = {'stories': stories, 'more': len(stories) == PAGE}
if not before:
topics = _news_text(request.user)['topics']
if category:
# The category's share of each topic.
mine = set(qs.values_list('pk', flat=True)[:500])
topics = [t for t in ({**t, 'ids': [i for i in t['ids'] if i in mine]} for t in topics)
if len(t['ids']) >= textproc.TOPIC_MIN]
out['topics'] = topics
return JsonResponse(out)
# ---- reading help (feeds/textproc.py)
FURNITURE_ITEMS = 40 # a source's recent articles that show its furniture
TEXT_DAYS = 2 # topics and related stories: of these days
TEXT_CACHE = 6 * 3600
def _furniture(source):
key = f'feeds:furniture:{source.id}'
keys = cache.get(key)
if keys is None:
texts = [textproc.plain(c) for c in Item.objects.filter(source=source, gone=False)
.order_by('-published').values_list('content', flat=True)[:FURNITURE_ITEMS]]
keys = textproc.furniture(texts)
cache.set(key, keys, TEXT_CACHE)
return keys
def _clean_text(i):
"""An item's text without its source's furniture, plain."""
return textproc.without_furniture(textproc.plain(i.content) or i.summary, _furniture(i.source))
def _news_text(user):
"""The news of the last days, looked at as a whole: topics [{label, ids}] and related
stories in the other language {story id: [ids]}. Cached until a story changes."""
since = timezone.now() - timedelta(days=TEXT_DAYS)
qs = Story.objects.filter(user=user, last_at__gte=since, items__gone=False).distinct()
latest = qs.order_by('-last_at').values_list('last_at', flat=True).first()
key = f'feeds:newstext:{user.id}:{qs.count()}:{latest.timestamp() if latest else 0}'
got = cache.get(key)
if got is not None:
return got
items = {}
for i in Item.objects.filter(story__in=qs, gone=False).select_related('source'):
items.setdefault(i.story_id, []).append(i)
tl, sl = [], []
for sid, its in items.items():
tl.append((sid, ' '.join(f'{i.title}. {i.summary}' for i in its), {i.source.title for i in its}))
text = ' '.join(f'{i.title}. {_clean_text(i)[:4000]}' for i in its)
sl.append((sid, textproc.language(text), textproc.names(text)))
got = {
'topics': textproc.topics(tl),
'related': {sid: [b for b, _ in lst] for sid, lst in textproc.related(sl).items()},
}
cache.set(key, got, TEXT_CACHE)
return got
@csrf_exempt
@require_http_methods(['GET'])
def story(request, pk):
if (err := _auth(request)):
return err
found = _stories(request.user, Story.objects.filter(user=request.user, pk=pk))
if not found:
return JsonResponse({'error': 'not found'}, status=404)
return JsonResponse({'story': found[0]})
@csrf_exempt
@require_http_methods(['GET'])
def story_overview(request, pk):
"""What a story's reports say together: a summary from all of them, what only one says,
each source's own words, and the same story in the other language."""
if (err := _auth(request)):
return err
s = Story.objects.filter(user=request.user, pk=pk).first()
if s is None:
return JsonResponse({'error': 'not found'}, status=404)
its = list(Item.objects.filter(story=s, gone=False).select_related('source').order_by('published'))
key = f'feeds:overview:{s.id}:{len(its)}:{sum(len(i.content) for i in its)}'
out = cache.get(key)
if out is None:
docs = [(i.source.title, _clean_text(i)) for i in its]
out = {}
if len({src for src, _ in docs}) >= 2:
summary = textproc.summarize(docs)
out['summary'] = [{'source': src, 'text': t} for src, t in summary]
said = {t for _, t in summary}
novel = {src: [t for t in ts if t not in said] for src, ts in textproc.novel(docs).items()}
out['novel'] = {src: ts for src, ts in novel.items() if ts}
out['words'] = textproc.word_choice(docs)
cache.set(key, out, TEXT_CACHE)
rel = _news_text(request.user)['related'].get(s.id, [])
related = _stories(request.user, Story.objects.filter(user=request.user, pk__in=rel)) if rel else []
return JsonResponse({**out, 'related': related})
# ---- blogs, items
@csrf_exempt
@require_http_methods(['GET'])
def blogs(request):
if (err := _auth(request)):
return err
qs = Item.objects.filter(user=request.user, kind=Source.BLOG, gone=False).select_related('source', 'episode_article')
if request.GET.get('source', '').isdigit():
qs = qs.filter(source_id=int(request.GET['source']))
elif 'folder' in request.GET:
# A folder shows all its blogs, the hidden ones too.
name = request.GET['folder']
ids = [s.id for s in Source.objects.filter(user=request.user, kind=Source.BLOG) if _in_folder(s, name)]
qs = qs.filter(source_id__in=ids)
else:
qs = qs.filter(source__hidden=False)
if request.GET.get('fav') == '1':
qs = qs.filter(source__fav=True)
unread_only = request.GET.get('unread') == '1'
if unread_only:
qs = qs.filter(read_at=None)
before = _when(request.GET.get('before'))
if before:
qs = qs.filter(published__lt=before)
# Unread: all of them at once (the page sorts them into days, favorites, rare blogs).
limit = BLOG_UNREAD_MAX if unread_only else BLOG_PAGE
items = list(qs.order_by('-published')[:limit])
return JsonResponse({'items': [_item_data(i) for i in items], 'more': len(items) == limit})
@csrf_exempt
@require_http_methods(['GET'])
def starred(request):
if (err := _auth(request)):
return err
items = Item.objects.filter(user=request.user, starred=True).select_related('source', 'episode_article').order_by('-published')
return JsonResponse({'items': [_item_data(i) for i in items[:200]]})
@csrf_exempt
@require_http_methods(['GET'])
def item(request, pk):
if (err := _auth(request)):
return err
i = Item.objects.filter(user=request.user, pk=pk, gone=False).select_related('source', 'episode_article').first()
if i is None:
return JsonResponse({'error': 'not found'}, status=404)
if refresh.needs_article(i, i.source):
refresh.load_article(i)
d = _item_data(i, content=True)
if not i.source.synthetic:
d['content'] = textproc.html_without_furniture(i.content, _furniture(i.source))
return JsonResponse({'item': d})
@csrf_exempt
@require_http_methods(['POST'])
def mark_read(request):
if (err := _auth(request)):
return err
body = _body(request) or {}
ids = [int(x) for x in body.get('ids') or [] if str(x).isdigit()]
qs = Item.objects.filter(user=request.user, id__in=ids)
n = qs.update(read_at=timezone.now() if body.get('read', True) else None)
return JsonResponse({'updated': n})
@csrf_exempt
@require_http_methods(['POST'])
def item_progress(request, pk):
"""How far the article was read (from any device); from READ_AT on it's read."""
if (err := _auth(request)):
return err
body = _body(request) or {}
try:
progress = min(1.0, max(0.0, float(body.get('progress'))))
except (TypeError, ValueError):
return JsonResponse({'error': 'progress 0-1'}, status=400)
i = Item.objects.filter(user=request.user, pk=pk).first()
if i is None:
return JsonResponse({'error': 'not found'}, status=404)
i.progress = progress
i.progress_at = timezone.now()
if progress >= READ_AT and i.read_at is None:
i.read_at = i.progress_at
i.save(update_fields=['progress', 'progress_at', 'read_at'])
return JsonResponse({'progress': i.progress, 'read': i.read_at is not None})
@csrf_exempt
@require_http_methods(['POST'])
def rename_folder(request):
"""A folder's new name, on every blog that carries it."""
if (err := _auth(request)):
return err
body = _body(request) or {}
old, new = _clean_folder(body.get('name')), _clean_folder(body.get('new'))
if not old or not new:
return JsonResponse({'error': 'Name fehlt'}, status=400)
n = 0
for s in Source.objects.filter(user=request.user, kind=Source.BLOG):
if _in_folder(s, old):
kept = [f for f in s.folders if f.casefold() != old.casefold() and f.casefold() != new.casefold()]
s.folders = kept + [new]
s.save(update_fields=['folders'])
n += 1
return JsonResponse({'blogs': n})
@csrf_exempt
@require_http_methods(['POST'])
def dissolve_folder(request):
"""The folder goes from every blog; the blogs stay subscribed."""
if (err := _auth(request)):
return err
name = _clean_folder((_body(request) or {}).get('name'))
n = 0
for s in Source.objects.filter(user=request.user, kind=Source.BLOG):
if name and _in_folder(s, name):
s.folders = [f for f in s.folders if f.casefold() != name.casefold()]
s.save(update_fields=['folders'])
n += 1
return JsonResponse({'blogs': n})
@csrf_exempt
@require_http_methods(['POST'])
def star(request, pk):
if (err := _auth(request)):
return err
body = _body(request) or {}
n = Item.objects.filter(user=request.user, pk=pk).update(starred=bool(body.get('starred', True)))
return JsonResponse({'updated': n})
@csrf_exempt
@require_http_methods(['GET'])
def notifications(request):
"""What a device should announce: posts marked when they came in, still
unread, of blogs that still have the bell on, after the id the device saw
last. Without `after` (a device's first time) nothing, only where to start.
The same for finished transcripts (`t_after`, `transcripts`, `t_last`; jobs the
user's episodes have, docs/transcription.md)."""
if (err := _auth(request)):
return err
qs = Item.objects.filter(user=request.user, notify=True)
last = qs.order_by('-id').values_list('id', flat=True).first() or 0
# Jobs run in id order, one at a time, so a job never finishes below an id already seen.
# (A job that is done with no data only reflects a feed transcript: no notice for it.)
from podcasts.models import TranscriptionJob
jobs = TranscriptionJob.objects.filter(
episode__feed__user=request.user, state=TranscriptionJob.DONE, data__isnull=False)
t_last = jobs.order_by('-id').values_list('id', flat=True).first() or 0
out = {'items': [], 'last': last, 'transcripts': [], 't_last': t_last}
t_after = request.GET.get('t_after', '')
if t_after.isdigit():
out['transcripts'] = [{
'job_id': j.id, 'episode_id': j.episode_id, 'title': j.episode.title,
'podcast': j.episode.feed.title, 'want_article': j.want_article,
} for j in jobs.filter(id__gt=int(t_after)).select_related('episode__feed').order_by('id')[:50]]
after = request.GET.get('after', '')
if after.isdigit():
items = (qs.filter(id__gt=int(after), read_at=None, gone=False, source__notify=True)
.select_related('source').order_by('-published')[:50])
out['items'] = [_item_data(i) for i in items]
return JsonResponse(out)
@csrf_exempt
@require_http_methods(['POST'])
def save_login(request):
"""Check a subscriber login for the source's site and keep it; then the
articles that came only in part are fetched again."""
if (err := _auth(request)):
return err
body = _body(request) or {}
source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
username = (body.get('username') or '').strip()
password = body.get('password') or ''
if source is None or not username or not password:
return JsonResponse({'error': 'Benutzername und Passwort nötig'}, status=400)
if source.synthetic:
return JsonResponse({'error': 'Diese Quelle hat keine Adresse'}, status=400)
try:
adapter = logins.check_and_store(request.user, source.url, username, password)
except logins.LoginFailed as e:
return JsonResponse({'error': str(e)}, status=400)
# Whatever came in part (or not at all) from this site: fetch again, the newest now.
items = Item.objects.filter(user=request.user, gone=False, content_full=False, url__icontains=adapter.domain)
items.update(fulltext_failed=False)
todo = list(items.select_related('source').order_by('-published')[:40])
refresh._articles(todo, deadline=time.monotonic() + 30)
done = Item.objects.filter(pk__in=[i.pk for i in todo], content_full=True).count()
return JsonResponse({'ok': True, 'login': logins.info(request.user.id, source.url), 'refetched': done})
@csrf_exempt
@require_http_methods(['POST'])
def delete_login(request):
if (err := _auth(request)):
return err
body = _body(request) or {}
source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
adapter = logins.adapter_for(source.url) if source else None
if adapter:
SiteLogin.objects.filter(user=request.user, domain=adapter.domain).delete()
logins.forget(request.user.id, adapter.domain)
return JsonResponse({'ok': True})
@csrf_exempt
@require_http_methods(['POST'])
def settings(request):
if (err := _auth(request)):
return err
body = _body(request) or {}
s = refresh.settings_for(request.user)
days = body.get('news_keep_days')
if isinstance(days, int) and 1 <= days <= 30:
s.news_keep_days = days
s.save(update_fields=['news_keep_days'])
return JsonResponse({'news_keep_days': s.news_keep_days})

10
playwright.config.js
View file

@ -6,7 +6,7 @@
// migrated and seeded with a test user.
const path = require('path');
const { defineConfig, devices } = require('@playwright/test');
const { BASE_URL, PORT, PYTHON, SERVER_ENV } = require('./e2e/env');
const { BASE_URL, SERVER_ENV } = require('./e2e/env');
module.exports = defineConfig({
testDir: './e2e',
@ -18,9 +18,6 @@ module.exports = defineConfig({
baseURL: BASE_URL,
trace: 'retain-on-failure',
screenshot: 'only-on-failure',
// A new service worker taking over reloads the page (28-serviceworker.js),
// mid-test. It isn't what these tests are about.
serviceWorkers: 'block',
},
projects: [
{
@ -30,10 +27,9 @@ module.exports = defineConfig({
],
globalSetup: require.resolve('./e2e/global-setup.js'),
webServer: {
command: `${PYTHON} manage.py runserver 127.0.0.1:${PORT} --noreload`,
command: 'python3 manage.py runserver 127.0.0.1:8000 --noreload',
url: BASE_URL + '/accounts/login/',
// Never reuse: a server already on the port has someone's real database behind it.
reuseExistingServer: false,
reuseExistingServer: !process.env.CI,
timeout: 30_000,
cwd: path.join(__dirname),
env: SERVER_ENV,

119
podcasts/article.py
View file

@ -1,119 +0,0 @@
"""The article written from an episode's transcript (docs/transcription.md): its text
is a feeds.Item in the user's favourite blog "Podcast-Artikel"."""
import hashlib
import json
from django.db import transaction
from django.http import JsonResponse
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from feeds import fetch
from feeds.models import Item, Source
from .models import EpisodeArticle, PodcastEpisode, TranscriptionJob
SOURCE_URL = 'diora://podcast-artikel'
SOURCE_TITLE = 'Podcast-Artikel'
TITLE_MAX = 300
SUMMARY_MAX = 2000
BODY_MAX = 1_000_000 # characters of body_html
MODEL_MAX = 100
KIND_MAX = 20
def _auth(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
return None
def _body(request):
try:
body = json.loads(request.body or b'{}')
except (ValueError, UnicodeDecodeError):
return None
return body if isinstance(body, dict) else None
def article_source(user):
"""The user's blog "Podcast-Artikel", made on first use."""
source, _ = Source.objects.get_or_create(user=user, url=SOURCE_URL, defaults={
'title': SOURCE_TITLE, 'kind': Source.BLOG, 'fav': True, 'synthetic': True})
return source
def _text(value, limit):
return value.strip() if isinstance(value, str) and len(value.strip()) <= limit else None
@csrf_exempt
@require_http_methods(['POST', 'DELETE'])
def episode_article(request, pk):
if (err := _auth(request)):
return err
ep = PodcastEpisode.objects.filter(pk=pk, feed__user=request.user).select_related('feed').first()
if ep is None:
return JsonResponse({'error': 'not found'}, status=404)
if request.method == 'DELETE':
art = EpisodeArticle.objects.filter(episode=ep).select_related('item').first()
if art is not None:
art.item.delete() # the article goes with it (cascade)
return JsonResponse({'ok': True})
body = _body(request)
if body is None:
return JsonResponse({'error': 'invalid JSON'}, status=400)
title = _text(body.get('title'), TITLE_MAX)
if not title:
return JsonResponse({'error': 'title needed (at most %d characters)' % TITLE_MAX}, status=400)
html_in = body.get('body_html')
if not isinstance(html_in, str) or not html_in.strip():
return JsonResponse({'error': 'body_html needed'}, status=400)
if len(html_in) > BODY_MAX:
return JsonResponse({'error': 'body_html too large'}, status=400)
summary = body.get('summary') or ''
if not isinstance(summary, str) or len(summary) > SUMMARY_MAX:
return JsonResponse({'error': 'summary too long'}, status=400)
model = body.get('model') or ''
kind = body.get('kind') or ''
if not isinstance(model, str) or len(model) > MODEL_MAX or not isinstance(kind, str) or len(kind) > KIND_MAX:
return JsonResponse({'error': 'model or kind too long'}, status=400)
url = getattr(ep, 'link', '') or ep.audio_url
content = fetch.clean_html(html_in, url)
if not fetch.plain(content):
return JsonResponse({'error': 'body_html empty'}, status=400)
fields = {
'url': url[:2000],
'title': title,
'author': (ep.feed.author or '')[:300],
'published': timezone.now(),
'summary': fetch.plain(summary),
'content': content,
'content_full': True, # nothing to fetch from a page
'words': fetch.word_count(content),
'image_url': (ep.artwork_url or ep.feed.artwork_url or '')[:2000],
'read_at': None, 'progress': 0, 'progress_at': None,
}
with transaction.atomic():
source = article_source(request.user)
art = EpisodeArticle.objects.filter(episode=ep).select_related('item').first()
if art is not None:
item = art.item
for k, v in fields.items():
setattr(item, k, v)
item.source = source
item.save()
art.model, art.kind = model, kind
art.save(update_fields=['model', 'kind'])
else:
guid = hashlib.sha256(f'diora-article:{ep.pk}'.encode()).hexdigest()
item = Item.objects.create(source=source, user=request.user, kind=Source.BLOG,
guid_hash=guid, **fields)
EpisodeArticle.objects.create(episode=ep, item=item, model=model, kind=kind)
# Saving fulfils the wish and frees the reservation.
TranscriptionJob.objects.filter(episode=ep).update(want_article=False, claimed_by='', claimed_until=None)
return JsonResponse({'item_id': item.id})

213
podcasts/audiobooks.py
View file

@ -1,213 +0,0 @@
"""Audiobooks: podcast subscriptions marked as one (PodcastFeed.kind), listened to like
a book in the library — chapters in order, one progress, bookmarks (docs/audiobooks.md)."""
import json
import re
from datetime import datetime, timezone as dt_timezone
from django.db.models import Max, Q
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from .models import EpisodeBookmark, EpisodeProgress, PodcastEpisode, PodcastFeed
MAX_NOTE = 1000
_FAR = datetime.max.replace(tzinfo=dt_timezone.utc)
_DIGITS = re.compile(r'(\d+)')
CHAPTER_FIELDS = ('id', 'title', 'audio_url', 'duration_seconds', 'artwork_url', 'pub_date', 'episode_number')
def _natural(title):
"""'Kapitel 2' before 'Kapitel 10'."""
return [int(part) if part.isdigit() else part.casefold() for part in _DIGITS.split(title or '')]
def chapter_key(ep):
number = ep['episode_number']
return (number is None, number or 0, ep['pub_date'] or _FAR, _natural(ep['title']), ep['id'])
def _chapters(user, feed):
"""The feed's episodes in chapter order, with the user's place and played mark."""
eps = sorted(PodcastEpisode.objects.filter(feed=feed).values(*CHAPTER_FIELDS), key=chapter_key)
progress = {
p['episode_id']: p for p in EpisodeProgress.objects.filter(user=user, episode__feed=feed)
.values('episode_id', 'position_seconds', 'played', 'updated_at')
}
out = []
for i, ep in enumerate(eps):
p = progress.get(ep['id']) or {}
out.append({
'episode__id': ep['id'],
'episode__title': ep['title'],
'episode__audio_url': ep['audio_url'],
'episode__duration_seconds': ep['duration_seconds'],
'episode__artwork_url': ep['artwork_url'],
'episode__feed__id': feed.id,
'episode__feed__title': feed.title,
'episode__feed__artwork_url': feed.artwork_url,
'episode__pub_date': ep['pub_date'].isoformat() if ep['pub_date'] else None,
'position_seconds': p.get('position_seconds') or 0,
'played': bool(p.get('played')),
'index': i,
# When it was last listened to (a place or played), for where the book goes on.
'_heard_at': p.get('updated_at') if (p.get('played') or p.get('position_seconds')) else None,
})
return out
def _next(chapters):
"""Where the book goes on: in the chapter listened to last if it isn't finished, else
in the first unfinished one after it (or before it, when the end was reached). Never
listened: the first unfinished one. None: all finished."""
open_ = [ch for ch in chapters if not ch['played']]
if not open_:
return None
heard = [ch for ch in chapters if ch['_heard_at']]
if heard:
last = max(heard, key=lambda ch: ch['_heard_at'])
after = [ch for ch in open_ if ch['index'] >= last['index']]
ch = after[0] if after else open_[0]
else:
ch = open_[0]
return {'episode_id': ch['episode__id'], 'index': ch['index'], 'position_seconds': ch['position_seconds']}
def _book(feed, chapters, last_played_at):
total = heard = done = 0
for ch in chapters:
dur = max(ch['episode__duration_seconds'] or 0, 0)
total += dur
if ch['played']:
done += 1
heard += dur
elif dur:
heard += min(ch['position_seconds'], dur)
nxt = _next(chapters)
return {
'feed_id': feed.id, 'title': feed.title, 'author': feed.author, 'artwork_url': feed.artwork_url,
'folder': feed.folder, 'folders': feed.folders or [],
'chapters': len(chapters), 'done': done, 'total_seconds': total, 'heard_seconds': heard,
'next': nxt, 'last_played_at': last_played_at.isoformat() if last_played_at else None,
}
def _bookmark_json(bm):
return {'id': bm.id, 'episode_id': bm.episode_id, 'position_ms': bm.position_ms, 'note': bm.note,
'created_at': bm.created_at.isoformat()}
def _last_played(user, feed_ids):
"""When each book was last listened to: a place or a played mark, not the rows that
only took its chapters out of "Neu" when it was marked."""
return dict(
EpisodeProgress.objects.filter(user=user, episode__feed_id__in=feed_ids)
.filter(Q(position_seconds__gt=0) | Q(played=True))
.values('episode__feed_id').annotate(at=Max('updated_at')).values_list('episode__feed_id', 'at')
)
@csrf_exempt
@require_http_methods(['GET'])
def audiobook_list(request):
"""The user's audiobooks with their progress, the last listened first."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
feeds = list(PodcastFeed.objects.filter(user=request.user, kind=PodcastFeed.AUDIOBOOK))
last = _last_played(request.user, [f.id for f in feeds])
books = [_book(f, _chapters(request.user, f), last.get(f.id)) for f in feeds]
books.sort(key=lambda b: b['title'].casefold())
books.sort(key=lambda b: b['last_played_at'] or '', reverse=True) # never listened: last
return JsonResponse({'books': books})
@csrf_exempt
@require_http_methods(['GET'])
def audiobook_detail(request, feed_id):
"""One audiobook: its progress, its chapters (shaped like queue entries, for the
player) and its bookmarks in chapter order."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
feed = PodcastFeed.objects.filter(pk=feed_id, user=request.user, kind=PodcastFeed.AUDIOBOOK).first()
if feed is None:
return JsonResponse({'error': 'not found'}, status=404)
chapters = _chapters(request.user, feed)
order = {ch['episode__id']: ch['index'] for ch in chapters}
marks = sorted(EpisodeBookmark.objects.filter(user=request.user, episode__feed=feed),
key=lambda bm: (order.get(bm.episode_id, 0), bm.position_ms, bm.id))
book = _book(feed, chapters, _last_played(request.user, [feed.id]).get(feed.id))
for ch in chapters:
del ch['_heard_at']
return JsonResponse({
'book': book,
'chapters': chapters,
'bookmarks': [_bookmark_json(bm) for bm in marks],
})
def _body(request):
try:
body = json.loads(request.body or b'{}')
except (json.JSONDecodeError, ValueError):
return None
return body if isinstance(body, dict) else None
def _note(value):
if value is None:
return ''
if not isinstance(value, str):
raise ValueError
return value.strip()[:MAX_NOTE]
@csrf_exempt
@require_http_methods(['POST'])
def bookmark_add(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
body = _body(request)
if body is None:
return JsonResponse({'error': 'invalid JSON'}, status=400)
episode = PodcastEpisode.objects.filter(pk=body.get('episode_id'), feed__user=request.user).first() \
if isinstance(body.get('episode_id'), int) else None
if episode is None:
return JsonResponse({'error': 'not found'}, status=404)
pos = body.get('position_ms')
if not isinstance(pos, (int, float)) or isinstance(pos, bool) or not 0 <= pos < 2 ** 31:
return JsonResponse({'error': 'invalid position_ms'}, status=400)
try:
note = _note(body.get('note'))
except ValueError:
return JsonResponse({'error': 'invalid note'}, status=400)
bm = EpisodeBookmark.objects.create(user=request.user, episode=episode, position_ms=int(pos), note=note)
return JsonResponse({'ok': True, 'bookmark': _bookmark_json(bm)})
@csrf_exempt
@require_http_methods(['POST'])
def bookmark_update(request, pk):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
bm = EpisodeBookmark.objects.filter(pk=pk, user=request.user).first()
body = _body(request)
if bm is None:
return JsonResponse({'error': 'not found'}, status=404)
if body is None:
return JsonResponse({'error': 'invalid JSON'}, status=400)
try:
bm.note = _note(body.get('note'))
except ValueError:
return JsonResponse({'error': 'invalid note'}, status=400)
bm.save(update_fields=['note'])
return JsonResponse({'ok': True, 'bookmark': _bookmark_json(bm)})
@csrf_exempt
@require_http_methods(['POST'])
def bookmark_delete(request, pk):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
EpisodeBookmark.objects.filter(pk=pk, user=request.user).delete()
return JsonResponse({'ok': True})

15
podcasts/chapters.py
View file

@ -128,28 +128,25 @@ FetchUnavailable = ChaptersUnavailable
FetchInvalid = ChaptersInvalid
def _open(url, headers, method='GET', timeout=CHAPTERS_TIMEOUT, **kwargs):
"""`method` (GET or POST) `url` with every redirect hop checked; the streamed response of
the final hop, which the caller has to close. `kwargs` (data, files) go to
requests; a POST only follows 307/308, which keep the method and body.
def _open(url, headers):
"""GET `url` with every redirect hop checked; the streamed response of
the final hop, which the caller has to close.
Raises ChaptersInvalid for permanent problems and ChaptersUnavailable for
ones that may go away (network trouble, server errors).
"""
redirects = (301, 302, 303, 307, 308) if method == 'GET' else (307, 308)
for _ in range(CHAPTERS_MAX_REDIRECTS + 1):
try:
assert_safe_url(url)
except WebDAVInputError as e:
raise ChaptersInvalid(str(e))
try:
send = requests.get if method == 'GET' else requests.post
r = send(url, timeout=timeout, allow_redirects=False, stream=True, headers=headers, **kwargs)
r = requests.get(url, timeout=CHAPTERS_TIMEOUT, allow_redirects=False, stream=True, headers=headers)
except requests.RequestException as e:
raise ChaptersUnavailable(e.__class__.__name__)
try:
_assert_peer_is_safe(r)
if r.status_code in redirects:
if r.status_code in (301, 302, 303, 307, 308):
location = r.headers.get('Location')
if not location:
raise ChaptersInvalid('redirect without a location')
@ -158,7 +155,7 @@ def _open(url, headers, method='GET', timeout=CHAPTERS_TIMEOUT, **kwargs):
continue
if r.status_code >= 500 or r.status_code == 429:
raise ChaptersUnavailable(f'HTTP {r.status_code}')
if r.status_code >= 400 or r.status_code in (301, 302, 303):
if r.status_code >= 400:
raise ChaptersInvalid(f'HTTP {r.status_code}')
except WebDAVInputError as e:
r.close()

530
podcasts/discover.py
View file

@ -1,530 +0,0 @@
"""Stöbern (docs/podcasts.md): finding episodes in the user's own podcasts — search in titles
and shownotes, similar episodes, people, topics, forgotten ones — and one controlled step out:
where people from them are guests elsewhere (Apple's episode search, fyyd), and podcasts like
the most listened ones (fyyd's recommendations). An episode from outside goes onto the
Merkliste; its podcast becomes an "Auf Abruf" subscription.
The full-text index is SQLite FTS5, contentless (only the index, no second copy of 100 MB of
shownotes): podcasts_episode_fts, rowid = the episode's id, filled by index_new() after every
refresh (new episodes have higher ids). Episodes deleted since stay in the index; every query
joins the episodes, so they never show.
"""
import html
import logging
import math
import random
import re
import unicodedata
import urllib.parse
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
import requests
from django.core.cache import cache
from django.db import connection
from django.db.models import Q
from django.http import JsonResponse
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from . import people as people_mod
from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastSettings
log = logging.getLogger(__name__)
FTS = 'podcasts_episode_fts'
VOCAB = 'podcasts_episode_fts_vocab'
BODY_CHARS = 6000 # of the shownotes, per episode
INDEX_BATCH = 2000
SIMILAR_TERMS = 10
HTTP_TIMEOUT = 6
USER_AGENT = 'diora podcast discovery'
CREATE_SQL = [
f"CREATE VIRTUAL TABLE IF NOT EXISTS {FTS} USING fts5(title, body, content='', contentless_delete=1, "
f"tokenize='unicode61 remove_diacritics 2')",
f"CREATE VIRTUAL TABLE IF NOT EXISTS {VOCAB} USING fts5vocab({FTS}, 'row')",
]
DROP_SQL = [f'DROP TABLE IF EXISTS {VOCAB}', f'DROP TABLE IF EXISTS {FTS}']
def _plain(text):
text = re.sub(r'<[^>]+>', ' ', text or '')
return re.sub(r'\s+', ' ', html.unescape(text)).strip()
def _fold(word):
"""A word as the index keeps it: lower case, without diacritics."""
w = unicodedata.normalize('NFKD', word.lower())
return ''.join(c for c in w if not unicodedata.combining(c))
_WORD = re.compile(r'\w+', re.UNICODE)
def _words(text):
return [_fold(w) for w in _WORD.findall(text or '') if not w.isdigit()]
def _quoted(term):
return '"' + term.replace('"', '""') + '"'
# ---- the index ------------------------------------------------------------------------------------
def index_new(limit=None):
"""Index the episodes newer than the newest one indexed. Returns how many."""
with connection.cursor() as cur:
cur.execute(f'SELECT max(rowid) FROM {FTS}')
last = cur.fetchone()[0] or 0
done = 0
while True:
rows = list(PodcastEpisode.objects.filter(id__gt=last).order_by('id')
.values_list('id', 'title', 'description')[:INDEX_BATCH])
if not rows:
return done
with connection.cursor() as cur:
cur.executemany(f'INSERT OR REPLACE INTO {FTS}(rowid, title, body) VALUES (%s, %s, %s)',
[(i, t or '', _plain(d)[:BODY_CHARS]) for i, t, d in rows])
done += len(rows)
last = rows[-1][0]
if limit and done >= limit:
return done
def _user_episode_sql(user, stream=False, other_than_feed=None):
"""SQL and params restricting fts rowids to the user's podcast episodes."""
sql = (f' JOIN podcasts_podcastepisode e ON e.id = {FTS}.rowid'
f' JOIN podcasts_podcastfeed pf ON pf.id = e.feed_id'
f' WHERE pf.user_id = %s AND pf.kind = %s')
params = [user.id, PodcastFeed.PODCAST]
if stream:
sql += ' AND pf.new_episodes != %s'
params.append(PodcastFeed.NEW_IGNORE)
if other_than_feed is not None:
sql += ' AND e.feed_id != %s'
params.append(other_than_feed)
return sql, params
def search_ids(user, q, stream=False, limit=50, offset=0):
"""Episode ids of the user's podcasts matching all words of q (the last one as a prefix,
while typing), titles weighing three times the shownotes, best first."""
words = _words(q)[:12]
if not words:
return []
match = ' '.join(_quoted(w) for w in words[:-1]) + (' ' if len(words) > 1 else '') + _quoted(words[-1]) + '*'
where, params = _user_episode_sql(user, stream)
with connection.cursor() as cur:
cur.execute(f'SELECT {FTS}.rowid FROM {FTS}{where} AND {FTS} MATCH %s '
f'ORDER BY bm25({FTS}, 3.0, 1.0) LIMIT %s OFFSET %s', params + [match, limit, offset])
return [r[0] for r in cur.fetchall()]
def similar_ids(user, episode, limit=20):
"""Episodes of the user's other podcasts most like `episode`: its rarest telling words
(tf·idf over the index) asked for together, best match first."""
tf = Counter()
for w in _words(episode.title):
tf[w] += 3
for w in _words(_plain(episode.description)[:BODY_CHARS]):
tf[w] += 1
terms = [w for w in tf if len(w) > 2]
if not terms:
return []
with connection.cursor() as cur:
cur.execute(f'SELECT count(*) FROM podcasts_podcastepisode')
n = cur.fetchone()[0] or 1
df = {}
for i in range(0, len(terms), 500):
chunk = terms[i:i + 500]
cur.execute(f'SELECT term, doc FROM {VOCAB} WHERE term IN ({",".join(["%s"] * len(chunk))})', chunk)
df.update(dict(cur.fetchall()))
scored = []
for w, c in tf.items():
d = df.get(w, 0)
if d < 2 or d > max(n * 0.05, 10): # only here, or everywhere: says nothing
continue
scored.append((c * math.log(n / d), w))
scored.sort(reverse=True)
best = [w for _, w in scored[:SIMILAR_TERMS]]
if not best:
return []
where, params = _user_episode_sql(user, other_than_feed=episode.feed_id)
with connection.cursor() as cur:
cur.execute(f'SELECT {FTS}.rowid FROM {FTS}{where} AND {FTS} MATCH %s '
f'ORDER BY bm25({FTS}, 3.0, 1.0) LIMIT %s',
params + [' OR '.join(_quoted(w) for w in best), limit])
return [r[0] for r in cur.fetchall()]
# ---- helpers ---------------------------------------------------------------------------------------
def _rows(user, ids):
"""The episodes as the episode lists give them, in the order of ids."""
from .views import _episode_list_data
if not ids:
return []
eps, _ = _episode_list_data(user, ids=ids, limit=len(ids))
by_id = {e['id']: e for e in eps}
return [by_id[i] for i in ids if i in by_id]
def _heard_by_feed(user):
"""{feed_id: episodes heard in the last 12 months} (as heard_365 in state/)."""
from .views import HEARD_MIN_SECONDS
since = timezone.now() - timedelta(days=365)
out = Counter()
for fid in (EpisodeProgress.objects.filter(user=user, updated_at__gte=since)
.filter(Q(played=True) | Q(position_seconds__gt=HEARD_MIN_SECONDS))
.values_list('episode__feed_id', flat=True)):
out[fid] += 1
return out
def _norm_url(url):
u = (url or '').strip().lower()
u = re.sub(r'^https?://', '', u)
return u.rstrip('/')
def _get_json(url, **params):
try:
r = requests.get(url, params=params, timeout=HTTP_TIMEOUT, headers={'User-Agent': USER_AGENT})
r.raise_for_status()
return r.json()
except Exception as e: # noqa: BLE001 - a service away is no error for the user
log.info('discover: %s: %s', url, e)
return None
# ---- people ----------------------------------------------------------------------------------------
PEOPLE_TTL = 12 * 3600
def people_of(user):
key = f'podcasts:people:{user.id}'
got = cache.get(key)
if got is None:
rows = list(PodcastEpisode.objects.filter(feed__user=user, feed__kind=PodcastFeed.PODCAST)
.values_list('id', 'feed_id', 'title'))
got = people_mod.people(rows, _heard_by_feed(user))[:200]
cache.set(key, got, PEOPLE_TTL)
return got
def person_ids(user, name, limit=200):
return list(PodcastEpisode.objects.filter(feed__user=user, feed__kind=PodcastFeed.PODCAST, title__icontains=name)
.order_by('-pub_date', '-id').values_list('id', flat=True)[:limit])
def elsewhere(user, name):
"""Episodes with `name` in podcasts the user doesn't have: Apple's episode search and fyyd's,
those that name the person in their title. [{source, podcast, feed_url, title, audio_url,
guid, pub_date, duration_seconds, artwork_url, link}]"""
key = f'podcasts:elsewhere:{user.id}:' + re.sub(r'\W+', '_', _fold(name))
got = cache.get(key)
if got is not None:
return got
mine = {_norm_url(u) for u in PodcastFeed.objects.filter(user=user).values_list('rss_url', flat=True)}
mine_titles = {t.casefold() for t in PodcastFeed.objects.filter(user=user).values_list('title', flat=True)}
folded = _fold(name)
out, seen = [], set()
def take(item):
if folded not in _fold(item['title']) or not item['feed_url'] or not item['audio_url']:
return
if _norm_url(item['feed_url']) in mine or item['podcast'].casefold() in mine_titles:
return
k = (_fold(item['title']), _fold(item['podcast']))
if k in seen:
return
seen.add(k)
out.append(item)
apple = _get_json('https://itunes.apple.com/search', term=name, media='podcast', entity='podcastEpisode',
limit=50, country='DE') or {}
for r in apple.get('results', []):
take({'source': 'apple', 'podcast': r.get('collectionName', ''), 'feed_url': r.get('feedUrl', ''),
'title': r.get('trackName', ''), 'audio_url': r.get('episodeUrl', ''), 'guid': r.get('episodeGuid', ''),
'pub_date': r.get('releaseDate'), 'duration_seconds': int((r.get('trackTimeMillis') or 0) / 1000),
'artwork_url': r.get('artworkUrl600') or r.get('artworkUrl160') or '',
'link': r.get('trackViewUrl', '')})
fy = _get_json('https://api.fyyd.de/0.2/search/episode', term=f'"{name}"', count=50) or {}
eps = [e for e in (fy.get('data') or []) if isinstance(e, dict) and folded in _fold(e.get('title', ''))]
feeds = {}
for pid in list(dict.fromkeys(e.get('podcast_id') for e in eps))[:12]:
p = _get_json('https://api.fyyd.de/0.2/podcast', podcast_id=pid) or {}
if isinstance(p.get('data'), dict):
feeds[pid] = p['data']
for e in eps:
p = feeds.get(e.get('podcast_id'))
if not p:
continue
take({'source': 'fyyd', 'podcast': p.get('title', ''), 'feed_url': p.get('xmlURL', ''),
'title': html.unescape(e.get('title', '')), 'audio_url': e.get('enclosure', ''), 'guid': e.get('guid', ''),
'pub_date': e.get('pubdate'), 'duration_seconds': int(e.get('duration') or 0),
'artwork_url': e.get('imgURL') or p.get('imgURL') or '', 'link': e.get('url_fyyd', '')})
out.sort(key=lambda i: i['pub_date'] or '', reverse=True)
cache.set(key, out, 6 * 3600)
return out
# ---- topics, forgotten, recommendations ------------------------------------------------------------
def topics_of(user, days=7):
"""Topics of the Strom's last days (feeds.textproc.topics, as for the news):
[{'label', 'ids'}], the largest first."""
from feeds import textproc
since = timezone.now() - timedelta(days=days)
# Titles only: the shownotes bring every podcast's furniture ("Link", "Abo", "Mediathek").
eps = list(PodcastEpisode.objects.filter(feed__user=user, feed__kind=PodcastFeed.PODCAST, pub_date__gte=since)
.exclude(feed__new_episodes=PodcastFeed.NEW_IGNORE)
.values_list('id', 'title', 'feed__title'))
stories = [(i, t, {ft}) for i, t, ft in eps]
return textproc.topics(stories, max_topics=10)
FORGOTTEN_DAYS = 60
def forgotten(user):
"""Unheard older episodes of the podcasts the user listens to most, and of the rare ones —
a handful each, the same all day."""
from .views import _intervals
rnd = random.Random(f'{user.id}:{timezone.localdate()}')
heard = _heard_by_feed(user)
started = set(EpisodeProgress.objects.filter(user=user).filter(Q(played=True) | Q(position_seconds__gt=0))
.values_list('episode_id', flat=True))
before = timezone.now() - timedelta(days=FORGOTTEN_DAYS)
def pick(feed_ids, n):
ids = [i for i in PodcastEpisode.objects.filter(feed_id__in=feed_ids, pub_date__lt=before)
.values_list('id', flat=True) if i not in started]
return rnd.sample(ids, min(n, len(ids)))
mine = list(PodcastFeed.objects.filter(user=user, kind=PodcastFeed.PODCAST)
.exclude(new_episodes=PodcastFeed.NEW_IGNORE).values_list('id', flat=True))
loved = [f for f in mine if heard.get(f, 0) >= 3]
intervals = _intervals(user)
rare = [f for f in mine if (intervals.get(f) or 0) >= 30]
return {'loved': _rows(user, pick(loved, 15)), 'rare': _rows(user, pick(rare, 10))}
RECOMMEND_FROM = 10 # the most listened podcasts asked about
RECOMMEND_TTL = 24 * 3600
def _fyyd_id(feed):
got = _get_json('https://api.fyyd.de/0.2/search/podcast', title=feed['title'], count=5) or {}
want = _norm_url(feed['rss_url'])
for p in got.get('data') or []:
if isinstance(p, dict) and (_norm_url(p.get('xmlURL')) == want or (p.get('title') or '').casefold() == feed['title'].casefold()):
return p.get('id')
return None
def recommend(user):
"""Podcasts like the user's most listened (fyyd's recommendations), the ones recommended for
several of them first; none the user has. [{title, feed_url, artwork_url, description, link,
because: [titles], count}]"""
key = f'podcasts:recommend:{user.id}'
got = cache.get(key)
if got is not None:
return got
heard = _heard_by_feed(user)
feeds = list(PodcastFeed.objects.filter(user=user, kind=PodcastFeed.PODCAST).values('id', 'title', 'rss_url'))
top = sorted((f for f in feeds if heard.get(f['id'])), key=lambda f: -heard[f['id']])[:RECOMMEND_FROM]
mine = {_norm_url(f['rss_url']) for f in feeds}
mine_titles = {f['title'].casefold() for f in feeds}
def ask(feed):
pid = _fyyd_id(feed)
if not pid:
return feed, []
r = _get_json('https://api.fyyd.de/0.2/podcast/recommend', podcast_id=pid, count=12) or {}
return feed, [p for p in r.get('data') or [] if isinstance(p, dict)]
found = {}
with ThreadPoolExecutor(max_workers=5) as pool:
for feed, recs in pool.map(ask, top):
for p in recs:
url = p.get('xmlURL') or ''
if not url or _norm_url(url) in mine or (p.get('title') or '').casefold() in mine_titles:
continue
k = _norm_url(url)
item = found.setdefault(k, {'title': p.get('title', ''), 'feed_url': url, 'artwork_url': p.get('imgURL') or '',
'description': _plain(p.get('description'))[:300],
'link': p.get('htmlURL') or '', 'because': [], 'count': 0})
if feed['title'] not in item['because']:
item['because'].append(feed['title'])
item['count'] += 1
out = sorted(found.values(), key=lambda i: (-i['count'], i['title'].casefold()))[:30]
cache.set(key, out, RECOMMEND_TTL)
return out
# ---- an episode from outside -------------------------------------------------------------------------
def take(user, item):
"""An episode from elsewhere onto the Merkliste: its podcast becomes an "Auf Abruf"
subscription (or is one already), the episode is found in it — or, when the feed no longer
lists it, made from what the search said. Returns (episode, created_feed)."""
from .views import _later_add, _refresh_feed
url = (item.get('feed_url') or '').strip()
feed = PodcastFeed.objects.filter(user=user, rss_url=url).first()
created = False
if feed is None:
feed = PodcastFeed.objects.create(
user=user, rss_url=url, title=(item.get('podcast') or url)[:300], artwork_url=(item.get('artwork_url') or '')[:1000],
**{**PodcastSettings.new_feed_defaults(user), 'new_episodes': PodcastFeed.NEW_IGNORE})
created = True
try:
_refresh_feed(feed)
except Exception as e: # noqa: BLE001 - the episode is made from the search below
log.info('discover: %s: %s', url, e)
eps = feed.episodes.all()
ep = None
if item.get('guid'):
ep = eps.filter(guid=item['guid']).first()
if ep is None and item.get('audio_url'):
ep = eps.filter(audio_url=item['audio_url']).first()
if ep is None and item.get('title'):
ep = eps.filter(title__iexact=item['title']).first()
if ep is None:
when = parse_datetime(item.get('pub_date') or '') if item.get('pub_date') else None
ep = PodcastEpisode.objects.create(
feed=feed, guid=(item.get('guid') or item.get('audio_url') or item.get('title'))[:1000],
title=(item.get('title') or '')[:500], audio_url=item.get('audio_url', '')[:1000],
duration_seconds=int(item.get('duration_seconds') or 0), pub_date=when,
artwork_url=(item.get('artwork_url') or '')[:1000])
_later_add(user, [ep.id])
try:
index_new()
except Exception: # noqa: BLE001
pass
return ep, created
# ---- views ---------------------------------------------------------------------------------------------
def _auth(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
return None
def _int(request, name, default, low, high):
try:
return min(max(int(request.GET.get(name, default)), low), high)
except ValueError:
return default
@require_http_methods(['GET'])
def search_view(request):
"""?q=…&stream=1&offset=&limit=: episodes whose title or shownotes have every word."""
err = _auth(request)
if err:
return err
ids = search_ids(request.user, request.GET.get('q', ''), stream=request.GET.get('stream') == '1',
limit=_int(request, 'limit', 50, 1, 200), offset=_int(request, 'offset', 0, 0, 10 ** 6))
return JsonResponse({'episodes': _rows(request.user, ids)})
@require_http_methods(['GET'])
def similar_view(request, pk):
err = _auth(request)
if err:
return err
ep = PodcastEpisode.objects.filter(pk=pk, feed__user=request.user).first()
if ep is None:
return JsonResponse({'error': 'not found'}, status=404)
return JsonResponse({'episodes': _rows(request.user, similar_ids(request.user, ep, _int(request, 'limit', 20, 1, 50)))})
@require_http_methods(['GET'])
def people_view(request):
"""The people in the user's podcasts: [{name, episodes, feeds, score}], the most relevant first."""
err = _auth(request)
if err:
return err
return JsonResponse({'people': people_of(request.user)[:_int(request, 'limit', 80, 1, 200)]})
@require_http_methods(['GET'])
def person_view(request):
"""?name=: the person's episodes in the user's podcasts, newest first."""
err = _auth(request)
if err:
return err
name = request.GET.get('name', '').strip()
if len(name) < 3:
return JsonResponse({'error': 'name required'}, status=400)
return JsonResponse({'name': name, 'episodes': _rows(request.user, person_ids(request.user, name))})
@require_http_methods(['GET'])
def elsewhere_view(request):
"""?name=: episodes with the person in podcasts the user doesn't have."""
err = _auth(request)
if err:
return err
name = request.GET.get('name', '').strip()
if len(name) < 3:
return JsonResponse({'error': 'name required'}, status=400)
return JsonResponse({'name': name, 'episodes': elsewhere(request.user, name)})
@require_http_methods(['GET'])
def topics_view(request):
"""?days=7: topics of the Strom's last days: [{label, ids}], plus their episodes."""
err = _auth(request)
if err:
return err
days = _int(request, 'days', 7, 1, 30)
key = f'podcasts:topics:{request.user.id}:{days}'
got = cache.get(key)
if got is None:
got = topics_of(request.user, days)
cache.set(key, got, 3600)
return JsonResponse({'topics': got})
@require_http_methods(['GET'])
def forgotten_view(request):
err = _auth(request)
if err:
return err
return JsonResponse(forgotten(request.user))
@require_http_methods(['GET'])
def recommend_view(request):
err = _auth(request)
if err:
return err
return JsonResponse({'podcasts': recommend(request.user)})
@csrf_exempt
@require_http_methods(['POST'])
def take_view(request):
"""{podcast, feed_url, title, audio_url, guid, pub_date, duration_seconds, artwork_url} (an
item of elsewhere/): onto the Merkliste; returns {episode_id, feed_id, subscribed}."""
import json
err = _auth(request)
if err:
return err
try:
item = json.loads(request.body or b'{}')
except (json.JSONDecodeError, ValueError):
return JsonResponse({'error': 'invalid JSON'}, status=400)
if not isinstance(item, dict) or not re.match(r'^https?://', item.get('feed_url') or '') or not item.get('audio_url'):
return JsonResponse({'error': 'feed_url and audio_url required'}, status=400)
ep, created = take(request.user, item)
return JsonResponse({'ok': True, 'episode_id': ep.id, 'feed_id': ep.feed_id, 'subscribed': created})

116
podcasts/folders.py
View file

@ -1,116 +0,0 @@
"""Folders of podcasts and audiobooks (docs/library.md): like tags, any number per
subscription (PodcastFeed.folders), shown as filter chips in their order
(PodcastSettings.folder_order). Renaming onto an existing name merges; deleting
a folder leaves its subscriptions where they are."""
import json
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from .models import PodcastFeed, PodcastSettings
MAX_FOLDERS = 20
MAX_NAME = 100
def clean_name(value):
return ' '.join(str(value or '').split())[:MAX_NAME]
def clean_list(value):
"""A list of folder names: trimmed, without empties and doubles (first spelling wins)."""
if not isinstance(value, list):
raise ValueError
out, seen = [], set()
for v in value:
if not isinstance(v, str):
raise ValueError
name = clean_name(v)
if name and name.casefold() not in seen:
seen.add(name.casefold())
out.append(name)
return out[:MAX_FOLDERS]
def set_folders(feed, names):
"""The feed's folders, and `folder` (the first) for older clients."""
feed.folders = names
feed.folder = names[0] if names else ''
def _body(request):
try:
body = json.loads(request.body or b'{}')
except (json.JSONDecodeError, ValueError):
return None
return body if isinstance(body, dict) else None
def _data(user):
counts = {}
for names in PodcastFeed.objects.filter(user=user).values_list('folders', flat=True):
for n in names or []:
counts[n] = counts.get(n, 0) + 1
order = [n for n in PodcastSettings.for_user(user).folder_order if n in counts]
rest = sorted((n for n in counts if n not in order), key=str.casefold)
return {'order': order + rest, 'folders': [{'name': n, 'count': counts[n]} for n in order + rest]}
@csrf_exempt
@require_http_methods(['GET', 'POST'])
def folders(request):
"""GET {order, folders: [{name, count}]}; POST {order: [names]} sets the chip order."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
if request.method == 'POST':
body = _body(request)
try:
order = clean_list((body or {}).get('order'))
except ValueError:
return JsonResponse({'error': 'order: a list of names'}, status=400)
s = PodcastSettings.for_user(request.user)
s.folder_order = order
s.save(update_fields=['folder_order'])
return JsonResponse(_data(request.user))
def _rewrite(user, fn):
for feed in PodcastFeed.objects.filter(user=user).only('id', 'folders', 'folder'):
names = fn(list(feed.folders or []))
if names != (feed.folders or []):
set_folders(feed, clean_list(names))
feed.save(update_fields=['folders', 'folder'])
@csrf_exempt
@require_http_methods(['POST'])
def rename_folder(request):
"""{old, new}: every subscription in `old` is in `new` instead (merged if it exists)."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
body = _body(request) or {}
old, new = clean_name(body.get('old')), clean_name(body.get('new'))
if not old or not new:
return JsonResponse({'error': 'old and new required'}, status=400)
_rewrite(request.user, lambda names: [new if n == old else n for n in names])
s = PodcastSettings.for_user(request.user)
s.folder_order = clean_list([new if n == old else n for n in s.folder_order])
s.save(update_fields=['folder_order'])
return JsonResponse(_data(request.user))
@csrf_exempt
@require_http_methods(['POST'])
def delete_folder(request):
"""{name}: the folder goes, its subscriptions stay."""
if not request.user.is_authenticated:
return JsonResponse({'error': 'authentication required'}, status=401)
name = clean_name((_body(request) or {}).get('name'))
if not name:
return JsonResponse({'error': 'name required'}, status=400)
_rewrite(request.user, lambda names: [n for n in names if n != name])
s = PodcastSettings.for_user(request.user)
s.folder_order = [n for n in s.folder_order if n != name]
s.save(update_fields=['folder_order'])
return JsonResponse(_data(request.user))

10
podcasts/management/commands/index_episodes.py
View file

@ -1,10 +0,0 @@
from django.core.management.base import BaseCommand
from podcasts.discover import index_new
class Command(BaseCommand):
help = 'Put the episodes not yet in the full-text index into it (podcasts/discover.py)'
def handle(self, *args, **options):
self.stdout.write(f'{index_new()} episodes indexed')

11
podcasts/management/commands/refresh_feeds.py
View file

@ -2,8 +2,7 @@ from django.core.management.base import BaseCommand
from django.utils import timezone
from podcasts.models import PodcastFeed
from podcasts.discover import index_new
from podcasts.views import _in_backoff_q, _refresh_feeds, refresh_due
from podcasts.views import _in_backoff_q, _refresh_feeds
class Command(BaseCommand):
@ -27,10 +26,6 @@ class Command(BaseCommand):
# A feed failing 3+ times in a row is tried every 2, 4, 8, 16,
# at most 24 hours instead of every hour.
qs = qs.exclude(_in_backoff_q(timezone.now()))
# Each feed as often as its interval says (docs/podcasts.md): by its rhythm,
# or hourly, daily, weekly. --feed-id and --full take every one.
if not options['full']:
qs = qs.filter(pk__in=refresh_due(list(qs.select_related('user'))))
if options['limit']:
qs = qs[: options['limit']]
@ -41,6 +36,4 @@ class Command(BaseCommand):
# Parallel downloads, one per distinct feed URL (see _refresh_feeds).
refreshed, failed, new = _refresh_feeds(qs, full=options['full'])
# New episodes into the full-text index (Stöbern, podcasts/discover.py).
indexed = index_new()
self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes, {indexed} indexed')
self.stdout.write(f'{refreshed} feeds refreshed, {failed} failed, +{new} episodes')

35
podcasts/migrations/0019_audiobooks.py
View file

@ -1,35 +0,0 @@
# Generated by Django 6.1 on 2026-10-16 10:38
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0018_episode_silences'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='kind',
field=models.CharField(choices=[('podcast', 'Podcast'), ('audiobook', 'Hörbuch')], default='podcast', max_length=10),
),
migrations.CreateModel(
name='EpisodeBookmark',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('position_ms', models.PositiveIntegerField()),
('note', models.CharField(blank=True, max_length=1000)),
('created_at', models.DateTimeField(auto_now_add=True)),
('episode', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bookmarks', to='podcasts.podcastepisode')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='episode_bookmarks', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['position_ms', 'id'],
},
),
]

18
podcasts/migrations/0020_audiobook_folder.py
View file

@ -1,18 +0,0 @@
# Generated by Django 6.1 on 2026-10-16 13:29
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0019_audiobooks'),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='folder',
field=models.CharField(blank=True, max_length=100),
),
]

32
podcasts/migrations/0021_folders_list.py
View file

@ -1,32 +0,0 @@
# Generated by Django 6.1 on 2026-10-16 18:21
from django.db import migrations, models
def folder_to_folders(apps, schema_editor):
"""An audiobook's one folder becomes its first (and only) folder in the list."""
PodcastFeed = apps.get_model('podcasts', 'PodcastFeed')
for feed in PodcastFeed.objects.exclude(folder=''):
feed.folders = [feed.folder]
feed.save(update_fields=['folders'])
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0020_audiobook_folder'),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='folders',
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name='podcastsettings',
name='folder_order',
field=models.JSONField(blank=True, default=list),
),
migrations.RunPython(folder_to_folders, migrations.RunPython.noop),
]

49
podcasts/migrations/0022_lists_and_levels.py
View file

@ -1,49 +0,0 @@
# Generated by Django 6.1 on 2026-10-19 15:53
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0021_folders_list'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='refresh_interval',
field=models.CharField(blank=True, choices=[('', 'Automatisch'), ('hourly', 'Stündlich'), ('daily', 'Täglich'), ('weekly', 'Wöchentlich')], default='', max_length=10),
),
migrations.AddField(
model_name='podcastsettings',
name='lists_v2',
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name='podcastsettings',
name='stream_seen_at',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AlterField(
model_name='podcastfeed',
name='new_episodes',
field=models.CharField(choices=[('inbox', 'Im Strom'), ('queue', 'Stamm: in die Merkliste'), ('play', 'Stamm: in die Wiedergabeliste'), ('ignore', 'Auf Abruf')], default='inbox', max_length=10),
),
migrations.CreateModel(
name='PodcastLater',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('added_at', models.DateTimeField(auto_now_add=True)),
('auto', models.BooleanField(default=False)),
('episode', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='later_by', to='podcasts.podcastepisode')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='podcast_later', to=settings.AUTH_USER_MODEL)),
],
options={
'unique_together': {('user', 'episode')},
},
),
]

27
podcasts/migrations/0023_episode_fts.py
View file

@ -1,27 +0,0 @@
# The full-text index for Stöbern (podcasts/discover.py): SQLite FTS5, contentless.
from django.db import migrations
from podcasts.discover import CREATE_SQL, DROP_SQL
def create(apps, schema_editor):
if schema_editor.connection.vendor != 'sqlite':
return
for sql in CREATE_SQL:
schema_editor.execute(sql)
def drop(apps, schema_editor):
if schema_editor.connection.vendor != 'sqlite':
return
for sql in DROP_SQL:
schema_editor.execute(sql)
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0022_lists_and_levels'),
]
operations = [migrations.RunPython(create, drop)]

18
podcasts/migrations/0024_feed_favorite.py
View file

@ -1,18 +0,0 @@
# Generated by Django 6.1 on 2026-10-19 16:45
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0023_episode_fts'),
]
operations = [
migrations.AddField(
model_name='podcastfeed',
name='favorite',
field=models.BooleanField(default=False),
),
]

33
podcasts/migrations/0025_podcast_radio.py
View file

@ -1,33 +0,0 @@
# Generated by Django 6.1 on 2026-10-25 12:04
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0024_feed_favorite'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddField(
model_name='playerstate',
name='podcast_radio',
field=models.BooleanField(default=False),
),
migrations.CreateModel(
name='PodcastRadioHeard',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('at', models.DateTimeField(auto_now_add=True)),
('episode', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='radio_heard_by', to='podcasts.podcastepisode')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='podcast_radio_heard', to=settings.AUTH_USER_MODEL)),
],
options={
'unique_together': {('user', 'episode')},
},
),
]

18
podcasts/migrations/0026_podcastepisode_sponsor.py
View file

@ -1,18 +0,0 @@
# Generated by Django 6.1 on 2026-10-25 15:11
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0025_podcast_radio'),
]
operations = [
migrations.AddField(
model_name='podcastepisode',
name='sponsor',
field=models.JSONField(blank=True, null=True),
),
]

18
podcasts/migrations/0027_podcastepisode_link.py
View file

@ -1,18 +0,0 @@
# Generated by Django 6.1 on 2026-11-09 08:19
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcasts', '0026_podcastepisode_sponsor'),
]
operations = [
migrations.AddField(
model_name='podcastepisode',
name='link',
field=models.URLField(blank=True, max_length=1000),
),
]

59
podcasts/migrations/0028_transcription.py
View file

@ -1,59 +0,0 @@
# Generated by Django 6.1 on 2026-11-09 08:20
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feeds', '0007_transcription'),
('podcasts', '0027_podcastepisode_link'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='EpisodeArticle',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('model', models.CharField(blank=True, max_length=100)),
('kind', models.CharField(blank=True, max_length=20)),
('created_at', models.DateTimeField(auto_now_add=True)),
('episode', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='article', to='podcasts.podcastepisode')),
('item', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='episode_article', to='feeds.item')),
],
),
migrations.CreateModel(
name='TranscriptionJob',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('state', models.CharField(choices=[('none', 'none'), ('queued', 'queued'), ('downloading', 'downloading'), ('transcribing', 'transcribing'), ('done', 'done'), ('error', 'error')], default='none', max_length=15)),
('chunks_done', models.PositiveIntegerField(default=0)),
('chunks_total', models.PositiveIntegerField(default=0)),
('error', models.CharField(blank=True, max_length=500)),
('want_article', models.BooleanField(default=False)),
('claimed_by', models.CharField(blank=True, max_length=60)),
('claimed_until', models.DateTimeField(blank=True, null=True)),
('data', models.JSONField(blank=True, null=True)),
('created_at', models.DateTimeField(auto_now_add=True)),
('started_at', models.DateTimeField(blank=True, null=True)),
('finished_at', models.DateTimeField(blank=True, null=True)),
('episode', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='transcription_job', to='podcasts.podcastepisode')),
],
),
migrations.CreateModel(
name='TranscriptionSettings',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('engine', models.CharField(choices=[('builtin', 'Eingebaut'), ('custom', 'Eigene Instanz')], default='builtin', max_length=10)),
('url', models.CharField(blank=True, max_length=500)),
('token_encrypted', models.TextField(blank=True)),
('model', models.CharField(blank=True, max_length=100)),
('language', models.CharField(blank=True, max_length=10)),
('parallel', models.PositiveSmallIntegerField(default=3)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='transcription_settings', to=settings.AUTH_USER_MODEL)),
],
),
]

162
podcasts/models.py
View file

@ -8,30 +8,17 @@ MAX_SPEED = 3.5
class PodcastFeed(models.Model):
# What happens to a new episode of this feed (one that passed the filter); with the
# lists of docs/podcasts.md (PodcastSettings.lists_v2) these are the subscription levels:
# inbox = Strom, queue = Stamm (into the Merkliste), play = Stamm straight into the
# Wiedergabeliste, ignore = Auf Abruf. Without lists_v2, queue and play both queue.
# What happens to a new episode of this feed (one that passed the filter).
NEW_INBOX = 'inbox'
NEW_QUEUE = 'queue'
NEW_PLAY = 'play'
NEW_IGNORE = 'ignore'
NEW_EPISODE_ACTIONS = [
(NEW_INBOX, 'Im Strom'),
(NEW_QUEUE, 'Stamm: in die Merkliste'),
(NEW_PLAY, 'Stamm: in die Wiedergabeliste'),
(NEW_IGNORE, 'Auf Abruf'),
(NEW_INBOX, 'In „Neu“'),
(NEW_QUEUE, 'In die Warteschlange'),
(NEW_IGNORE, 'Nicht anzeigen'),
]
# How often the feed is fetched: '' = by its rhythm (refresh_due()), or fixed.
REFRESH_INTERVALS = [('', 'Automatisch'), ('hourly', 'Stündlich'), ('daily', 'Täglich'),
('weekly', 'Wöchentlich')]
# Volume factors: below 1 quieter, above 1 louder (the apps boost those).
VOLUMES = (0.2, 0.5, 1.0, 1.5, 2.0, 2.5)
# What it is: a podcast, or an audiobook (Hörbuch, Hörspiel), which lives in the
# library: its episodes are chapters in order, never news (docs/audiobooks.md).
PODCAST = 'podcast'
AUDIOBOOK = 'audiobook'
KINDS = [(PODCAST, 'Podcast'), (AUDIOBOOK, 'Hörbuch')]
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='podcast_feeds')
rss_url = models.URLField(max_length=1000)
@ -66,10 +53,6 @@ class PodcastFeed(models.Model):
volume = models.FloatField(default=1.0) # one of VOLUMES
# New episodes: what happens to them, and which ones count at all.
new_episodes = models.CharField(max_length=10, choices=NEW_EPISODE_ACTIONS, default=NEW_INBOX)
refresh_interval = models.CharField(max_length=10, choices=REFRESH_INTERVALS, default='', blank=True)
# A favourite subscription (★ in the Abos): only a mark and a filter, whatever its level —
# it puts nothing on the Merkliste by itself (docs/podcasts.md).
favorite = models.BooleanField(default=False)
# Comma-separated words, case-insensitive, matched against the title.
include_words = models.CharField(max_length=500, blank=True)
exclude_words = models.CharField(max_length=500, blank=True)
@ -86,11 +69,6 @@ class PodcastFeed(models.Model):
episode_filter = models.JSONField(default=list, blank=True)
# Skip silence in this podcast's episodes: null = as the device is set.
skip_silence = models.BooleanField(null=True, blank=True)
kind = models.CharField(max_length=10, choices=KINDS, default=PODCAST)
# Its folders (docs/library.md): like tags, any number; shown as filter chips. `folder`
# is the first of them, for older clients.
folders = models.JSONField(default=list, blank=True)
folder = models.CharField(max_length=100, blank=True)
# The vocabulary of episode_sort and episode_filter: the podcast page's
# sort menu and filter chips (native-podcast.js EP_SORTS and FILTERS).
@ -118,16 +96,15 @@ class PodcastFeed(models.Model):
self._synced = (self.__dict__.get('auto_queue'), self.__dict__.get('new_episodes'))
def _sync_auto_queue(self):
"""Keep auto_queue == (new_episodes is 'queue' or 'play'). Whichever of the two
"""Keep auto_queue == (new_episodes == 'queue'). Whichever of the two
was changed since loading (or, for a new feed, set away from its
default) wins; new_episodes if both were."""
auto_before, new_before = getattr(self, '_synced', (False, self.NEW_INBOX))
queues = (self.NEW_QUEUE, self.NEW_PLAY)
if self.new_episodes != new_before or self.auto_queue == auto_before:
self.auto_queue = self.new_episodes in queues
self.auto_queue = self.new_episodes == self.NEW_QUEUE
elif self.auto_queue:
self.new_episodes = self.NEW_QUEUE
elif self.new_episodes in queues:
elif self.new_episodes == self.NEW_QUEUE:
self.new_episodes = self.NEW_INBOX
def save(self, *args, **kwargs):
@ -164,8 +141,6 @@ class PodcastEpisode(models.Model):
title = models.CharField(max_length=500)
description = models.TextField(blank=True)
audio_url = models.URLField(max_length=1000)
# The episode's page, from the feed item's <link> (for sharing); '' when it has none.
link = models.URLField(max_length=1000, blank=True)
duration_seconds = models.IntegerField(default=0)
pub_date = models.DateTimeField(null=True, blank=True)
episode_number = models.IntegerField(null=True, blank=True)
@ -186,9 +161,6 @@ class PodcastEpisode(models.Model):
# The pauses for skipping silence in the browser (silence.py), found on
# first request: {"spans": [[start, end], …]} (+ "error"); null = not yet.
silences = models.JSONField(null=True, blank=True)
# The ad and the outro of a YouTube episode (sponsor.py, SponsorBlock), asked for
# on first request: {"spans": [[start, end], …], "at": when, (+ "error")}.
sponsor = models.JSONField(null=True, blank=True)
class Meta:
unique_together = ('feed', 'guid')
@ -219,68 +191,6 @@ class EpisodeTranscript(models.Model):
return f'Transcript of {self.episode}'
class TranscriptionSettings(models.Model):
"""Where a user's episodes are transcribed (docs/transcription.md): the
built-in Whisper instance (DIORA_WHISPER_URL) or their own."""
BUILTIN = 'builtin'
CUSTOM = 'custom'
ENGINES = [(BUILTIN, 'Eingebaut'), (CUSTOM, 'Eigene Instanz')]
user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='transcription_settings')
engine = models.CharField(max_length=10, choices=ENGINES, default=BUILTIN)
# The instance's base, e.g. https://whisper.example.org/v1 (custom only).
url = models.CharField(max_length=500, blank=True)
# The API token, Fernet-encrypted like feeds.SiteLogin.secret; never handed out.
token_encrypted = models.TextField(blank=True)
model = models.CharField(max_length=100, blank=True)
# '' = automatic, else an ISO code ("de").
language = models.CharField(max_length=10, blank=True)
parallel = models.PositiveSmallIntegerField(default=3) # 1 to 8 pieces at once
def __str__(self):
return f'Transcription settings of {self.user}'
class TranscriptionJob(models.Model):
"""One episode's transcription run, its Whisper result and the wish for an
article. Kept apart from EpisodeTranscript, which caches the feed's
transcripts and is rewritten when the feed changes them."""
NONE, QUEUED, DOWNLOADING, TRANSCRIBING, DONE, ERROR = (
'none', 'queued', 'downloading', 'transcribing', 'done', 'error')
STATES = [(s, s) for s in (NONE, QUEUED, DOWNLOADING, TRANSCRIBING, DONE, ERROR)]
episode = models.OneToOneField(PodcastEpisode, on_delete=models.CASCADE, related_name='transcription_job')
state = models.CharField(max_length=15, choices=STATES, default=NONE)
chunks_done = models.PositiveIntegerField(default=0)
chunks_total = models.PositiveIntegerField(default=0)
error = models.CharField(max_length=500, blank=True)
want_article = models.BooleanField(default=False)
# The device writing the article (a short name) and until when it holds the wish.
claimed_by = models.CharField(max_length=60, blank=True)
claimed_until = models.DateTimeField(null=True, blank=True)
# {"segments": [{start, end, speaker, text}], "source": {type: "whisper", model, language}}
data = models.JSONField(null=True, blank=True)
created_at = models.DateTimeField(auto_now_add=True)
started_at = models.DateTimeField(null=True, blank=True)
finished_at = models.DateTimeField(null=True, blank=True)
def __str__(self):
return f'Transcription of {self.episode} ({self.state})'
class EpisodeArticle(models.Model):
"""The article written from an episode's transcript; its text is the
feeds.Item (in the blog "Podcast-Artikel")."""
episode = models.OneToOneField(PodcastEpisode, on_delete=models.CASCADE, related_name='article')
item = models.OneToOneField('feeds.Item', on_delete=models.CASCADE, related_name='episode_article')
model = models.CharField(max_length=100, blank=True)
kind = models.CharField(max_length=20, blank=True) # Interview, Gespräch, Monolog, Diskussion
created_at = models.DateTimeField(auto_now_add=True)
def __str__(self):
return f'Article on {self.episode}'
# An episode counts as played this close to its end (AntennaPod's default),
# the same rule as the apps' "als gehört zählen".
PLAYED_BEFORE_END_SECONDS = 30
@ -312,21 +222,6 @@ class EpisodeProgress(models.Model):
return f'{self.user} - {self.episode}'
class EpisodeBookmark(models.Model):
"""A place in an episode kept with a note (an audiobook's bookmarks)."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='episode_bookmarks')
episode = models.ForeignKey(PodcastEpisode, on_delete=models.CASCADE, related_name='bookmarks')
position_ms = models.PositiveIntegerField()
note = models.CharField(max_length=1000, blank=True)
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
ordering = ['position_ms', 'id']
def __str__(self):
return f'{self.user} - {self.episode} @ {self.position_ms} ms'
class PodcastQueue(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='podcast_queue')
episode = models.ForeignKey(PodcastEpisode, on_delete=models.CASCADE, related_name='queued_by')
@ -340,38 +235,6 @@ class PodcastQueue(models.Model):
def __str__(self):
return f'{self.user} queue pos {self.position}: {self.episode}'
class PodcastLater(models.Model):
"""The Merkliste (docs/podcasts.md): episodes the user wants to hear some day. It may
grow and never expires; shown grouped by podcast, so it has no order of its own. An
episode can be in it and in the Wiedergabeliste (PodcastQueue) at once; heard, it leaves
both. auto: put there by a Stamm subscription, not by hand."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='podcast_later')
episode = models.ForeignKey(PodcastEpisode, on_delete=models.CASCADE, related_name='later_by')
added_at = models.DateTimeField(auto_now_add=True)
auto = models.BooleanField(default=False)
class Meta:
unique_together = ('user', 'episode')
def __str__(self):
return f'{self.user} Merkliste: {self.episode}'
class PodcastRadioHeard(models.Model):
"""An episode the Podcast-Radio is done with (docs/podcasts.md): it ran to its end there or
was skipped. Apart from the progress on purpose: it counts as heard nowhere else (no
history, no played mark, not in heard_365). Kept, so the radio never comes back to it."""
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='podcast_radio_heard')
episode = models.ForeignKey(PodcastEpisode, on_delete=models.CASCADE, related_name='radio_heard_by')
at = models.DateTimeField(auto_now_add=True)
class Meta:
unique_together = ('user', 'episode')
def __str__(self):
return f'{self.user} Radio: {self.episode}'
class PodcastSettings(models.Model):
"""A user's podcast preferences. Where episodes go in the queue: when added
by hand (also when played from outside the queue), and when a feed that
@ -407,14 +270,6 @@ class PodcastSettings(models.Model):
# imported: [{episode_id, size (bytes, 0 = unknown)}]. The app offers once
# to download them again, then clears the list.
import_downloads = models.JSONField(default=list, blank=True)
# The order of the podcast/audiobook folders (their chips), names.
folder_order = models.JSONField(default=list, blank=True)
# The two lists of docs/podcasts.md: PodcastQueue is the Wiedergabeliste (the player goes
# through it and stops at its end), PodcastLater the Merkliste, and Stamm subscriptions
# fill the Merkliste. Off: the one queue as before (apps not updated yet).
lists_v2 = models.BooleanField(default=False)
# When the user last looked at the Strom: episodes found after it carry a mark.
stream_seen_at = models.DateTimeField(null=True, blank=True)
# Playback, the same on every device (AntennaPod's playback settings).
# Null = not set yet: each device keeps its own value; the first app that
@ -483,9 +338,6 @@ class PlayerState(models.Model):
user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='player_state')
kind = models.CharField(max_length=10, choices=KINDS, default='', blank=True)
episode = models.ForeignKey(PodcastEpisode, null=True, blank=True, on_delete=models.SET_NULL, related_name='+')
# The episode plays in the Podcast-Radio (docs/podcasts.md): shown so elsewhere, and a
# transfer goes on with the radio there rather than playing it the usual way.
podcast_radio = models.BooleanField(default=False)
station_url = models.CharField(max_length=1000, blank=True)
station_name = models.CharField(max_length=300, blank=True)
station_id = models.IntegerField(null=True, blank=True)

122
podcasts/people.py
View file

@ -1,122 +0,0 @@
"""People in the user's podcasts (docs/podcasts.md, Stöbern): who is talked with, who gives a
talk — found in episode titles and descriptions, no service asked.
A name is two or three capitalised words ("Peter Decker", "Ulrike Eifler", "Jean-Luc Mélenchon"),
possibly with a particle ("Wolfgang von Goethe"). Two kinds of evidence:
- strong: right after a cue that announces a person — "mit", "w/", "with", "feat.", "Gast:",
"im Gespräch mit", "interview with", "von" in a talk's title, " - Name - " between dashes;
- weak: anywhere else in a title or description.
A name counts when it has strong evidence in at least STRONG_MIN episodes, or turns up in at least
FEEDS_MIN podcasts (a person known across the user's world). Words that are no names (places,
institutions, weekdays, sentence starts) are left out by a list and by the rule that a name's
first word must not be one of them.
"""
import html
import re
from collections import Counter, defaultdict
_W = r"[A-ZÄÖÜÀ-Ý][a-zäöüßà-ÿ]+(?:-[A-ZÄÖÜÀ-Ý][a-zäöüßà-ÿ]+)?"
_PART = r"(?:von|van|der|de|da|di|du|la|le|zu|ten|del|bin|al)"
_NAME = rf"{_W}(?:\s+{_PART})?(?:\s+{_W}){{1,2}}"
NAME_RE = re.compile(rf"(?<![\w\-])({_NAME})(?![\w])")
CUE_RE = re.compile(
r"(?i:\bmit\b|\bw/|\bwith\b|\bfeat\.?|\bft\.|\bGast:|\bGäste:|\bguest:|\binterview(?:t)? with\b|"
r"\bim Gespräch mit\b|\bin conversation with\b|\bvon\b|\bby\b)\s*(" + _NAME + r")|\s[-–|]\s(" + _NAME + r")")
# (only the cue ignores case: a name's words must start with a capital)
# First or last words that make a capitalised pair no person.
NOT_NAME_WORDS = set('''
Der Die Das Den Dem Des Ein Eine Einen Einem Einer Und Oder Aber Mit Von Nach Bei Für Auf Aus Im In Am Zum Zur
Vor Seit Als Wie Wenn Teil Folge Episode Staffel Kapitel Band Live Special Bonus Trailer Podcast Interview
Gespräch Vortrag Diskussion Lesung Hörspiel Feature Interviews Folgen Talk Show News Update Week Day Days
The A An And Or But With From For After Before On At By To Of As It This That These Those Our Your Their
Mr Mrs Ms Dr Prof Herr Frau Doktor Professor Sir Saint Sankt St
Monday Tuesday Wednesday Thursday Friday Saturday Sunday Montag Dienstag Mittwoch Donnerstag Freitag
Samstag Sonntag January February March April May June July August September October November December
Januar Februar März Juni Juli Oktober Dezember Frühling Sommer Herbst Winter
Deutschland Deutschlands Deutsche Deutschen Deutscher Europa Europas Europäische Europäischen Amerika
Amerikas Usa China Chinas Russland Russlands Ukraine Israel Gaza Iran Frankreich Großbritannien England
Berlin Hamburg München Wien Zürich Köln Leipzig Dresden London Paris New York Washington Moskau
United States Kingdom Union Republic Republik Bank Partei Linke Linken Grüne Grünen Spd Cdu Afd Fdp Bsw
Universität University College Institute Institut Stiftung Verlag Radio Deutschlandfunk Kultur Nova
Zeitung Magazin Review Times Post News Press Media Netflix Youtube Spotify Apple Google Amazon Twitter
Google Meta Tesla Marxismus Kapitalismus Sozialismus Kommunismus Faschismus Imperialismus Krieg Frieden
Gott Jesus Christus Weltkrieg Revolution Kalter Kalten Krise Wahl Wahlen Bundestag Bundesregierung
Remaster Remastered Audio Enhanced Edition Version Full Part Chapter Book Season Classic Klassiker
Hegel Marx Engels Kant Nietzsche Freud Lacan
Doku Hd Reupload Reaction Re View Neue Neuer Zu Gast Gäste My Last Free Speech Open Source Big Tech Cup World
War Files Kanzler Bundeskanzler Präsident Präsidentin President Minister Ministerin Trumps Künstliche
White House Middle East Silicon Valley Wall Street Mental Health Red Thread Critical Role Trio Infernal
Asian Japan Nhk Arte Spiegel Zeit Faz Taz Bild Planet Money Lage Nation Inside Austria Today Daily Weekly
Political Science Letzte Foreign Private Golden Keine Tag El Los Las San Santa Rio Fort Port Lake Mount
'''.split())
STRONG_MIN = 2
FEEDS_MIN = 3
def plain(text):
text = re.sub(r'<[^>]+>', ' ', text or '')
return re.sub(r'\s+', ' ', html.unescape(text)).strip()
def _clean(name):
words = name.split()
# "Mit Peter Decker" etc.: drop leading words that are no name.
while words and words[0] in NOT_NAME_WORDS:
words = words[1:]
while words and words[-1] in NOT_NAME_WORDS:
words = words[:-1]
caps = [w for w in words if w[0].isupper()]
if len(caps) < 2 or any(w in NOT_NAME_WORDS for w in caps):
return None
if any(len(w) > 1 and w.isupper() for w in caps): # "WAS TUN", "ARTE Reportage": no person
return None
return ' '.join(words)
def mentions(title):
"""(strong, weak): the names in one episode's title."""
strong, weak = set(), set()
for m in CUE_RE.finditer(title or ''):
n = _clean(m.group(1) or m.group(2))
if n:
strong.add(n)
for m in NAME_RE.finditer(title or ''):
n = _clean(m.group(1))
if n:
weak.add(n)
return strong, weak - strong
def people(episodes, weights=None):
"""episodes: [(episode_id, feed_id, title)]; weights: {feed_id: how much the user listens}
(heard_365). The people found, the most relevant first: [{'name', 'episodes', 'feeds'}].
Only titles: descriptions carry credits and boilerplate. First names are learnt from the
strong mentions ("mit Peter Decker": Peter); a name found elsewhere in a title counts only
with such a first name."""
found = [(ep_id, feed_id, *mentions(title)) for ep_id, feed_id, title in episodes]
surnames = defaultdict(set)
for _, _, strong, _ in found:
for n in strong:
w = n.split()
surnames[w[0]].add(w[-1])
# A first name: one that comes with at least two different surnames in strong mentions.
first_names = {f for f, last in surnames.items() if len(last) >= 2 and f not in NOT_NAME_WORDS}
eps = defaultdict(set)
feeds = defaultdict(set)
strong_n = Counter()
for ep_id, feed_id, strong, weak in found:
for n in strong:
strong_n[n] += 1
for n in {n for n in strong | weak if n.split()[0] in first_names}:
eps[n].add(ep_id)
feeds[n].add(feed_id)
weights = weights or {}
out = []
for n, ids in eps.items():
if strong_n[n] >= STRONG_MIN or len(feeds[n]) >= FEEDS_MIN:
score = len(ids) + 5 * sum(min(weights.get(f, 0), 50) for f in feeds[n]) ** 0.5
out.append({'name': n, 'episodes': len(ids), 'feeds': len(feeds[n]), 'score': round(score, 1)})
out.sort(key=lambda p: (-p['score'], p['name']))
return out

Some files were not shown because too many files have changed in this diff Show more