diff --git a/.forgejo/workflows/test.yml b/.forgejo/workflows/test.yml index a95c9a4..39d8ef1 100644 --- a/.forgejo/workflows/test.yml +++ b/.forgejo/workflows/test.yml @@ -29,7 +29,40 @@ jobs: /tmp/venv/bin/pip install --no-cache-dir -r requirements.txt - name: Django system check - run: /tmp/venv/bin/python manage.py check + # The SQLite file lives in data/ (a volume in production, git-ignored). + # Since the podcast models have JSON fields, the check opens the + # database to ask whether it supports them — so the folder must exist. + run: | + mkdir -p data + /tmp/venv/bin/python manage.py check - name: Run tests run: /tmp/venv/bin/python manage.py test + + e2e: + # The browser tests (e2e/, docs/refactoring.md): a real runserver on its own + # throwaway database, a book uploaded, read, highlighted, the theme switched. + runs-on: ubuntu-latest + container: + image: catthehacker/ubuntu:act-24.04 + + steps: + - uses: actions/checkout@v4 + + - name: Install Python dependencies + run: | + python -m venv /tmp/venv + /tmp/venv/bin/pip install --no-cache-dir -r requirements.txt + + - name: Install Playwright and Chromium + run: | + npm ci + npx playwright install --with-deps chromium + + - name: Browser tests + env: + CI: '1' + E2E_PYTHON: /tmp/venv/bin/python + run: | + mkdir -p data + npx playwright test diff --git a/accounts/middleware.py b/accounts/middleware.py index 3ceb447..addb990 100644 --- a/accounts/middleware.py +++ b/accounts/middleware.py @@ -22,4 +22,9 @@ class ApiTokenAuthMiddleware: api_token = ApiToken.objects.select_related('user').filter(token=token).first() if api_token is not None: request.user = api_token.user + # Browsers never attach this header on their own, so a + # token request can't be a cross-site forgery; native + # clients have no CSRF cookie to send. (CsrfViewMiddleware + # checks this flag in process_view, which runs after us.) + request._dont_enforce_csrf_checks = True return self.get_response(request) diff --git a/accounts/migrations/0006_webdav_inbox.py b/accounts/migrations/0006_webdav_inbox.py new file mode 100644 index 0000000..0588bad --- /dev/null +++ b/accounts/migrations/0006_webdav_inbox.py @@ -0,0 +1,33 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0005_webdavsource'), + ] + + operations = [ + migrations.AddField( + model_name='webdavsource', + name='inbox', + field=models.BooleanField(default=False), + ), + migrations.CreateModel( + name='WebDAVSeen', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('path', models.CharField(max_length=1000)), + ('size', models.BigIntegerField(default=0)), + ('modified', models.CharField(blank=True, max_length=100)), + ('claimed_at', models.DateTimeField(blank=True, null=True)), + ('done_at', models.DateTimeField(blank=True, null=True)), + ('result', models.CharField(blank=True, max_length=20)), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='seen', to='accounts.webdavsource')), + ], + options={ + 'unique_together': {('source', 'path')}, + }, + ), + ] diff --git a/accounts/migrations/0007_uservault.py b/accounts/migrations/0007_uservault.py new file mode 100644 index 0000000..5f7570e --- /dev/null +++ b/accounts/migrations/0007_uservault.py @@ -0,0 +1,28 @@ +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0006_webdav_inbox'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='UserVault', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=40)), + ('ct', models.TextField()), + ('iv', models.CharField(max_length=64)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='vault', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'name')}, + }, + ), + ] diff --git a/accounts/models.py b/accounts/models.py index 38e8229..c06b29d 100644 --- a/accounts/models.py +++ b/accounts/models.py @@ -56,6 +56,9 @@ class WebDAVSource(models.Model): username = models.CharField(max_length=200, blank=True) password = models.CharField(max_length=500, blank=True) root_path = models.CharField(max_length=500, blank=True, default='') + # "Eingang" (docs/library.md): new books in it come into the library by themselves — + # a device with the key fetches, encrypts and uploads them (WebDAVSeen keeps which). + inbox = models.BooleanField(default=False) created_at = models.DateTimeField(auto_now_add=True) last_used_at = models.DateTimeField(null=True, blank=True) @@ -89,6 +92,36 @@ class WebDAVSource(models.Model): return url +class WebDAVSeen(models.Model): + """A file of an "Eingang" source a device took (or is taking) into the library: taken + files aren't offered again; a claim keeps two devices from taking the same file at once + and runs out when a device didn't finish.""" + source = models.ForeignKey(WebDAVSource, on_delete=models.CASCADE, related_name='seen') + path = models.CharField(max_length=1000) + size = models.BigIntegerField(default=0) + modified = models.CharField(max_length=100, blank=True) + claimed_at = models.DateTimeField(null=True, blank=True) + done_at = models.DateTimeField(null=True, blank=True) + result = models.CharField(max_length=20, blank=True) # imported | known | too_big | failed + + class Meta: + unique_together = ('source', 'path') + + +class UserVault(models.Model): + """Small private settings of the devices, end-to-end encrypted with the book key + (docs/ai.md: the Anthropic API key of the "KI" features). The server keeps the blob, + one per name, and never sees what is in it.""" + user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='vault') + name = models.CharField(max_length=40) + ct = models.TextField() + iv = models.CharField(max_length=64) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = ('user', 'name') + + def _generate_token(): return secrets.token_hex(32) diff --git a/accounts/sync.py b/accounts/sync.py index dcd2422..69742ff 100644 --- a/accounts/sync.py +++ b/accounts/sync.py @@ -3,7 +3,8 @@ from django.http import JsonResponse from django.utils import timezone from django.views.decorators.http import require_http_methods -from books.models import EBook, EBookProgress, EBookHighlights, EBookBookmarks +from books.models import EBook, EBookProgress, EBookHighlights, EBookBookmarks, book_rows +from books.views import _now_ms, progress_state from podcasts.models import PodcastFeed, EpisodeProgress, PodcastQueue from radio.models import SavedStation @@ -19,21 +20,22 @@ def sync_snapshot(request): endpoints (e.g. GET /books//data/), which this same token also unlocks. Writes (progress, highlights, bookmarks, ...) go through those existing endpoints too, to reuse their merge semantics rather than - duplicating it here.""" + duplicating it here. + + ?only=books: just the books' part. The apps' book sync needs nothing else, + and the podcast part is by far the biggest (every episode ever played: + megabytes for an account that moved over from AntennaPod).""" if not request.user.is_authenticated: return JsonResponse({'error': 'authentication required'}, status=401) user = request.user - books = list(EBook.objects.filter(user=user).values('id', 'meta_ct', 'meta_iv', 'uploaded_at', 'is_read')) - for b in books: - b['uploaded_at'] = b['uploaded_at'].isoformat() + books = [{k: b[k] for k in ('id', 'meta_ct', 'meta_iv', 'uploaded_at', 'is_read')} for b in book_rows(user)] - book_progress = list( - EBookProgress.objects.filter(user=user).values('book_id', 'scroll_fraction', 'position_anchor', 'updated_at') - ) - for p in book_progress: - p['updated_at'] = p['updated_at'].isoformat() + book_progress = [ + {'book_id': p.book_id, 'updated_at': p.updated_at.isoformat(), **progress_state(p)} + for p in EBookProgress.objects.filter(user=user) + ] book_highlights = list(EBookHighlights.objects.filter(user=user).values('book_id', 'ct', 'iv', 'updated_at')) for h in book_highlights: @@ -43,6 +45,16 @@ def sync_snapshot(request): for bm in book_bookmarks: bm['updated_at'] = bm['updated_at'].isoformat() + if request.GET.get('only') == 'books': + return JsonResponse({ + 'server_time': timezone.now().isoformat(), + 'server_now': _now_ms(), + 'books': books, + 'book_progress': book_progress, + 'book_highlights': book_highlights, + 'book_bookmarks': book_bookmarks, + }) + podcast_feeds = list( PodcastFeed.objects.filter(user=user) .annotate(latest_episode_at=Max('episodes__pub_date')) @@ -71,6 +83,7 @@ def sync_snapshot(request): return JsonResponse({ 'server_time': timezone.now().isoformat(), + 'server_now': _now_ms(), 'books': books, 'book_progress': book_progress, 'book_highlights': book_highlights, diff --git a/accounts/tests.py b/accounts/tests.py index 4d4e05a..a77ac03 100644 --- a/accounts/tests.py +++ b/accounts/tests.py @@ -42,6 +42,20 @@ class ApiTokenAuthMiddlewareTests(TestCase): resp = self.client.get('/books/', HTTP_AUTHORIZATION=f'Bearer {self.token.token}') self.assertEqual(resp.status_code, 200) + def test_bearer_token_needs_no_csrf_token(self): + # import_m3u is not csrf_exempt: without a token it must still be protected. + from django.test import Client + client = Client(enforce_csrf_checks=True) + m3u = '#EXTM3U\n#EXTINF:-1,Test FM\nhttps://example.com/stream\n' + resp = client.post('/radio/import/', data=m3u, content_type='text/plain') + self.assertIn(resp.status_code, (302, 401, 403)) + resp = client.post('/radio/import/', data=m3u, content_type='text/plain', + HTTP_AUTHORIZATION=f'Bearer {self.token.token}') + self.assertNotEqual(resp.status_code, 403) + resp = client.post('/radio/import/', data=m3u, content_type='text/plain', + HTTP_AUTHORIZATION='Bearer not-a-real-token') + self.assertIn(resp.status_code, (302, 401, 403)) + def test_regenerate_view_requires_login_and_rotates_token(self): resp = self.client.post('/accounts/api-token/regenerate/') self.assertEqual(resp.status_code, 302) @@ -71,6 +85,12 @@ class SyncSnapshotTests(TestCase): resp = self.client.get('/api/sync/') self.assertEqual(resp.status_code, 401) + def test_only_books(self): + resp = self.client.get('/api/sync/?only=books', HTTP_AUTHORIZATION=f'Bearer {self.token.token}') + data = resp.json() + self.assertEqual(set(data), {'server_time', 'server_now', 'books', 'book_progress', 'book_highlights', 'book_bookmarks'}) + self.assertEqual(data['book_highlights'][0]['ct'], 'hl-ct') + def test_snapshot_shape_and_data(self): resp = self.client.get('/api/sync/', HTTP_AUTHORIZATION=f'Bearer {self.token.token}') self.assertEqual(resp.status_code, 200) @@ -526,3 +546,136 @@ class CloudFetchCachingTests(TestCase): with patch('books.webdav.fetch_file', return_value=b'EPUB-BYTES'): resp = self.client.get(f'/books/cloud/{self.source.pk}/fetch/?path=Dune.epub') self.assertEqual(resp['Cache-Control'], 'no-store') + + +class WebDAVEditTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username='alice', password='pw12345678') + self.other = User.objects.create_user(username='bob', password='pw12345678') + self.source = WebDAVSource.objects.create( + user=self.user, label='cloud', base_url='https://cloud.example.com', + username='alice', password='app-pw', + ) + + def _edit(self, pk, **data): + form = {'label': 'cloud', 'base_url': 'https://cloud.example.com', 'username': 'alice', 'root_path': ''} + form.update(data) + with patch('accounts.views.list_directory', return_value=[]): + return self.client.post(f'/accounts/webdav/{pk}/edit/', form) + + def test_sets_start_folder_and_keeps_password(self): + self.client.force_login(self.user) + resp = self._edit(self.source.pk, root_path='Books', password='') + self.assertEqual(resp.status_code, 302) + self.source.refresh_from_db() + self.assertEqual(self.source.root_path, 'Books') + self.assertEqual(self.source.password, 'app-pw') + + def test_new_password_replaces_the_old_one(self): + self.client.force_login(self.user) + self._edit(self.source.pk, password='new-pw') + self.source.refresh_from_db() + self.assertEqual(self.source.password, 'new-pw') + + def test_name_and_url_are_required(self): + self.client.force_login(self.user) + self._edit(self.source.pk, label='', root_path='Books') + self.source.refresh_from_db() + self.assertEqual(self.source.root_path, '') + + def test_foreign_source_is_not_editable(self): + self.client.force_login(self.other) + self._edit(self.source.pk, root_path='x') + self.source.refresh_from_db() + self.assertEqual(self.source.root_path, '') + + def test_settings_page_does_not_show_the_password(self): + self.client.force_login(self.user) + html = self.client.get('/accounts/settings/').content.decode() + self.assertIn(f'/accounts/webdav/{self.source.pk}/edit/', html) + self.assertNotIn('app-pw', html) + + +class CloudInboxTests(TestCase): + """An "Eingang" source (docs/library.md): its new books are handed out once, claimed for + one device for ten minutes, and not again once a device took them.""" + + def setUp(self): + self.user = User.objects.create_user('inbox', password='pw12345678') + self.client.force_login(self.user) + self.src = WebDAVSource.objects.create(user=self.user, label='Bücher', base_url='https://b.example.org/dav/', + username='d', password='x', inbox=True) + WebDAVSource.objects.create(user=self.user, label='andere', base_url='https://c.example.org/dav/') + + def listing(self, path=''): + if path == '': + return [{'name': 'Sub', 'path': 'Sub', 'is_dir': True, 'size': 0, 'modified': '', 'is_book': False}, + {'name': 'A - Eins (2020).epub', 'path': 'A - Eins (2020).epub', 'is_dir': False, + 'size': 100, 'modified': 'Mon', 'is_book': True}, + {'name': 'notiz.txt', 'path': 'notiz.txt', 'is_dir': False, 'size': 1, 'modified': '', 'is_book': False}] + return [{'name': 'B.pdf', 'path': 'Sub/B.pdf', 'is_dir': False, 'size': 200, 'modified': '', 'is_book': True}] + + def inbox(self): + with patch('books.webdav.list_directory', side_effect=lambda source, path='': self.listing(path)): + return self.client.get('/books/cloud/inbox/').json() + + def done(self, path, result): + return self.client.post(f'/books/cloud/{self.src.pk}/done/', json.dumps({'path': path, 'result': result}), + content_type='application/json') + + def test_new_books_once_then_claimed_then_done(self): + files = self.inbox()['files'] + self.assertEqual(sorted(f['path'] for f in files), ['A - Eins (2020).epub', 'Sub/B.pdf']) + self.assertEqual(self.inbox()['files'], []) # claimed for this device: no one else gets them now + self.assertEqual(self.done('A - Eins (2020).epub', 'imported').status_code, 200) + self.assertEqual(self.done('Sub/B.pdf', 'failed').status_code, 200) + from accounts.models import WebDAVSeen + WebDAVSeen.objects.filter(path='Sub/B.pdf').update(claimed_at=None) # the claim ran out + self.assertEqual([f['path'] for f in self.inbox()['files']], ['Sub/B.pdf']) + + def test_a_changed_file_counts_as_new(self): + self.inbox() + self.done('A - Eins (2020).epub', 'imported') + self.done('Sub/B.pdf', 'known') + self.assertEqual(self.inbox()['files'], []) + old = self.listing + self.listing = lambda path='': [dict(e, size=999) if e['path'].endswith('.epub') else e for e in old(path)] + self.assertEqual([f['path'] for f in self.inbox()['files']], ['A - Eins (2020).epub']) + + def test_only_own_sources_and_valid_results(self): + other = User.objects.create_user('x', password='pw12345678') + foreign = WebDAVSource.objects.create(user=other, label='f', base_url='https://d.example.org/dav/', inbox=True) + self.assertEqual(self.client.post(f'/books/cloud/{foreign.pk}/done/', '{}', content_type='application/json').status_code, 404) + self.inbox() + self.assertEqual(self.done('A - Eins (2020).epub', 'whatever').status_code, 400) + self.client.logout() + self.assertEqual(self.client.get('/books/cloud/inbox/').status_code, 401) + + def test_settings_switch_the_inbox(self): + r = self.client.post(f'/accounts/webdav/{self.src.pk}/edit/', {'label': 'Bücher', 'base_url': self.src.base_url}) + self.assertEqual(r.status_code, 302) + self.src.refresh_from_db() + self.assertFalse(self.src.inbox) + + + +class VaultTests(TestCase): + """The vault keeps what the devices encrypted, per name; only for its owner.""" + + def test_keep_replace_remove(self): + u = User.objects.create_user('v', password='pw12345678') + self.client.force_login(u) + post = lambda body: self.client.post('/accounts/vault/ai/', json.dumps(body), content_type='application/json') + self.assertEqual(self.client.get('/accounts/vault/ai/').json(), {'ct': '', 'iv': ''}) + self.assertEqual(post({'ct': 'abc', 'iv': 'i1'}).status_code, 200) + self.assertEqual(post({'ct': 'def', 'iv': 'i2'}).status_code, 200) + self.assertEqual(self.client.get('/accounts/vault/ai/').json(), {'ct': 'def', 'iv': 'i2'}) + self.assertEqual(post({'iv': 'x'}).status_code, 400) + other = User.objects.create_user('w', password='pw12345678') + self.client.force_login(other) + self.assertEqual(self.client.get('/accounts/vault/ai/').json()['ct'], '') + self.client.force_login(u) + self.assertEqual(post({'ct': ''}).status_code, 200) + self.assertEqual(self.client.get('/accounts/vault/ai/').json()['ct'], '') + self.client.logout() + self.assertEqual(self.client.get('/accounts/vault/ai/').status_code, 401) diff --git a/accounts/urls.py b/accounts/urls.py index 4bee71f..341bd8d 100644 --- a/accounts/urls.py +++ b/accounts/urls.py @@ -17,6 +17,8 @@ urlpatterns = [ path('change-password/', views.change_password, name='change_password'), path('api-token/regenerate/', views.regenerate_api_token, name='regenerate_api_token'), path('webdav/add/', views.webdav_add, name='webdav_add'), + path('webdav//edit/', views.webdav_edit, name='webdav_edit'), path('webdav//test/', views.webdav_test, name='webdav_test'), path('webdav//delete/', views.webdav_delete, name='webdav_delete'), + path('vault//', views.vault, name='vault'), ] diff --git a/accounts/views.py b/accounts/views.py index 519c073..7b09742 100644 --- a/accounts/views.py +++ b/accounts/views.py @@ -12,7 +12,7 @@ from django.views.decorators.http import require_http_methods from radio import lastfm as lastfm_module -from .models import ApiToken, WebDAVSource +from .models import ApiToken, UserVault, WebDAVSource from .webdav import WebDAVError, list_directory User = get_user_model() @@ -97,7 +97,7 @@ def lastfm_connect(request): return render(request, 'accounts/settings.html', { 'profile': request.user.profile, 'has_lastfm': request.user.profile.has_lastfm(), - 'lastfm_error': f"Could not connect to Last.fm: {exc}", + 'lastfm_error': f"Verbindung zu Last.fm fehlgeschlagen: {exc}", }) @@ -283,6 +283,7 @@ def webdav_add(request): username=request.POST.get('username', '').strip()[:200], password=request.POST.get('password', '')[:500], root_path=request.POST.get('root_path', '').strip()[:500], + inbox=request.POST.get('inbox') == '1', ) level, message = _probe_source(source) @@ -290,6 +291,36 @@ def webdav_add(request): return redirect('settings') +@login_required +@require_http_methods(['POST']) +def webdav_edit(request, pk): + source = WebDAVSource.objects.filter(pk=pk, user=request.user).first() + if not source: + messages.error(request, 'Verbindung nicht gefunden.') + return redirect('settings') + + label = request.POST.get('label', '').strip() + base_url = request.POST.get('base_url', '').strip() + if not label or not base_url: + messages.error(request, 'Name und Server-URL sind erforderlich.') + return redirect('settings') + + source.label = label[:100] + source.base_url = base_url[:500] + source.username = request.POST.get('username', '').strip()[:200] + source.root_path = request.POST.get('root_path', '').strip()[:500] + source.inbox = request.POST.get('inbox') == '1' + # The stored password is never sent to the page: empty means keep it. + password = request.POST.get('password', '') + if password: + source.password = password[:500] + source.save() + + level, message = _probe_source(source) + messages.add_message(request, level, message) + return redirect('settings') + + @login_required @require_http_methods(['POST']) def webdav_test(request, pk): @@ -310,3 +341,38 @@ def webdav_delete(request, pk): if deleted: messages.success(request, 'Verbindung entfernt.') return redirect('settings') + + + +# --------------------------------------------------------------------------- +# Vault: small device settings, end-to-end encrypted (docs/ai.md) +# --------------------------------------------------------------------------- + +VAULT_MAX = 64 * 1024 + + +@csrf_exempt +@require_http_methods(['GET', 'POST']) +def vault(request, name): + """GET {ct, iv} ('' while there is none); POST {ct, iv} replaces it, {ct: ''} removes it. + The blob is encrypted on the devices with the book key; the server only keeps it.""" + if not request.user.is_authenticated: + return JsonResponse({'error': 'authentication required'}, status=401) + if request.method == 'GET': + v = UserVault.objects.filter(user=request.user, name=name).first() + return JsonResponse({'ct': v.ct if v else '', 'iv': v.iv if v else ''}) + if len(request.body) > VAULT_MAX: + return JsonResponse({'error': 'too large'}, status=413) + try: + body = json.loads(request.body or b'{}') + except (json.JSONDecodeError, ValueError): + return JsonResponse({'error': 'invalid JSON'}, status=400) + ct = body.get('ct') if isinstance(body, dict) else None + iv = body.get('iv') if isinstance(body, dict) else None + if ct == '': + UserVault.objects.filter(user=request.user, name=name).delete() + return JsonResponse({'ok': True}) + if not isinstance(ct, str) or not isinstance(iv, str) or not iv or len(iv) > 64: + return JsonResponse({'error': 'ct and iv required'}, status=400) + UserVault.objects.update_or_create(user=request.user, name=name, defaults={'ct': ct, 'iv': iv}) + return JsonResponse({'ok': True}) diff --git a/books/migrations/0005_progress_last_read_wins.py b/books/migrations/0005_progress_last_read_wins.py new file mode 100644 index 0000000..baf8ff9 --- /dev/null +++ b/books/migrations/0005_progress_last_read_wins.py @@ -0,0 +1,43 @@ +from django.db import migrations, models + + +def keep_what_we_know(apps, schema_editor): + """The continue point so far was the furthest place: it is that, and it was + last saved at updated_at.""" + EBookProgress = apps.get_model('books', 'EBookProgress') + for p in EBookProgress.objects.all().iterator(): + p.furthest_fraction = p.scroll_fraction + p.furthest_anchor = p.position_anchor + p.read_at = int(p.updated_at.timestamp() * 1000) + p.save(update_fields=['furthest_fraction', 'furthest_anchor', 'read_at']) + + +class Migration(migrations.Migration): + + dependencies = [ + ('books', '0004_ebook_is_read'), + ] + + operations = [ + migrations.AddField( + model_name='ebookprogress', + name='read_at', + field=models.BigIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='ebookprogress', + name='device', + field=models.CharField(blank=True, default='', max_length=40), + ), + migrations.AddField( + model_name='ebookprogress', + name='furthest_fraction', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='ebookprogress', + name='furthest_anchor', + field=models.CharField(blank=True, default='', max_length=30), + ), + migrations.RunPython(keep_what_we_know, migrations.RunPython.noop), + ] diff --git a/books/migrations/0006_covers_and_prefs.py b/books/migrations/0006_covers_and_prefs.py new file mode 100644 index 0000000..488374e --- /dev/null +++ b/books/migrations/0006_covers_and_prefs.py @@ -0,0 +1,36 @@ +# Generated by Django 6.1 on 2026-10-16 18:21 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('books', '0005_progress_last_read_wins'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='ebook', + name='cover_ct', + field=models.TextField(blank=True, default=''), + ), + migrations.AddField( + model_name='ebook', + name='cover_iv', + field=models.CharField(blank=True, default='', max_length=32), + ), + migrations.CreateModel( + name='LibraryPrefs', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('prefs_ct', models.TextField(blank=True, default='')), + ('prefs_iv', models.CharField(blank=True, default='', max_length=32)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='library_prefs', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/books/migrations/0007_list_indexes.py b/books/migrations/0007_list_indexes.py new file mode 100644 index 0000000..054c9b1 --- /dev/null +++ b/books/migrations/0007_list_indexes.py @@ -0,0 +1,23 @@ +# Generated by Django 6.1 on 2026-10-19 13:24 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('books', '0006_covers_and_prefs'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddIndex( + model_name='ebook', + index=models.Index(fields=['user', 'uploaded_at', 'is_read'], name='ebook_list_idx'), + ), + migrations.AddIndex( + model_name='ebook', + index=models.Index(fields=['user', 'cover_iv'], name='ebook_cover_idx'), + ), + ] diff --git a/books/migrations/0008_ebook_file.py b/books/migrations/0008_ebook_file.py new file mode 100644 index 0000000..e014bbc --- /dev/null +++ b/books/migrations/0008_ebook_file.py @@ -0,0 +1,38 @@ +# Generated by Django 6.1 on 2026-10-19 15:02 +# The books' files move out of their rows into a table of their own (docs/database.md): +# copied in one statement, then the two columns are dropped (ALTER TABLE DROP COLUMN). + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('books', '0007_list_indexes'), + ] + + operations = [ + migrations.CreateModel( + name='EBookFile', + fields=[ + ('book', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='file', serialize=False, to='books.ebook')), + ('data_ct', models.TextField()), + ('data_iv', models.CharField(max_length=32)), + ], + ), + migrations.RunSQL( + 'INSERT INTO books_ebookfile (book_id, data_ct, data_iv) SELECT id, data_ct, data_iv FROM books_ebook', + reverse_sql='UPDATE books_ebook SET ' + 'data_ct = (SELECT f.data_ct FROM books_ebookfile f WHERE f.book_id = books_ebook.id), ' + 'data_iv = (SELECT f.data_iv FROM books_ebookfile f WHERE f.book_id = books_ebook.id)', + ), + migrations.RemoveField( + model_name='ebook', + name='data_ct', + ), + migrations.RemoveField( + model_name='ebook', + name='data_iv', + ), + ] diff --git a/books/models.py b/books/models.py index 9bc06c7..e9e3b16 100644 --- a/books/models.py +++ b/books/models.py @@ -2,27 +2,91 @@ from django.db import models from django.contrib.auth.models import User +class EBookManager(models.Manager): + def create(self, data_ct=None, data_iv=None, **kwargs): + """A book, and with data_ct/data_iv its file (EBookFile) in the same go.""" + book = super().create(**kwargs) + if data_ct is not None: + EBookFile.objects.create(book=book, data_ct=data_ct, data_iv=data_iv or '') + return book + + class EBook(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ebooks') meta_ct = models.TextField() # base64 AES-GCM ciphertext of {title, author, filename} meta_iv = models.CharField(max_length=32) # hex IV for metadata - data_ct = models.TextField() # base64 AES-GCM ciphertext of raw EPUB bytes - data_iv = models.CharField(max_length=32) # hex IV for EPUB data + # The file itself is in EBookFile (book.file), not here (docs/database.md). uploaded_at = models.DateTimeField(auto_now_add=True) is_read = models.BooleanField(default=False) + # The cover, end-to-end encrypted like the rest (docs/library.md): a client that has + # the file extracts it, encrypts it with the user's key and sends it once. + cover_ct = models.TextField(blank=True, default='') + cover_iv = models.CharField(max_length=32, blank=True, default='') + + objects = EBookManager() class Meta: ordering = ['uploaded_at'] + # The list reads these from indexes (book_rows()); they made it fast while the file + # was still in this row (docs/database.md), and cost nothing now. + indexes = [ + models.Index(fields=['user', 'uploaded_at', 'is_read'], name='ebook_list_idx'), + models.Index(fields=['user', 'cover_iv'], name='ebook_cover_idx'), + ] def __str__(self): return f"EBook #{self.pk} (user={self.user_id})" +class EBookFile(models.Model): + """A book's file, encrypted like everything else: base64 AES-GCM of the EPUB/PDF bytes. + In a table of its own (docs/database.md): with the file in the book's row, every change of a + title, a folder or "gelesen" rewrote megabytes, and the list read past every file.""" + book = models.OneToOneField(EBook, on_delete=models.CASCADE, primary_key=True, related_name='file') + data_ct = models.TextField() + data_iv = models.CharField(max_length=32) + + def __str__(self): + return f"EBookFile of #{self.book_id}" + + +def book_rows(user): + """Every book of the user as {id, meta_ct, meta_iv, uploaded_at (ISO), is_read, has_cover}, + oldest first — read without passing a book's file: meta_ct and meta_iv stand before it in + the row, the rest comes from the covering indexes ebook_list_idx and ebook_cover_idx.""" + books = {b['id']: b for b in EBook.objects.filter(user=user).order_by().values('id', 'meta_ct', 'meta_iv')} + for book_id, at, read in EBook.objects.filter(user=user).order_by().values_list('id', 'uploaded_at', 'is_read'): + if book_id in books: + books[book_id].update(uploaded_at=at, is_read=read) + covered = set(EBook.objects.filter(user=user).exclude(cover_iv='').order_by().values_list('id', flat=True)) + out = sorted((b for b in books.values() if 'uploaded_at' in b), key=lambda b: (b['uploaded_at'], b['id'])) + for b in out: + b['uploaded_at'] = b['uploaded_at'].isoformat() + b['has_cover'] = b['id'] in covered + return out + + +class LibraryPrefs(models.Model): + """The library's own settings, encrypted with the user's key (the folder order: + folder names are private, docs/library.md). One blob, replaced as a whole.""" + user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='library_prefs') + prefs_ct = models.TextField(blank=True, default='') + prefs_iv = models.CharField(max_length=32, blank=True, default='') + updated_at = models.DateTimeField(auto_now=True) + + class EBookProgress(models.Model): + """Where to continue a book: the place it was last read, on any device + (docs/progress-sync.md), plus the furthest place ever reached.""" user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ebook_progress') book = models.ForeignKey(EBook, on_delete=models.CASCADE, related_name='progress') scroll_fraction = models.FloatField(default=0.0) position_anchor = models.CharField(max_length=30, blank=True, default='') + # When the reader was at that place (ms since epoch, server clock); null before 2026-10-12. + read_at = models.BigIntegerField(null=True, blank=True) + device = models.CharField(max_length=40, blank=True, default='') + furthest_fraction = models.FloatField(default=0.0) + furthest_anchor = models.CharField(max_length=30, blank=True, default='') updated_at = models.DateTimeField(auto_now=True) class Meta: diff --git a/books/tests.py b/books/tests.py new file mode 100644 index 0000000..c8e711c --- /dev/null +++ b/books/tests.py @@ -0,0 +1,292 @@ +"""The reading position: the place last read wins (docs/progress-sync.md).""" +import json +from unittest.mock import patch + +from django.contrib.auth.models import User +from django.test import TestCase + +from accounts.models import ApiToken +from .models import EBook, EBookProgress + +NOW = 1_760_270_000_000 # the server's clock in these tests (ms) +MIN = 60_000 + + +class ProgressTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username='alice', password='pw12345678') + self.auth = {'HTTP_AUTHORIZATION': f'Bearer {ApiToken.objects.create(user=self.user).token}'} + self.book = EBook.objects.create(user=self.user, meta_ct='m', meta_iv='i', data_ct='d', data_iv='i') + self.now = NOW + clock = patch('books.views._now_ms', side_effect=lambda: self.now) + clock.start() + self.addCleanup(clock.stop) + + def save(self, anchor, frac=None, read_at=None, sent_at=None, device='Handy', **extra): + body = {'scroll_fraction': frac if frac is not None else 0.1, 'position_anchor': anchor, **extra} + if read_at is not None: + body.update(read_at=read_at, sent_at=sent_at if sent_at is not None else self.now, device=device) + resp = self.client.post(f'/books/{self.book.id}/progress/', json.dumps(body), + content_type='application/json', **self.auth) + self.assertEqual(resp.status_code, 200) + return resp.json() + + def state(self): + return EBookProgress.objects.get(book=self.book) + + # -- the new rule + + def test_newer_place_wins_even_if_it_is_earlier_in_the_book(self): + self.save('300:0.5', read_at=NOW - 10 * MIN, device='Handy') + r = self.save('12:0.0', read_at=NOW - 1 * MIN, device='diora Desktop') + self.assertTrue(r['stored']) + p = self.state() + self.assertEqual((p.position_anchor, p.device, p.read_at), ('12:0.0', 'diora Desktop', NOW - MIN)) + self.assertEqual(p.furthest_anchor, '300:0.5') + + def test_an_older_write_arriving_late_does_not_win(self): + # The phone read offline at 10:00 and syncs at 12:00; the desktop read at 11:00. + self.save('40:0.0', read_at=NOW - 60 * MIN, device='diora Desktop') + r = self.save('90:0.0', read_at=NOW - 120 * MIN, device='Handy') + self.assertFalse(r['stored']) + self.assertEqual(r['position_anchor'], '40:0.0') + p = self.state() + self.assertEqual((p.position_anchor, p.device), ('40:0.0', 'diora Desktop')) + self.assertEqual(p.furthest_anchor, '90:0.0', 'the late write still counts as reached') + + def test_resaving_an_unchanged_place_keeps_its_time_and_cannot_win(self): + self.save('50:0.0', read_at=NOW - 5 * MIN, device='Handy') + self.save('70:0.0', read_at=NOW - 2 * MIN, device='diora Desktop') + # The phone, idle since, saves its place again on going to the background. + self.now += MIN + r = self.save('50:0.0', read_at=NOW - 5 * MIN, device='Handy') + self.assertFalse(r['stored']) + self.assertEqual(self.state().position_anchor, '70:0.0') + + def test_the_device_clock_error_is_taken_out(self): + # A tolino 3 hours behind: it read one minute ago by its clock, and sends now by its clock. + behind = 3 * 60 * MIN + self.save('10:0.0', read_at=NOW - 30 * MIN, device='Handy') + r = self.save('20:0.0', read_at=NOW - behind - MIN, sent_at=NOW - behind, device='tolino') + self.assertTrue(r['stored']) + self.assertEqual(self.state().read_at, NOW - MIN) + + def test_nothing_wins_from_the_future(self): + self.save('10:0.0', read_at=NOW + 24 * 60 * MIN, sent_at=NOW) + self.assertEqual(self.state().read_at, NOW) + self.now += MIN + self.assertTrue(self.save('11:0.0', read_at=self.now, sent_at=self.now)['stored']) + + def test_equal_times_take_the_new_write(self): + self.save('10:0.0', read_at=NOW - MIN) + self.assertTrue(self.save('11:0.0', read_at=NOW - MIN)['stored']) + + def test_force_sets_the_place_now(self): + self.save('300:0.0', read_at=NOW - MIN) + self.assertTrue(self.save('0:0.0', frac=0.0, force=True)['stored']) + p = self.state() + self.assertEqual((p.position_anchor, p.read_at), ('0:0.0', NOW)) + self.assertEqual(p.furthest_anchor, '300:0.0') + + def test_furthest_only_grows(self): + self.save('80:0.0', read_at=NOW - 3 * MIN) + self.save('20:0.0', read_at=NOW - 2 * MIN) + self.save('81:0.0', read_at=NOW - 9 * MIN) + p = self.state() + self.assertEqual((p.position_anchor, p.furthest_anchor), ('20:0.0', '81:0.0')) + + def test_pdfs_compare_by_fraction(self): + self.save('', frac=0.7, read_at=NOW - 2 * MIN) + self.save('', frac=0.3, read_at=NOW - MIN) + p = self.state() + self.assertEqual((p.scroll_fraction, p.furthest_fraction), (0.3, 0.7)) + + def test_response_is_the_state_after_the_write(self): + r = self.save('5:0.25', frac=0.05, read_at=NOW - MIN, device='Browser') + self.assertEqual(r['server_now'], NOW) + for key in ('scroll_fraction', 'position_anchor', 'read_at', 'device', + 'furthest_fraction', 'furthest_anchor', 'last_read'): + self.assertIn(key, r) + self.assertEqual(r['read_at'], NOW - MIN) + + def test_implausible_times_count_as_an_old_app(self): + self.save('10:0.0', read_at=NOW - MIN) + # read_at=True or seconds instead of ms are not times: only forward, like an old app. + self.assertFalse(self.save('5:0.0', read_at=True)['stored']) + self.assertFalse(self.save('5:0.0', read_at=1_760_270_000)['stored']) + + # -- apps from before the change (no read_at) + + def test_old_apps_only_move_forward(self): + self.save('50:0.0', read_at=NOW - MIN) + self.assertFalse(self.save('20:0.0')['stored']) + self.assertTrue(self.save('60:0.0')['stored']) + self.assertEqual(self.state().read_at, NOW) + + def test_old_apps_resaving_the_same_place_change_nothing(self): + self.save('50:0.0', read_at=NOW - 5 * MIN, device='Handy') + self.now += MIN + self.assertFalse(self.save('50:0.0')['stored']) + p = self.state() + self.assertEqual((p.read_at, p.device), (NOW - 5 * MIN, 'Handy')) + + def test_first_write_of_an_old_app_is_stored(self): + self.assertTrue(self.save('0:0.0', frac=0.0)['stored']) + + def test_old_force_still_works(self): + self.save('50:0.0', read_at=NOW - MIN) + self.assertTrue(self.save('1:0.0', force=True)['stored']) + + # -- reading + + def test_book_list_and_snapshot_carry_the_state(self): + self.save('7:0.5', frac=0.2, read_at=NOW - MIN, device='Browser') + resp = self.client.get('/books/', **self.auth) + b = resp.json()[0] + self.assertEqual(resp['X-Server-Now'], str(NOW)) + self.assertEqual((b['position_anchor'], b['read_at'], b['device'], b['furthest_anchor']), + ('7:0.5', NOW - MIN, 'Browser', '7:0.5')) + self.assertTrue(b['last_read'].startswith('2025-10-12T')) + snap = self.client.get('/api/sync/?only=books', **self.auth).json() + p = snap['book_progress'][0] + self.assertEqual((p['book_id'], p['read_at'], p['device']), (self.book.id, NOW - MIN, 'Browser')) + self.assertIn('server_now', snap) + + def test_rows_from_before_have_their_save_time_as_read_at(self): + p = EBookProgress.objects.create(user=self.user, book=self.book, scroll_fraction=0.4, position_anchor='9:0.0') + b = self.client.get('/books/', **self.auth).json()[0] + self.assertEqual(b['read_at'], int(p.updated_at.timestamp() * 1000)) + + def test_a_never_opened_book(self): + b = self.client.get('/books/', **self.auth).json()[0] + self.assertEqual((b['read_at'], b['last_read'], b['position_anchor']), (None, None, '')) + + +class CoverAndPrefsTests(TestCase): + """Covers and the library's settings travel encrypted (docs/library.md).""" + + def setUp(self): + self.user = User.objects.create_user(username='alice', password='pw12345678') + self.auth = {'HTTP_AUTHORIZATION': f'Bearer {ApiToken.objects.create(user=self.user).token}'} + self.book = EBook.objects.create(user=self.user, meta_ct='m', meta_iv='i', data_ct='d', data_iv='i') + + def post(self, url, body, **auth): + return self.client.post(url, json.dumps(body), content_type='application/json', **(auth or self.auth)) + + def test_a_cover_is_kept_and_listed(self): + url = f'/books/{self.book.id}/cover/' + self.assertEqual(self.client.get(url, **self.auth).status_code, 404) + self.assertFalse(self.client.get('/books/', **self.auth).json()[0]['has_cover']) + self.assertEqual(self.post(url, {'cover_ct': 'Q0lQSEVS', 'cover_iv': 'a1b2c3d4e5f60718293a4b5c'}).status_code, 200) + self.assertEqual(self.client.get(url, **self.auth).json(), + {'cover_ct': 'Q0lQSEVS', 'cover_iv': 'a1b2c3d4e5f60718293a4b5c'}) + self.assertTrue(self.client.get('/books/', **self.auth).json()[0]['has_cover']) + + def test_bad_or_huge_covers_are_refused(self): + url = f'/books/{self.book.id}/cover/' + self.assertEqual(self.post(url, {'cover_ct': 'x', 'cover_iv': 'not hex!'}).status_code, 400) + self.assertEqual(self.post(url, {'cover_ct': 'x' * 300_000, 'cover_iv': 'a1b2c3d4'}).status_code, 400) + + def test_covers_are_ones_own(self): + bob = User.objects.create_user(username='bob', password='pw12345678') + bob_auth = {'HTTP_AUTHORIZATION': f'Bearer {ApiToken.objects.create(user=bob).token}'} + url = f'/books/{self.book.id}/cover/' + self.assertEqual(self.post(url, {'cover_ct': 'x', 'cover_iv': 'a1b2c3d4'}, **bob_auth).status_code, 404) + self.assertEqual(self.client.get(url, **bob_auth).status_code, 404) + + def test_prefs_are_one_encrypted_blob(self): + self.assertEqual(self.client.get('/books/prefs/', **self.auth).json(), {'prefs_ct': '', 'prefs_iv': ''}) + self.post('/books/prefs/', {'prefs_ct': 'AAA', 'prefs_iv': 'a1b2c3d4e5f6'}) + self.post('/books/prefs/', {'prefs_ct': 'BBB', 'prefs_iv': 'a1b2c3d4e5f7'}) + self.assertEqual(self.client.get('/books/prefs/', **self.auth).json(), {'prefs_ct': 'BBB', 'prefs_iv': 'a1b2c3d4e5f7'}) + + +class BookRowsReadPastNoFileTests(TestCase): + """The list must not read a column that stands after the file (data_ct) in a book's row: + SQLite would walk through the whole file to get there (docs/database.md).""" + + def test_the_list_queries_use_covering_indexes(self): + from django.contrib.auth.models import User + from django.db import connection + from django.test.utils import CaptureQueriesContext + from books.models import EBook, book_rows + user = User.objects.create_user('rows', password='x') + EBook.objects.create(user=user, meta_ct='m', meta_iv='i', data_ct='d' * 100000, data_iv='j', cover_iv='c', cover_ct='cc') + EBook.objects.create(user=user, meta_ct='m2', meta_iv='i2', data_ct='d', data_iv='j') + with CaptureQueriesContext(connection) as q: + rows = book_rows(user) + self.assertEqual([r['meta_ct'] for r in rows], ['m', 'm2']) + self.assertEqual([r['has_cover'] for r in rows], [True, False]) + late = ('uploaded_at', 'is_read', 'cover_iv', 'cover_ct', 'data_ct', 'data_iv') + with connection.cursor() as cur: + for query in q.captured_queries: + sql = query['sql'] + if any(col in sql for col in late): + cur.execute('EXPLAIN QUERY PLAN ' + sql.replace('%s', '?'), [user.id] if '?' in sql or '%s' in sql else []) + plan = ' '.join(str(r) for r in cur.fetchall()) + self.assertIn('COVERING INDEX', plan, sql) + + +class EBookFileTests(TestCase): + """The file lives in EBookFile; upload, data, replace-data and rekey go there.""" + + def setUp(self): + from django.contrib.auth.models import User + self.user = User.objects.create_user('filer', password='pw-filer-123') + self.client.force_login(self.user) + + def test_upload_read_replace_rekey_delete(self): + import base64, json + from books.models import EBook, EBookFile + data = base64.b64encode(b'x' * 100).decode() + r = self.client.post('/books/upload/', json.dumps({'meta_ct': 'm', 'meta_iv': 'i', 'data_ct': data, 'data_iv': 'd'}), + content_type='application/json') + pk = r.json()['id'] + self.assertEqual(EBookFile.objects.get(book_id=pk).data_ct, data) + self.assertEqual(self.client.get(f'/books/{pk}/data/').json(), {'data_ct': data, 'data_iv': 'd'}) + self.client.post(f'/books/{pk}/replace-data/', json.dumps({'data_ct': 'bmV3', 'data_iv': 'd2'}), content_type='application/json') + self.assertEqual(self.client.get(f'/books/{pk}/data/').json()['data_ct'], 'bmV3') + self.client.post(f'/books/{pk}/rekey/', json.dumps({'meta_ct': 'm3', 'meta_iv': 'i3', 'data_ct': 'cmVr', 'data_iv': 'd3'}), + content_type='application/json') + self.assertEqual(EBook.objects.get(pk=pk).meta_ct, 'm3') + self.assertEqual(EBookFile.objects.get(book_id=pk).data_iv, 'd3') + self.client.post(f'/books/{pk}/delete/') + self.assertFalse(EBookFile.objects.filter(book_id=pk).exists()) + + def test_someone_elses_file_is_not_found(self): + from django.contrib.auth.models import User + from books.models import EBook + other = User.objects.create_user('other-filer', password='pw-other-123') + b = EBook.objects.create(user=other, meta_ct='m', meta_iv='i', data_ct='ZA==', data_iv='i') + self.assertEqual(self.client.get(f'/books/{b.id}/data/').status_code, 404) + + +class StaleMetaTests(TestCase): + """A meta write that starts from an older meta (base_iv) is refused with the current one.""" + + def setUp(self): + from django.contrib.auth.models import User + from books.models import EBook + self.user = User.objects.create_user('metauser', password='pw-meta-123') + self.client.force_login(self.user) + self.book = EBook.objects.create(user=self.user, meta_ct='m1', meta_iv='aa', data_ct='ZA==', data_iv='i') + + def post(self, body): + import json + return self.client.post(f'/books/{self.book.id}/meta/', json.dumps(body), content_type='application/json') + + def test_current_base_is_taken(self): + self.assertEqual(self.post({'meta_ct': 'm2', 'meta_iv': 'bb', 'base_iv': 'aa'}).status_code, 200) + self.book.refresh_from_db() + self.assertEqual((self.book.meta_ct, self.book.meta_iv), ('m2', 'bb')) + + def test_old_base_is_refused_with_the_current_meta(self): + self.post({'meta_ct': 'm2', 'meta_iv': 'bb', 'base_iv': 'aa'}) + r = self.post({'meta_ct': 'old', 'meta_iv': 'cc', 'base_iv': 'aa'}) + self.assertEqual(r.status_code, 409) + self.assertEqual((r.json()['meta_ct'], r.json()['meta_iv']), ('m2', 'bb')) + self.book.refresh_from_db() + self.assertEqual(self.book.meta_ct, 'm2') + + def test_without_base_as_before(self): + self.assertEqual(self.post({'meta_ct': 'm3', 'meta_iv': 'dd'}).status_code, 200) diff --git a/books/urls.py b/books/urls.py index a9d738b..75804a8 100644 --- a/books/urls.py +++ b/books/urls.py @@ -5,6 +5,8 @@ urlpatterns = [ path('', views.book_list, name='book_list'), path('upload/', views.upload_book, name='upload_book'), path('cloud/sources/', webdav.cloud_sources, name='cloud_sources'), + path('cloud/inbox/', webdav.cloud_inbox, name='cloud_inbox'), + path('cloud//done/', webdav.cloud_done, name='cloud_done'), path('cloud//browse/', webdav.cloud_browse, name='cloud_browse'), path('cloud//fetch/', webdav.cloud_fetch, name='cloud_fetch'), path('metadata-lookup/', views.lookup_book_metadata, name='lookup_book_metadata'), @@ -12,6 +14,8 @@ urlpatterns = [ path('/delete/', views.delete_book, name='delete_book'), path('/read/', views.set_book_read, name='set_book_read'), path('/meta/', views.update_book_meta, name='update_book_meta'), + path('/cover/', views.book_cover, name='book_cover'), + path('prefs/', views.library_prefs, name='library_prefs'), path('/replace-data/', views.replace_book_data, name='replace_book_data'), path('/rekey/', views.rekey_book, name='rekey_book'), path('/progress/', views.save_progress, name='save_book_progress'), diff --git a/books/views.py b/books/views.py index 121a604..299f02e 100644 --- a/books/views.py +++ b/books/views.py @@ -1,15 +1,18 @@ import base64 import json import re +import time import xml.etree.ElementTree as ET +from datetime import datetime, timezone as dt_timezone import requests from django.conf import settings +from django.db import transaction from django.http import JsonResponse from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods -from .models import EBook, EBookProgress, EBookHighlights, EBookBookmarks +from .models import EBook, EBookFile, EBookProgress, EBookHighlights, EBookBookmarks, LibraryPrefs, book_rows def _require_auth(request): @@ -247,11 +250,40 @@ def _progress_is_further(new_anchor, new_frac, old_anchor, old_frac): """True if the new reading position is at least as far into the book as the old one. Compares by anchor block index (precise, decoupled from layout); falls back to scroll_fraction only when an anchor is missing (PDFs).""" - nb, ni = _anchor_parts(new_anchor) - ob, oi = _anchor_parts(old_anchor) - if nb >= 0 and ob >= 0: - return ni >= oi if nb == ob else nb >= ob - return new_frac >= old_frac + return _cmp_progress(new_anchor, new_frac, old_anchor, old_frac) >= 0 + + +def _cmp_progress(a_anchor, a_frac, b_anchor, b_frac): + """> 0 if A is further into the book than B, < 0 if behind, 0 if level.""" + ab, ai = _anchor_parts(a_anchor) + bb, bi = _anchor_parts(b_anchor) + if ab >= 0 and bb >= 0: + if ab != bb: + return ab - bb + return (ai > bi) - (ai < bi) + return (a_frac > b_frac) - (a_frac < b_frac) + + +def _now_ms(): + return int(time.time() * 1000) + + +def progress_state(p): + """A book's reading state as every client sees it (docs/progress-sync.md); + p may be None (never opened).""" + if p is None: + return {'scroll_fraction': 0.0, 'position_anchor': '', 'read_at': None, 'device': '', + 'furthest_fraction': 0.0, 'furthest_anchor': '', 'last_read': None} + read_at = p.read_at if p.read_at is not None else int(p.updated_at.timestamp() * 1000) + return { + 'scroll_fraction': p.scroll_fraction, + 'position_anchor': p.position_anchor, + 'read_at': read_at, + 'device': p.device, + 'furthest_fraction': p.furthest_fraction, + 'furthest_anchor': p.furthest_anchor, + 'last_read': datetime.fromtimestamp(read_at / 1000, tz=dt_timezone.utc).isoformat(), + } @require_http_methods(['GET']) @@ -259,26 +291,18 @@ def book_list(request): err = _require_auth(request) if err: return err - books = list( - request.user.ebooks.values('id', 'meta_ct', 'meta_iv', 'uploaded_at', 'is_read') - ) - for b in books: - b['uploaded_at'] = b['uploaded_at'].isoformat() - # Include saved scroll_fraction for each book - progress_map = { - p.book_id: (p.scroll_fraction, p.updated_at, p.position_anchor) - for p in EBookProgress.objects.filter(user=request.user) - } + books = book_rows(request.user) + progress_map = {p.book_id: p for p in EBookProgress.objects.filter(user=request.user)} highlighted_ids = set( EBookHighlights.objects.filter(user=request.user).values_list('book_id', flat=True) ) for b in books: - prog = progress_map.get(b['id']) - b['scroll_fraction'] = prog[0] if prog else 0.0 - b['last_read'] = prog[1].isoformat() if prog else None - b['position_anchor'] = prog[2] if prog else '' + b.update(progress_state(progress_map.get(b['id']))) b['has_highlights'] = b['id'] in highlighted_ids - return JsonResponse(books, safe=False) + response = JsonResponse(books, safe=False) + # The list is a bare array (old clients expect that): the server clock goes in a header. + response['X-Server-Now'] = str(_now_ms()) + return response @csrf_exempt @@ -326,12 +350,77 @@ def update_book_meta(request, pk): if not meta_ct or not meta_iv: return JsonResponse({'error': 'meta_ct, meta_iv required'}, status=400) + # base_iv: the IV of the meta the client changed. Another one now (changed elsewhere + # since): refused with the current meta, for the client to apply its change to that. + # A queued write from a device with an old list (a title or folder set meanwhile) + # used to put the old meta back. Clients that don't send base_iv write as before. + base_iv = body.get('base_iv') + if base_iv and base_iv != book.meta_iv: + return JsonResponse({'error': 'stale', 'meta_ct': book.meta_ct, 'meta_iv': book.meta_iv}, status=409) + book.meta_ct = meta_ct book.meta_iv = meta_iv book.save(update_fields=['meta_ct', 'meta_iv']) return JsonResponse({'ok': True}) +MAX_COVER = 200_000 # characters of base64: a 300×450 JPEG is far below +MAX_PREFS = 100_000 + + +def _hex_iv(value): + return isinstance(value, str) and 8 <= len(value) <= 32 and all(c in '0123456789abcdefABCDEF' for c in value) + + +@csrf_exempt +@require_http_methods(['GET', 'POST']) +def book_cover(request, pk): + """A book's cover, encrypted on the device with the user's key (docs/library.md): + GET {cover_ct, cover_iv} (404 without one), POST the same to set it.""" + err = _require_auth(request) + if err: + return err + book = EBook.objects.filter(pk=pk, user=request.user).only('id', 'cover_ct', 'cover_iv').first() + if book is None: + return JsonResponse({'error': 'not found'}, status=404) + if request.method == 'GET': + if not book.cover_ct: + return JsonResponse({'error': 'no cover'}, status=404) + return JsonResponse({'cover_ct': book.cover_ct, 'cover_iv': book.cover_iv}) + try: + body = json.loads(request.body) + except (json.JSONDecodeError, ValueError): + return JsonResponse({'error': 'invalid JSON'}, status=400) + ct, iv = body.get('cover_ct'), body.get('cover_iv') + if not isinstance(ct, str) or not ct or len(ct) > MAX_COVER or not _hex_iv(iv): + return JsonResponse({'error': 'cover_ct, cover_iv required (cover at most 150 KB)'}, status=400) + book.cover_ct, book.cover_iv = ct, iv + book.save(update_fields=['cover_ct', 'cover_iv']) + return JsonResponse({'ok': True}) + + +@csrf_exempt +@require_http_methods(['GET', 'POST']) +def library_prefs(request): + """The library's settings, encrypted with the user's key ({folder_order: [...]}): + GET {prefs_ct, prefs_iv} ('' while there are none), POST the same, replacing them.""" + err = _require_auth(request) + if err: + return err + if request.method == 'GET': + p = LibraryPrefs.objects.filter(user=request.user).first() + return JsonResponse({'prefs_ct': p.prefs_ct if p else '', 'prefs_iv': p.prefs_iv if p else ''}) + try: + body = json.loads(request.body) + except (json.JSONDecodeError, ValueError): + return JsonResponse({'error': 'invalid JSON'}, status=400) + ct, iv = body.get('prefs_ct'), body.get('prefs_iv') + if not isinstance(ct, str) or not ct or len(ct) > MAX_PREFS or not _hex_iv(iv): + return JsonResponse({'error': 'prefs_ct, prefs_iv required'}, status=400) + LibraryPrefs.objects.update_or_create(user=request.user, defaults={'prefs_ct': ct, 'prefs_iv': iv}) + return JsonResponse({'ok': True}) + + @csrf_exempt @require_http_methods(['POST']) def upload_book(request): @@ -360,16 +449,17 @@ def upload_book(request): return JsonResponse({'error': 'invalid base64 in data_ct'}, status=400) if raw_size > max_bytes: - return JsonResponse({'error': 'file too large (max 50 MB)'}, status=400) + return JsonResponse({'error': f'file too large (max {settings.EBOOK_MAX_BYTES // (1024 * 1024)} MB)'}, status=400) - book = EBook.objects.create( - user=request.user, - meta_ct=meta_ct, - meta_iv=meta_iv, - data_ct=data_ct, - data_iv=data_iv, - ) - EBookProgress.objects.create(user=request.user, book=book) + with transaction.atomic(): + book = EBook.objects.create( + user=request.user, + meta_ct=meta_ct, + meta_iv=meta_iv, + data_ct=data_ct, + data_iv=data_iv, + ) + EBookProgress.objects.create(user=request.user, book=book) return JsonResponse({'ok': True, 'id': book.id}) @@ -379,12 +469,10 @@ def get_book_data(request, pk): if err: return err - try: - book = EBook.objects.get(pk=pk, user=request.user) - except EBook.DoesNotExist: + f = EBookFile.objects.filter(book_id=pk, book__user=request.user).first() + if f is None: return JsonResponse({'error': 'not found'}, status=404) - - return JsonResponse({'data_ct': book.data_ct, 'data_iv': book.data_iv}) + return JsonResponse({'data_ct': f.data_ct, 'data_iv': f.data_iv}) @csrf_exempt @@ -412,14 +500,12 @@ def replace_book_data(request, pk): if not all([data_ct, data_iv]): return JsonResponse({'error': 'data_ct and data_iv required'}, status=400) - update_fields = ['data_ct', 'data_iv'] - book.data_ct = data_ct - book.data_iv = data_iv - if meta_ct and meta_iv: - book.meta_ct = meta_ct - book.meta_iv = meta_iv - update_fields += ['meta_ct', 'meta_iv'] - book.save(update_fields=update_fields) + with transaction.atomic(): + EBookFile.objects.update_or_create(book=book, defaults={'data_ct': data_ct, 'data_iv': data_iv}) + if meta_ct and meta_iv: + book.meta_ct = meta_ct + book.meta_iv = meta_iv + book.save(update_fields=['meta_ct', 'meta_iv']) return JsonResponse({'ok': True}) @@ -448,11 +534,11 @@ def rekey_book(request, pk): if not all([meta_ct, meta_iv, data_ct, data_iv]): return JsonResponse({'error': 'meta_ct, meta_iv, data_ct, data_iv required'}, status=400) - book.meta_ct = meta_ct - book.meta_iv = meta_iv - book.data_ct = data_ct - book.data_iv = data_iv - book.save(update_fields=['meta_ct', 'meta_iv', 'data_ct', 'data_iv']) + with transaction.atomic(): + EBookFile.objects.update_or_create(book=book, defaults={'data_ct': data_ct, 'data_iv': data_iv}) + book.meta_ct = meta_ct + book.meta_iv = meta_iv + book.save(update_fields=['meta_ct', 'meta_iv']) return JsonResponse({'ok': True}) @@ -497,22 +583,50 @@ def save_progress(request, pk): if isinstance(raw_anchor, str) and re.fullmatch(r'\d{1,7}:\d(\.\d{1,6})?', raw_anchor): position_anchor = raw_anchor - force = bool(body.get('force', False)) + force = body.get('force') is True + device = body.get('device') if isinstance(body.get('device'), str) else '' + now = _now_ms() + read_at = _client_ms(body.get('read_at')) + sent_at = _client_ms(body.get('sent_at')) - progress, _ = EBookProgress.objects.get_or_create( - user=request.user, - book=book, - ) - # Always advance to the furthest-read position (by anchor block index, so a - # transiently wrong scroll_fraction can't freeze the position) unless the - # client explicitly forces a reset (e.g. "start over" button in the reader). - if force or _progress_is_further(position_anchor, scroll_fraction, - progress.position_anchor, progress.scroll_fraction): - progress.scroll_fraction = scroll_fraction - progress.position_anchor = position_anchor - progress.save(update_fields=['scroll_fraction', 'position_anchor', 'updated_at']) + with transaction.atomic(): + progress, created = EBookProgress.objects.select_for_update().get_or_create(user=request.user, book=book) + if created: + progress.read_at = None + if force: + when, store = now, True + elif read_at is not None: + # The device's clock may be off: move its time onto ours by the + # difference at sending, and nothing counts as later than now. + when = read_at + (now - sent_at) if sent_at is not None else read_at + when = min(when, now) + store = progress.read_at is None or when >= progress.read_at + else: + # An app from before 2026-10-12: only forward, as it used to be — but + # strictly, so re-saving the same place changes nothing. + when = now + store = created or _cmp_progress(position_anchor, scroll_fraction, + progress.position_anchor, progress.scroll_fraction) > 0 + if store: + progress.scroll_fraction = scroll_fraction + progress.position_anchor = position_anchor + progress.read_at = when + progress.device = device[:40] + if _cmp_progress(position_anchor, scroll_fraction, + progress.furthest_anchor, progress.furthest_fraction) > 0 or created: + progress.furthest_fraction = scroll_fraction + progress.furthest_anchor = position_anchor + progress.save() - return JsonResponse({'ok': True}) + return JsonResponse({'ok': True, 'stored': store, 'server_now': now, **progress_state(progress)}) + + +def _client_ms(value): + """A time from a client (ms since epoch) or None if it isn't a plausible one.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + value = int(value) + return value if 10 ** 12 <= value < 10 ** 14 else None @csrf_exempt diff --git a/books/webdav.py b/books/webdav.py index 11ed64f..d43b72e 100644 --- a/books/webdav.py +++ b/books/webdav.py @@ -12,12 +12,16 @@ browser and only then POSTed to /books/upload/ as ciphertext, exactly like a local upload — nothing readable is ever stored server-side. """ +import json +from datetime import timedelta + from django.conf import settings from django.http import HttpResponse, JsonResponse from django.utils import timezone +from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods -from accounts.models import WebDAVSource +from accounts.models import WebDAVSeen, WebDAVSource from accounts.webdav import WebDAVError, WebDAVInputError, list_directory, fetch_file from .views import _require_auth @@ -34,11 +38,94 @@ def cloud_sources(request): sources = request.user.webdav_sources.all() return JsonResponse({'sources': [ - {'id': s.id, 'label': s.label, 'host': s.base_url} + {'id': s.id, 'label': s.label, 'host': s.base_url, 'inbox': s.inbox} for s in sources ]}) +# ---- "Eingang" (docs/library.md): new books of a source into the library by themselves ---- + +INBOX_DEPTH = 3 # sub-folders looked into +INBOX_MAX = 20 # files handed out per call +INBOX_CLAIM = timedelta(minutes=10) + + +def _inbox_files(source): + """The source's book files, its sub-folders too (INBOX_DEPTH deep).""" + out, todo = [], [('', 0)] + while todo: + path, depth = todo.pop(0) + for e in list_directory(source, path): + if e['is_dir'] and depth < INBOX_DEPTH: + todo.append((e['path'], depth + 1)) + elif e['is_book']: + out.append(e) + return out + + +@require_http_methods(['GET']) +def cloud_inbox(request): + """The files of the user's "Eingang" sources not taken into the library yet, claimed for + this device for ten minutes (another device asking meanwhile doesn't get them): + {files: [{source, label, path, name, size, modified}], errors: [{source, label, error}]}. + A file that changed (other size) counts as new again.""" + unauthorized = _require_auth(request) + if unauthorized: + return unauthorized + now = timezone.now() + files, errors = [], [] + for source in request.user.webdav_sources.filter(inbox=True): + try: + entries = _inbox_files(source) + except (WebDAVError, WebDAVInputError) as exc: + errors.append({'source': source.id, 'label': source.label, 'error': str(exc)}) + continue + seen = {s.path: s for s in source.seen.all()} + for e in entries: + if len(files) >= INBOX_MAX: + break + s = seen.get(e['path']) + if s and s.size == e['size']: + if s.done_at or (s.claimed_at and now - s.claimed_at < INBOX_CLAIM): + continue + WebDAVSeen.objects.update_or_create( + source=source, path=e['path'][:1000], + defaults={'size': e['size'], 'modified': e['modified'][:100], 'claimed_at': now, + 'done_at': None, 'result': ''}) + files.append({'source': source.id, 'label': source.label, 'path': e['path'], + 'name': e['name'].rsplit('/', 1)[-1], 'size': e['size'], 'modified': e['modified']}) + source.last_used_at = now + source.save(update_fields=['last_used_at']) + return JsonResponse({'files': files, 'errors': errors}) + + +@csrf_exempt +@require_http_methods(['POST']) +def cloud_done(request, pk): + """{path, result}: a device finished with an "Eingang" file — imported, known (already in the + library), too_big: not offered again; failed: offered again after the claim ran out.""" + unauthorized = _require_auth(request) + if unauthorized: + return unauthorized + source = _get_source(request, pk) + if not source: + return JsonResponse({'error': 'unknown source'}, status=404) + try: + body = json.loads(request.body or b'{}') + except (json.JSONDecodeError, ValueError): + return JsonResponse({'error': 'invalid JSON'}, status=400) + result = body.get('result') if isinstance(body, dict) else None + if result not in ('imported', 'known', 'too_big', 'failed'): + return JsonResponse({'error': 'result: imported, known, too_big or failed'}, status=400) + seen = WebDAVSeen.objects.filter(source=source, path=str(body.get('path', ''))[:1000]).first() + if not seen: + return JsonResponse({'error': 'unknown file'}, status=404) + seen.result = result + seen.done_at = None if result == 'failed' else timezone.now() + seen.save(update_fields=['result', 'done_at']) + return JsonResponse({'ok': True}) + + @require_http_methods(['GET']) def cloud_browse(request, pk): unauthorized = _require_auth(request) diff --git a/diora/__init__.py b/diora/__init__.py index e69de29..102c0ca 100644 --- a/diora/__init__.py +++ b/diora/__init__.py @@ -0,0 +1,2 @@ +# SQLite in WAL mode for every connection (diora/sqlite.py). +from . import sqlite # noqa: E402,F401 diff --git a/diora/search.py b/diora/search.py new file mode 100644 index 0000000..2bdb5c7 --- /dev/null +++ b/diora/search.py @@ -0,0 +1,101 @@ +"""The search across diora (docs/search.md): one query, what the server knows of every area — +podcasts and audiobooks one follows, podcast episodes (the full-text index, podcasts/discover.py), +news and blog sources, their articles, saved radio stations. The books aren't here: their titles +are encrypted, the page searches them itself (static/js/search.js). + +Every word of the query has to be there, as the start of a word ("plattf" finds "Plattform"); +case and diacritics don't count.""" +from django.db.models import Q +from django.http import JsonResponse +from django.views.decorators.http import require_http_methods + +from feeds.models import Item, Source +from podcasts import discover +from podcasts.models import PodcastFeed +from radio.models import SavedStation + +LIMIT = 8 # per area; the area's own view has the rest +ARTICLE_SCAN = 400 # articles looked at after the database's rough match + + +def _words(q): + return [w for w in discover._words(q) if w][:8] + + +def _matches(words, *texts): + """Every word as the start of a word in the texts ("plattf" finds "Plattform").""" + have = discover._words(' '.join(t or '' for t in texts)) + return bool(words) and all(any(h.startswith(w) for h in have) for w in words) + + +def _rank(words, title): + """Title starting with the query first, then title words, then the rest.""" + t = discover._words(title) + if t[:len(words)] == words or (t and t[0].startswith(words[0])): + return 0 + return 1 if all(any(h.startswith(w) for h in t) for w in words) else 2 + + +def _feeds(user, words): + out = {'podcasts': [], 'audiobooks': []} + rows = PodcastFeed.objects.filter(user=user).values('id', 'title', 'author', 'artwork_url', 'kind', 'rss_url', + 'favorite', 'new_episodes') + found = [f for f in rows if _matches(words, f['title'], f['author'])] + found.sort(key=lambda f: (_rank(words, f['title']), (f['title'] or '').lower())) + for f in found: + key = 'audiobooks' if f['kind'] == PodcastFeed.AUDIOBOOK else 'podcasts' + out[key].append(f) + return {k: v[:LIMIT] for k, v in out.items()} | {'podcasts_total': len(out['podcasts']), + 'audiobooks_total': len(out['audiobooks'])} + + +def _sources(user, words): + rows = Source.objects.filter(user=user).values('id', 'title', 'url', 'kind', 'icon_url', 'category') + found = [s for s in rows if _matches(words, s['title'], s['url'], s['category'])] + found.sort(key=lambda s: (_rank(words, s['title']), (s['title'] or '').lower())) + return found[:LIMIT] + + +def _articles(user, words): + """Articles whose title or author has every word, the newest first. The database narrows by + the longest word (LIKE, as case-blind as SQLite is), Python decides.""" + longest = max(words, key=len) + rough = Q(title__icontains=longest) | Q(author__icontains=longest) + if longest != longest.capitalize(): + rough |= Q(title__contains=longest.capitalize()) + rows = (Item.objects.filter(user=user, gone=False).filter(rough).order_by('-published') + .values('id', 'title', 'author', 'kind', 'published', 'image_url', 'read_at', 'source__title')[:ARTICLE_SCAN]) + found = [i for i in rows if _matches(words, i['title'], i['author'])] + return [{'id': i['id'], 'title': i['title'], 'author': i['author'], 'kind': i['kind'], + 'published': i['published'].isoformat() if i['published'] else None, 'image_url': i['image_url'], + 'read': i['read_at'] is not None, 'source': i['source__title']} for i in found[:LIMIT * 2]], len(found) + + +def _stations(user, words): + rows = SavedStation.objects.filter(user=user).values('id', 'name', 'url', 'tags', 'country', 'favicon_url', + 'is_favorite') + found = [s for s in rows if _matches(words, s['name'], s['tags'], s['country'])] + found.sort(key=lambda s: (_rank(words, s['name']), not s['is_favorite'], s['name'].lower())) + return found[:LIMIT] + + +def search(user, q): + words = _words(q) + if not words or len(''.join(words)) < 2: + return {'q': q, 'empty': True} + out = {'q': q} + out.update(_feeds(user, words)) + ids = discover.search_ids(user, q, limit=LIMIT * 2) + out['episodes'] = discover._rows(user, ids) + out['sources'] = _sources(user, words) + out['articles'], out['articles_total'] = _articles(user, words) + out['stations'] = _stations(user, words) + return out + + +@require_http_methods(['GET']) +def search_view(request): + """?q=…: what matches in every area (see the module's text).""" + if not request.user.is_authenticated: + return JsonResponse({'error': 'authentication required'}, status=401) + return JsonResponse(search(request.user, request.GET.get('q', '')[:200])) diff --git a/diora/settings.py b/diora/settings.py index ffbe165..e0c9112 100644 --- a/diora/settings.py +++ b/diora/settings.py @@ -31,12 +31,14 @@ INSTALLED_APPS = [ 'books', 'gpodder', 'tts', + 'feeds', + 'projects', ] -EBOOK_MAX_BYTES = 50 * 1024 * 1024 # 50 MB +EBOOK_MAX_BYTES = 100 * 1024 * 1024 # 100 MB (big scanned PDFs, complete editions) -# Encrypted uploads are base64-encoded (~33% overhead) so allow ~75 MB body -DATA_UPLOAD_MAX_MEMORY_SIZE = 75 * 1024 * 1024 +# Encrypted uploads are base64-encoded (~33% overhead) so allow ~140 MB body +DATA_UPLOAD_MAX_MEMORY_SIZE = 140 * 1024 * 1024 MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', @@ -76,7 +78,9 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': BASE_DIR / 'data' / os.environ.get('DIORA_DB_NAME', 'db.sqlite3'), - 'OPTIONS': {'timeout': 20}, + # WAL (diora/sqlite.py); a transaction takes the write lock when it begins, so one that + # reads first never fails on upgrading it (WAL's busy timeout can't help there). + 'OPTIONS': {'timeout': 20, 'transaction_mode': 'IMMEDIATE'}, } } @@ -89,7 +93,7 @@ AUTH_PASSWORD_VALIDATORS = [ {'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'}, ] -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = 'de' TIME_ZONE = 'UTC' USE_I18N = True USE_TZ = True diff --git a/diora/sqlite.py b/diora/sqlite.py new file mode 100644 index 0000000..873117d --- /dev/null +++ b/diora/sqlite.py @@ -0,0 +1,23 @@ +"""SQLite in write-ahead-log mode (docs/database.md): readers no longer wait for a writer. + +In the old rollback-journal mode a long write (the podcast refresh every 15 minutes, a book of +30 MB going in) kept every reader out until it was done, and requests failed with "database is +locked" — even GET /books/. WAL is set once per database file and stays; synchronous=NORMAL is +the safe pairing for it. A connection that can't switch (another one holds the file) just goes +on as it is; the next one switches. +""" +from django.db.backends.signals import connection_created + + +def _sqlite_wal(sender, connection, **kwargs): + if connection.vendor != 'sqlite': + return + with connection.cursor() as cursor: + try: + cursor.execute('PRAGMA journal_mode=WAL') + except Exception: # noqa: BLE001 - locked right now; another connection will do it + pass + cursor.execute('PRAGMA synchronous=NORMAL') + + +connection_created.connect(_sqlite_wal, dispatch_uid='diora_sqlite_wal') diff --git a/diora/test_search.py b/diora/test_search.py new file mode 100644 index 0000000..863a8a3 --- /dev/null +++ b/diora/test_search.py @@ -0,0 +1,62 @@ +"""The search across diora (diora/search.py).""" +from datetime import datetime, timezone + +from django.contrib.auth.models import User +from django.test import TestCase + +from feeds.models import Item, Source +from podcasts import discover +from podcasts.models import PodcastEpisode, PodcastFeed +from radio.models import SavedStation + + +class SearchTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username='sucher', password='pw12345678') + self.client.force_login(self.user) + other = User.objects.create_user(username='andere', password='pw12345678') + when = datetime(2026, 3, 1, tzinfo=timezone.utc) + self.pod = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/p', title='Ökonomie heute', + author='Peter Decker') + self.book = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/b', title='Das Kapital', + author='Karl Marx', kind=PodcastFeed.AUDIOBOOK) + PodcastFeed.objects.create(user=other, rss_url='https://example.org/p', title='Ökonomie heute') + self.ep = PodcastEpisode.objects.create(feed=self.pod, guid='e1', title='Die Ökonomie der Plattformen', + audio_url='https://example.org/e1.mp3', pub_date=when) + PodcastEpisode.objects.create(feed=self.book, guid='k1', title='Kapitel 1: Die Ware', + audio_url='https://example.org/k1.mp3', pub_date=when) + discover.index_new() + blog = Source.objects.create(user=self.user, url='https://blog.example/feed', title='Plattform-Blog', kind=Source.BLOG) + Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='a', title='Plattformarbeit und Streik', + published=when) + Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='b', title='Gartenarbeit', published=when) + Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='c', title='Plattformen, alt', + published=when, gone=True) + SavedStation.objects.create(user=self.user, name='Radio Plattform', url='https://radio.example/s', tags='jazz') + + def find(self, q): + return self.client.get('/search/', {'q': q}).json() + + def test_every_area(self): + r = self.find('plattf') + self.assertEqual([e['id'] for e in r['episodes']], [self.ep.id]) + self.assertEqual([a['title'] for a in r['articles']], ['Plattformarbeit und Streik']) + self.assertEqual([s['title'] for s in r['sources']], ['Plattform-Blog']) + self.assertEqual([s['name'] for s in r['stations']], ['Radio Plattform']) + + def test_podcasts_and_audiobooks_apart_case_and_accents_blind(self): + r = self.find('okonomie') + self.assertEqual([f['id'] for f in r['podcasts']], [self.pod.id]) + self.assertEqual(r['audiobooks'], []) + r = self.find('marx kap') + self.assertEqual([f['id'] for f in r['audiobooks']], [self.book.id]) + self.assertEqual(r['episodes'], [], 'an audiobook\'s chapters are no podcast episodes') + + def test_every_word_and_nothing_for_too_little(self): + self.assertEqual(self.find('plattform garten')['articles'], []) + self.assertTrue(self.find('a')['empty']) + self.assertEqual(self.client.get('/search/', {'q': 'x'}).status_code, 200) + + def test_only_signed_in(self): + self.client.logout() + self.assertEqual(self.client.get('/search/', {'q': 'plattform'}).status_code, 401) diff --git a/diora/urls.py b/diora/urls.py index 283cf2c..68dceb6 100644 --- a/diora/urls.py +++ b/diora/urls.py @@ -5,15 +5,19 @@ from django.urls import path, include from django.views.static import serve as serve_static from accounts.sync import sync_snapshot +from diora.search import search_view urlpatterns = [ path('admin/', admin.site.urls), path('accounts/', include('accounts.urls')), path('podcasts/', include('podcasts.urls')), path('books/', include('books.urls')), + path('feeds/', include('feeds.urls')), + path('projects/', include('projects.urls')), path('tts/', include('tts.urls')), path('api/2/', include('gpodder.urls')), path('api/sync/', sync_snapshot, name='api_sync'), + path('search/', search_view, name='search'), # Served at the root (not /static/js/sw.js) so its default scope covers # the whole app — a service worker's scope is limited to its own script's # directory unless the server sends Service-Worker-Allowed, so registering diff --git a/docs/ai.md b/docs/ai.md new file mode 100644 index 0000000..3ed9ece --- /dev/null +++ b/docs/ai.md @@ -0,0 +1,57 @@ +# KI: Claude schlägt vor, der Nutzer entscheidet + +Since 2026-10-24. Organisation help with Claude (Anthropic) across diora: books and magazines +first; podcasts, news/blogs and projects follow (same building blocks). + +## The key: the user's own, end-to-end encrypted, only on the devices + +Two providers, chosen in the settings (decided 2026-10-25): **Anthropic** (the official SDK) or +**OpenRouter** (the same Claude model, `anthropic/claude-opus-5.5`). OpenRouter's +Anthropic-compatible endpoint refuses the headers a browser call needs (`anthropic-version` …, +checked by a CORS preflight), so diora talks to OpenRouter's chat completions instead: plain +`fetch`, `response_format` json_schema (strict), `reasoning.effort`; no fallback on refusals, and +OpenRouter sees what is sent too. The vault keeps `{provider, anthropic_key, openrouter_key}` +(an old `{anthropic_key}` still reads as Anthropic); a key is checked before it is kept +(Anthropic: the model's info; OpenRouter: `GET /api/v1/key`). + +Decided with the user: the API key is entered in the app ("KI-Assistent", the ✦ of a +feature or the app menu) and stored like the books — encrypted with the book key on the device, +kept by the server as a blob it can't read: + +- `GET/POST /accounts/vault//` (`accounts.UserVault`, name `ai`): `{ct, iv}`; `{ct: ''}` + removes it. Its own store, not the library prefs (`/books/prefs/` is rewritten whole when + the folder order is saved). +- The devices decrypt it into memory only and call Claude **directly** from the page + (browser, the apps' WebView, the desktop). The diora server sees neither the key nor what is + sent; Anthropic sees what a feature needs (titles, authors, folder names …). Anthropic bills + the user. +- Without the book key on a device (or after a password change, which changes the book key) + the key can't be read: the settings ask for it again. Not on diora Reader (e-readers). +- Server-side jobs (in the background, without an app open) were left out on purpose: they + would need the key on the server. The project Radar has its own server key (docs/projects.md). + +## Building blocks (`static/js/ai.js`, an ES module) + +- The official SDK, `@anthropic-ai/sdk` 0.133.0, bundled for the browser with esbuild 0.28.2: + `static/vendor/anthropic-sdk-0.133.0.mjs` (licence beside it). `dangerouslyAllowBrowser`: + the key is the user's own. To update: `npm install @anthropic-ai/sdk@ esbuild` in an empty + folder, `esbuild entry.mjs --bundle --format=esm --platform=browser --target=es2020 --minify` + with `export { default } from '@anthropic-ai/sdk'`, and change the import in ai.js. +- `DioraAI.ask({system, prompt, schema, effort})`: model `claude-opus-5-5`, structured output + (`output_config.format` json_schema), effort `low` by default, streamed, `fallbacks: "default"` + (beta `server-side-fallback-2026-07-01`) for declined requests; refusals and over-long + answers become plain German messages. +- `DioraAI.review({title, intro, items, onApply})`: the suggestions with checkboxes — nothing + changes until the user ticks and presses "Übernehmen". +- `DioraAI.settings()`, `withKey(fn)`, `progress()`, `showError()`, `toast()`; the sheets close + with Android's back key; ⚙ in each sheet opens the settings. + +## Features + +| area | where | what Claude gets | what it suggests | +|---|---|---|---| +| Bücher, Zeitschriften | ✦ in the library's head (`static/js/app/55-library-tools.js`) | per book: id, title, author, folders, series, magazine mark; the existing folders | folders to add (existing first; new ones only for ≥ 3 books), series with volume, cleaned title/author where broken, magazine mark | +| Podcasts | ✦ in Abos (diora-android `web-native/static/native-podcast.js`, copied to `static/podcasts/`) | per subscription: title, author, folders, level, heard in 12 months, publishing interval, three newest episode titles | folders to add (≤ 2), level Stamm / Strom / Auf Abruf (only with a clear reason) | +| Nachrichten, Blogs | ✦ in the feeds' bar (`static/js/feeds.js`; the apps' section bar presses it) | per source: title, address, kind, Ressort or folders, recent titles shown | Ressort for news, folders for blogs, news ↔ blog when clearly wrong | + +Applied through `_updateBookMeta` (the encrypted meta, as when editing by hand). diff --git a/docs/audiobooks.md b/docs/audiobooks.md new file mode 100644 index 0000000..00d57af --- /dev/null +++ b/docs/audiobooks.md @@ -0,0 +1,112 @@ +# Audiobooks (Hörbücher & Hörspiele) + +A podcast subscription can be marked as an audiobook. It then leaves the +podcast world (no "Neu", no queue automation, not in the podcast lists) and +shows up in the library ("Bücher") next to the e-books, where it is listened to +like a book: its episodes are chapters in order, "Weiterhören" goes on where +the last chapter stopped, the whole book has one progress, places can be kept +as bookmarks with a note, and the sleep timer can stop at the end of a chapter. + +## Server (diora-web) + +### The mark + +`PodcastFeed.kind`: `"podcast"` (default) or `"audiobook"`. A feed setting like +the others: in `state.feeds[]` (`/podcasts/state/`), set with +`POST /podcasts/feeds//settings/ {"kind": "audiobook"}`. + +Marked as an audiobook: +- its episodes leave "Neu" (dismissed, not played); new ones never get there + and are never queued, whatever `new_episodes` says; +- it is left out of "Folgen" (`/podcasts/episodes/` without `feed_id`); +- nothing else changes: places, played marks, downloads, settings stay. + Episodes of it already in the queue stay there until played or removed. + +### Chapters and progress + +Chapter order: `episode_number` (if set), then `pub_date`, then the title in +natural order ("Kapitel 2" before "Kapitel 10"), then id. + +`GET /podcasts/audiobooks/` → `{"books": [book, …]}`, the last listened first: + +``` +book = { + "feed_id", "title", "author", "artwork_url", + "chapters": 24, "done": 6, // played chapters + "total_seconds": 32400, "heard_seconds": 11400, + "next": {"episode_id", "index", "position_seconds"} | null, // null: finished + "last_played_at": ISO | null +} +``` + +`heard_seconds` = played chapters' durations + the places in the others. +Durations the feed doesn't have count as 0 until a player reports them +(`progress/save` with `duration_seconds`, see youtube.md). + +`GET /podcasts/audiobooks//` → `{"book": book, "chapters": [ch, …], +"bookmarks": [bm, …]}` with + +``` +ch = { // the queue entry's shape, so it can be handed to the player as it is + "episode__id", "episode__title", "episode__audio_url", "episode__duration_seconds", + "episode__artwork_url", "episode__feed__id", "episode__feed__title", + "episode__feed__artwork_url", "episode__pub_date", "position_seconds", "played", + "index" +} +bm = {"id", "episode_id", "position_ms", "note", "created_at"} // in chapter order, then place +``` + +### Bookmarks + +- `POST /podcasts/bookmarks/ {"episode_id", "position_ms", "note"}` → `{"ok", "bookmark": bm}` +- `POST /podcasts/bookmarks// {"note"}` → `{"ok", "bookmark": bm}` +- `POST /podcasts/bookmarks//delete/` → `{"ok"}` + +Any episode can have bookmarks; the library shows those of audiobooks. + +## The player (both hosts: podcasts-web.js and Android) + +One new host call, next to `podcastPlayEpisode`: + +`podcastPlayBook(json)` with `json = {"feed_id", "chapters": [ch, …], "index", "position_ms"}` +(`position_ms` optional: else the chapter's own place) + +- plays `chapters[index]` **without putting anything in the queue**; +- the book is what plays next: when a chapter ends it counts as played and the + next chapter follows from its own place; after the last one playing stops; + "next" (⏭, media keys) goes to the next chapter without marking the one left + as played; +- playing anything else (a queue episode, the radio's own handover aside) ends + the book context; the book context survives a restart of the page/app; +- the sleep timer needs nothing new: "end of chapter" is `podcastSleep(-1)` + (end of the episode) or, for an episode with chapters of its own, the minutes + until that chapter's end (`podcastSleep(ms / 60000)`, fractions are fine). + +`playerState()` (unchanged) tells the library which chapter plays where. + +## The library (diora-web `static/js/audiobooks.js`, everywhere) + +One file for the web page, the desktop app and the Android app (like +feeds.js; full flavour only, it needs the podcast host). It draws into +`#tab-books`, above `#book-list`, in its own container `#audiobook-shelf`: + +- "Weiterhören": the audiobook listened to last (unless finished); +- "Hörbücher": a card per audiobook — picture, title, author, + "3 h 10 von 9 h · Kapitel 7/24" and a progress bar; +- "+ Hörbuch": pick one of the podcast subscriptions to mark as an audiobook; +- the book view: picture, progress, ▶ Weiterhören / Von vorn, the chapter list + (tap: play it from its place; played ones ✓), bookmarks (tap: play from there; + note editable; ✕), "+ Lesezeichen" (at the place where this book plays now), + "Schlummer: Kapitelende", and ⋮ → "Wieder als Podcast", "Alles als ungehört". + +It talks to the server with fetch and to the player through the host +(`window.DioraNative` on Android, `window.DioraPodcastHost` elsewhere): +`podcastState`, `podcastFeedSettings`, `podcastPlayBook`, `playerState`, +`podcastSleep`, `podcastMarkPlayedAll`. + +## The podcast UI (native-podcast.js, diora-android) + +- feed settings sheet: "Als Hörbuch" (sets `kind`); +- audiobook feeds left out of Abos, Neu, Folgen, search results of the + subscriptions; a hint "In Bücher → Hörbücher" where one was moved; +- the player shows a book chapter as "Kapitel 7/24 · " when it plays one. diff --git a/docs/database.md b/docs/database.md new file mode 100644 index 0000000..3484c04 --- /dev/null +++ b/docs/database.md @@ -0,0 +1,47 @@ +# The database + +One SQLite file, `data/db.sqlite3` (on the server `/home/marwin/diora-prod-data`, mounted into +`diora-prod` at `/app/data`). The books live in it too, encrypted, so it is large (2.5 GB in +October 2026) and some writes are big (a scanned book of 30 MB). + +## Write-ahead log (since 2026-10-19) + +`diora/sqlite.py` puts every connection in WAL mode (`PRAGMA journal_mode=WAL`, set once per +file, it stays) with `synchronous=NORMAL`; `settings.DATABASES` has `transaction_mode: +IMMEDIATE`, so a transaction takes the write lock when it begins. + +Why: in the rollback-journal mode before, a long write kept all readers out. The podcast +refresh (cron, every 15 minutes) and big uploads made requests fail with "database is +locked", GET /books/ among them — the apps then saw no new books and no changed titles. +With WAL, readers read on while one writer writes; writers still take turns (20 s timeout). + +Next to `db.sqlite3` there are now `db.sqlite3-wal` and `db.sqlite3-shm`. They belong to the +database: a copy of `db.sqlite3` alone may miss the latest writes. To copy it safely: + + docker exec diora-prod python -c "import sqlite3; s=sqlite3.connect('data/db.sqlite3'); d=sqlite3.connect('data/backup.sqlite3'); s.backup(d)" + +or stop the container first. Back to the old mode, if ever needed: +`PRAGMA journal_mode=DELETE` (with no other connection open) and remove `diora/sqlite.py`'s hook. + +## The book list never reads past the file + +A book's row is `user, meta_ct, meta_iv, data_ct (the file), data_iv, uploaded_at, is_read, +cover_ct, cover_iv`. SQLite stores a big value in a chain of overflow pages, and to reach a +column *after* it, it walks the whole chain: listing 427 books read gigabytes and took 37 s +(the desktop app gives up after 10 s, so it never synced). Since then `books.models.book_rows()` +reads `id, meta_ct, meta_iv` from the rows (they stand before the file) and `uploaded_at, +is_read` / whether there is a cover from the covering indexes `ebook_list_idx` and +`ebook_cover_idx` — no list query touches a file. `books/tests.py` checks the query plans. + +## The files in a table of their own (since 2026-10-19) + +`books.models.EBookFile` (one row per book, `book_id` its key) holds `data_ct`/`data_iv`; +`EBook` has only the small fields. Before, every change of a title, a folder or "gelesen" +rewrote the whole row with its file (megabytes, for a scanned volume tens of them), and every +`EBook.objects.get()` loaded the file. `EBook.objects.create(data_ct=…, data_iv=…)` still +works (the manager makes the EBookFile too); the API is unchanged. + +Migration `books/0008_ebook_file` copied 430 files in one statement and dropped the two +columns: about 2 minutes on the server (rehearsed on a copy), the site down meanwhile. The +pages the old columns used are free now and get reused; `VACUUM` (exclusive, minutes, at a +quiet time) would give them back to the disk. diff --git a/docs/library.md b/docs/library.md new file mode 100644 index 0000000..257e978 --- /dev/null +++ b/docs/library.md @@ -0,0 +1,226 @@ +# Folders and the library (redesign) + +## Folders everywhere: tags shown as filter chips + +One idea of a folder for every area — Bücher, Zeitschriften, Hörbücher, +Podcasts, Blogs (and the news' Ressorts, which already work this way): + +- A thing can be in **several folders** (they work like tags); none is fine. +- Folders are **filters**, not rooms: a row of chips on top — + `Alle · n · … · Ohne Ordner` — tapping one filters in place; no + "inside a folder" view, no back button. On a wide window the row wraps. +- **Assigning**: mark one or more things (long press / ☐ select mode, or ⋮ on + one), then "In Ordner …": a sheet with every folder of the area as checkboxes + and "Neuer Ordner". No typing a name that must match. +- **Managing** (⋮ on the chip row → "Ordner verwalten"): rename, merge (rename + onto an existing name), delete (the things stay, only the folder goes), + reorder by dragging. The order is the chip order. +- **Automatic groups** stay automatic and look different from folders (no + chip; a stack/group heading): audiobook series, magazine periodicals. + +### Where folders are kept + +| Area | Field | Notes | +|------|-------|-------| +| E-books, magazines | encrypted meta `folders: [names]` | `folder` (the first) kept for older clients; read `folders` or else `[folder]` | +| Audiobooks, podcasts | `PodcastFeed.folders` (JSON list) | feed setting `folders`; `folder` (string) still accepted | +| Blogs | `Source.folders` (exists) | | +| News | `Source.category` (one Ressort) | already chips in Nachrichten | + +The folder order per area: e-books in the encrypted user settings blob (names +are private), the others on the server (`/podcasts/folders/`, `/feeds/folders/`, +`{order: [names]}`). + +### Podcasts: a category's episodes, by time + +The folder chips stand over „Folgen“ too: with one chosen it lists every episode of +that category (`GET /podcasts/episodes/?folder=`, empty: the podcasts in none). +„Folgen“, „Neu“ and one podcast's page group their episodes by time — Heute, Gestern, +Diese Woche (the last 7 days), Diesen Monat (30 days), then the months — after the +Kino feed. In „Folgen“ (sort „Nach Zeitraum, seltene zuerst“, the default) and „Neu“ +the podcasts that seldom come go first within a group: `interval_days` of each feed +in `/podcasts/state/`, the mean gap between its newest 10 episodes (one episode: its +age, at least 30 days, the same when all came within an hour; kept an hour). Heard is a filter chip, not an order. + +### Shared code: `static/js/folders.js` (diora-web, every platform) + +``` +DioraFolders.chips(el, {folders: [{name, count}], active, none: count|0, onPick(name|''|null), onManage()}) +DioraFolders.assign({title, folders: [names], selected: [names], onDone(names)}) // the sheet +DioraFolders.manage({folders: [names], onRename(old, new), onDelete(name), onOrder(names)}) +``` +Each area keeps its own storage and calls these for the look and the dialogs. + +## The library (Bücher, Zeitschriften) + +- **Header**: title, search (title, author), sort (zuletzt gelesen · Titel · + Autor · hinzugefügt · kürzeste / längste zuerst), ⊞/☰ grid or list, `+` (Buch hinzufügen, Aus Cloud). +- **Length** (since 2026-10-25): the pages in the encrypted meta (`pages`) — a PDF's pages, an + EPUB's text by 1800 characters a page (`bookPagesOfFile`, 40-epub.js; the reader's own + estimate). Set on upload and when a book is opened; for older books the device works it out + once when a length sort is chosen ("Länge wird ermittelt … 3 von 80": file from the offline + copy or the server, decrypted on the device). Unknown lengths come last; the cards show + "· 320 S." while sorted by length. +- **Filter chips**: Alle · Angefangen · Ungelesen · Gelesen — then the folders. +- **Weiterlesen**: one card on top (cover, title, "Seite 132 von 186 · gestern", + ▶), not repeated in the list below. +- **Grid** (default on phone and desktop): covers with a progress line, title. + **List** (default on e-ink / diora Reader): one row per book, small cover, + title, author, % / ✓. + +### Covers, end-to-end encrypted + +A client that has a book's file extracts its cover (EPUB: the OPF cover image; +PDF: page 1), scales it to 240 px wide JPEG, **encrypts it with the user's key** +(AES-GCM like the meta) and uploads it once: `POST /books//cover/ +{cover_ct, cover_iv}`. `GET /books/` lists `has_cover`; `GET /books//cover/` +returns `{cover_ct, cover_iv}`. Clients cache the decrypted image locally. The +server never sees a cover. + +### Editions + +Two or more books can be **editions of one work** (the PDF scan and the EPUB, two +translations …): the same random key in each one's encrypted meta, `work: "w…"` +(`''` or missing = none). The list shows a work once — the edition read last, else an EPUB, +else the first added — with its formats on the cover ("EPUB · PDF", in the list view after +the author). Each edition keeps its own file, position and highlights. + +- Joining: choose several books → "Als Ausgaben verbinden", or a book's ⋮ → "Als Ausgabe + verbinden …" (other books listed, similar titles first, with a search). +- ⋮ → "Ausgaben (n) …" lists the editions: open one, "Lösen" sets one apart (the last one + left is set apart with it), "Weitere Ausgabe verbinden …". +- For the apps' own book sheets: `window.bookEditions(id)` opens that sheet, + `window.bookEditionsOf(id)` returns the editions (the book alone if it has none). + +### Generated covers + +A book without a cover (or one set to `cover: 'generated'` in its meta) gets one +drawn by `static/js/gencover.js`: an SVG, the same on every device, never stored. +A **concept** (motif, palettes, typeface) is chosen by score: + +- words in the title (3), the shelf category (2) and the author (2, only at a + name's start). A stem of five letters or more counts anywhere (German + compounds), a shorter one at a word's start, one of three letters as a whole + word; `^` / `$` in the list force a word's start / end. +- the folders, by name (`FOLDER_THEMES`): the first counts 2.5 — more than an + author or shelf word, less than a title word — further ones 1. + +A work of Marx or Engels (author, or a volume of the MEW series) is always a **blue volume** +(blau: blue cloth, gold rules and capitals, the spine on the left with the volume number); +no other book is. + +Concepts: konstruktiv (Marxist and political titles), geometrie (Philosophie: rings, turning squares, +triangle/circle/square), psyche (Psychoanalyse: two ellipses, the Borromean +knot, a spiral), karte (Geschichte), kosmos (Science-Fiction), ornament +(Fantasy), raster (Politik, Soziologie), schatten (Krimi: a red line, a lit +doorway), grusel (Horror), klassiker (patterned paper with a title plate), +sakral (Religion & Mythologie), spiel (Kinder & Jugend), code (Informatik), +typo, wellen, botanik, berge, zellen, and since 2026-10-25 funk (talk and radio: waveform, +mast, speech bubbles), musik (record, staff), kunst (colour fields, brushstrokes), essen (plate +on a checked cloth, citrus), sport (track, pitch), geld (price chart, coins), reise (road, +route with compass) — and bauhaus when nothing matches (shapes, quarter-circle tiles, a +circle crossed by diagonals). Most concepts have two or three motifs. A hash of title and +author picks motif, palette, layout and every detail. + +**Square** (`{square: true}`, since 2026-10-25): audiobooks and podcasts without a picture, or +whose picture doesn't load (audiobooks.js; diora-android native-podcast.js `art()`, before the +initial letter). 300 × 300, three title lines at most; the band of the motif shown is centred +in what the text leaves free. Podcasts (`podcast: true`) nothing matches get funk, not +bauhaus; a Marx/Engels audiobook gets konstruktiv (the blue volume is a binding). + +### Export as XML (since 2026-10-24) + +⇩ in the library's head (`static/js/app/55-library-tools.js`) saves `diora-bibliothek-.xml`: +made on the device from the list it has decrypted (nothing new goes to the server), every book +and magazine issue with its diora metadata — title, author, file name, ISBN, pages, shelf +category, series and volume, work (editions), magazine mark, cover choice, folders —, the +reading position (``), when it was uploaded, whether it has highlights, and an issue's articles with +their state. The folder order heads the file. Root ``. + +✦ beside it: ordering with Claude (docs/ai.md). + +### Eingang: books from a cloud folder by themselves (since 2026-10-24) + +A cloud source (Einstellungen → Cloud-Verbindungen, WebDAV: `accounts.WebDAVSource`, +`accounts/webdav.py`) can be marked **Eingang**: its new books come into the library without +anyone importing them. The server can't do it alone — the books are encrypted with the key +on the devices — so a device does: + +1. `static/js/app/57-cloud-inbox.js`: after the book list has loaded (at most every 5 + minutes; only with the real key in `localStorage`, never on diora Reader) it asks + `GET /books/cloud/inbox/` → `{files: [{source, label, path, name, size, modified}], errors}`: + the `.epub`/`.pdf` files of the Eingang sources (three folders deep) no device has taken. + The server claims them for this device for ten minutes (another device asking meanwhile + doesn't get them). +2. Each file: fetched through the server (`/books/cloud//fetch/`), imported like any file + (`_importEbookBuffer`, `skipKnown`: same title and author → left out), then + `POST /books/cloud//done/ {path, result}` with `imported`, `known`, `too_big` (not + offered again) or `failed` (again once the claim ran out). +3. The server keeps them in `accounts.WebDAVSeen` (source, path, size, modified): a file whose + size changed counts as new. The folder itself is never changed. + +The first such source: Shelfmark's e-book downloads on the Hetzner server, served read-only by +dufs (set up by the "Media" session; its notes: `C:\Users\m\homelab-handoff\04-hoerbuecher.md`). + +## Zeitschriften: das Heft und seine Artikel + +An issue is a unit (the editors' selection and order, its date, its theme) and +so is each article in it. Neither is reduced to the other +(`static/js/app/54-issues.js`): + +- **The issue stays the document.** It opens and reads through as before, in + the editors' order. The reader marks the article one is in: a running head + under the bar ("Schwerpunkt · Artikel 8 von 23 · Titel"), in the apps' footer + "Artikel 8/23", and a **pause at each article's end** (`aside.iss-end`): + "✓ gelesen", the issue's count, "Weiter im Heft" (the next article not read + yet, skipping those read or skipped), "Zum Inhalt", "überspringen", + "Doch nicht gelesen". In a PDF the pause is a bar that comes up when one + turns from an article's last page onto the next. +- **The states determine each other.** An article one was in and whose end one + reached counts as read (a jump — from the contents, "Weiter" — reads + nothing). The issue counts "7 von 23 gelesen · 2 übersprungen"; "Im Heft + weiterlesen" goes to the article one was in, else the next one open. + "Heft abschließen" marks the rest as skipped, never as read. +- **In the library** an issue shows one bar with a segment per article (read, + skipped, open, the one one is in). A tap opens its contents: Rubriken, the + articles with their state, a teaser and the reading time; the ⋮ / mark + sets read / unread / skipped by hand. + +**The index** comes from the file, on the device: + +- **EPUB**: the table of contents, by file — an article is a file (the first + entry pointing into it); the entries after it into the same file are its + sections, whatever their depth (GegenStandpunkt puts some at the top level); + an entry whose children point into other files is a Rubrik. A one-file EPUB: + the leaves. No table of contents: each spine file with text. The teaser is + the headline under the title (the headings before the first paragraph that + the title doesn't say, not numbered sections), else the first paragraph. +- **PDF**: a real outline (word processors' `_Hlk…`/`_Toc…` leftovers don't + count); most magazines' PDFs have none, so mostly **the contents page + printed in the issue**: among the first 16 pages the one with most numbered + rows (numbers that merely count 1, 2, 3 … are a list, not a contents), plus + its neighbours with entries. A row: the page number at the start + ("14 Der Krieg gegen Palästina", Jacobin), or at the end, maybe with the + author in a column before the title ("Nathan Sperber | Beyond Neoliberalism? + | 23", NLR); lines under it in its type size continue the title, an author + line ("Von …", a name) gives the author, lines in other type the teaser; a + title in capitals may begin on the lines above its number, a lone word on top + is its kicker (LuXemburg: GESPRÄCH, INFOGRAFIK); "Name: Titel" + (manuskripte). Capitals in the contents' type are Rubriken. The printed + numbers are mapped to the file's pages by the offset most titles agree on + (found in the page's text, a headline in capitals preferred); running heads + and the contents page's own head are dropped. + +Reading times count 200 words a minute. It is kept in the encrypted meta as +`issue: {v: 2, a: [{h, t, r, x, m, a?, p?, e?}]}` (href, title, Rubrik, teaser, +minutes, author, a PDF's first and last page), or `{v: 2, a: [], none: true}` +for an issue without a usable table of contents (read as a whole). An index of +another version is built anew. Built in the +background for issues without one (not on e-ink: there when the issue is +opened, or it comes from another device). + +**The states** are in the meta too: `issueRead: {href: 'r' | 's'}`, +`issueAt: href` (the article one was in). A change goes to the server after +2.5 s; offline it waits in `localStorage.diora_issue_pending` and goes with the +next save (or when the connection is back). diff --git a/docs/navigation.md b/docs/navigation.md new file mode 100644 index 0000000..0ef6b34 --- /dev/null +++ b/docs/navigation.md @@ -0,0 +1,111 @@ +# Navigation: Lesen and Hören + +diora's areas are sorted by what one does, not where things come from: + +| Group | Sections | +|-------|----------| +| — | Home | +| Lesen | Bücher · Zeitschriften · Nachrichten · Blogs | +| Hören | Hörbücher · Podcasts · Radio · Ambient (· Fokus, web page only) | +| Projekte | Projekte (docs/projects.md; `side`) | +| — (foot) | Suche (docs/search.md; `footer`, `transient`) · Abos & Quellen (docs/sources.md; `footer`) | + +- **Desktop / wide window (sidebar):** every section is its own entry, under + its group's heading. Nothing inside a section repeats the choice. +- **Phone (tab bar):** four tabs, **Home · Lesen · Hören · Projekte** (since 2026-10-19). + Lesen and Hören show a row of their sections at the top (segments); each group remembers + the section last used. The search is a magnifier in the bar on top (the apps' section bars, + the page's narrow bar), never a tab; Abos & Quellen is in the app menu. The mini player + stays below as it is. +- **Sidebar / wide bar:** `side` sections (Projekte) and then the `footer` ones (Suche, + Abos & Quellen) stand at its foot. A `transient` section (Suche) is never remembered as + the one to open next. +- **diora Reader** (tolino, e-readers): only Lesen, without Home. **TV:** Hören. + +Anything new goes into one of the groups. Managing (subscribing, settings, +folders) is in "Abos & Quellen" (docs/sources.md; done). + +## One definition: `static/js/nav.js` (diora-web) + +Loaded by the page on every platform, before the other area scripts. It owns +the sections, which panel shows each, switching between them, the URL hash +and the remembered choice. The bars are drawn from it. **It is the one source for the +navigation.** + +Underneath, `static/js/app/20-tabs.js` keeps `TOP_TABS`: the list of *panels* +(`#tab-`) that `showTab(name)` can show — home, radio, focus, podcasts, books, feeds, +ambient, audiobooks, sources, suche, projekte. That is not a second navigation: nav.js +maps each section to one of these panels (several sections share one: Bücher/Zeitschriften +→ books, Nachrichten/Blogs → feeds) and calls `showTab`. A new area needs both: its panel in +`TOP_TABS` (and the template) and its section in nav.js. + +``` +window.DioraNav = { + sections(), // [{id, group, label, icon, available}] in order (only those the page has) + groups(), // [{id: 'home'|'lesen'|'hoeren'|'projekte', label, icon, sections: [id…]}] + icon(name), // a section's or group's icon as SVG (also 'menu') + current(), // the section id shown now + go(id), // show a section ('lesen'/'hoeren': that group's last section) + onChange(fn), // fn(sectionId) after every change +} +``` + +| Section id | Group | Label | Panel | View inside the panel | +|------------|-------|-------|-------|-----------------------| +| `home` | — | Home | `#tab-home` | | +| `books` | lesen | Bücher | `#tab-books` | e-books (not magazines) | +| `magazines` | lesen | Zeitschriften | `#tab-books` | magazines | +| `news` | lesen | Nachrichten | `#tab-feeds` | news | +| `blogs` | lesen | Blogs | `#tab-feeds` | blogs | +| `audiobooks` | hoeren | Hörbücher | `#tab-audiobooks` | (audiobooks.js) | +| `podcasts` | hoeren | Podcasts | `#tab-podcasts` | | +| `radio` | hoeren | Radio | `#tab-radio` | | +| `ambient` | hoeren | Ambient | `#tab-ambient` | | +| `focus` | hoeren | Fokus | `#tab-focus` | web page only (`webOnly`) | +| `projekte` | projekte | Projekte | `#tab-projekte` | (projects.js) | +| `suche` | (foot) | Suche | `#tab-suche` | (search.js); never remembered | +| `sources` | (foot) | Abos & Quellen | `#tab-sources` | (sources.js) | + +`go()` calls the page's `showTab(panel)` as before (so everything hooked onto +showTab keeps working) and then the panel's view hook: +`window.__dioraBooksView('books'|'magazines')`, +`window.__dioraFeedsView('news'|'blogs')`. The panels' own switches for that +choice (feeds.js News/Blogs) are hidden. `_currentTopTab` stays the panel name. + +Hash: `#books`, `#magazines`, `#news`, `#blogs`, `#audiobooks`, … (old hashes +`#feeds`, `#radio/saved` keep working). Remembered: `localStorage` +`diora_nav_section` and `diora_nav_last_`. + +**Zeitschriften**: the books marked as magazines (encrypted meta `magazine`), or, unmarked, +those in a library folder called "Zeitschriften" (or "Magazine"); issues and articles: +docs/library.md. + +### The segment row (phone) + +nav.js draws `#nav-segments` above the panels: the current group's sections, +when the layout is the tab bar (not `html.nui-sidebar`) and the group has more +than one available section. Styled with the theme tokens; the native CSS may +restyle it. + +## Who draws what + +| Where | Bar | Drawn by | +|-------|-----|----------| +| Android (phone) | `#nui-tabbar`: Home · Lesen · Hören · Projekte | native-radio.js, from `DioraNav.groups()` | +| Android tablet, desktop (`html.nui-sidebar`) | rail: Home, headings Lesen/Hören with their sections, at the foot Projekte · Suche · Abos & Quellen, Menü | native-radio.js, from `DioraNav.sections()` | +| Apps, every section | a bar on top (`.nui-tab-bar`; `.nui-section-bar` for the generic ones) with the section's name, its actions and the search magnifier | native-ui.js `SECTION_BARS`, native-radio.js, native-podcast.js | +| Web page | `#tabs` (narrow: groups + segments; wide: all sections under headings) | nav.js | + +A tab/rail button calls `DioraNav.go(id)`; the active state comes from +`DioraNav.onChange`. + +## Phases + +1. Navigation (this document), Hörbücher as their own panel, Zeitschriften by folder. +2. "Abos & Quellen": podcasts, YouTube channels, news sources, blogs and the + audiobook mark in one place; the everyday views without managing. +3. Library (done): audiobook series are found in the titles ("Das Kapital – + Erster Band" …) and shown as one stack; audiobooks have folders + (`PodcastFeed.folders`, a list; `folder` is its first, for older clients); a book is marked as a magazine in its encrypted + meta (`magazine: true/false`; unset: the "Zeitschriften" folder decides), and + Zeitschriften groups the issues by periodical. diff --git a/docs/news-text.md b/docs/news-text.md new file mode 100644 index 0000000..273c74c --- /dev/null +++ b/docs/news-text.md @@ -0,0 +1,46 @@ +# Lesehilfen für Nachrichten (feeds/textproc.py) + +Klassische Textverfahren, kein Sprachmodell; reines Python auf dem Server, für die Nachrichten +eines Nutzers auf einmal. Ausprobiert an echten Nachrichten (Oktober 2026), übernommen wurde, +was dort trug. + +## Floskeln + +Sätze, die eine Quelle unter mindestens drei ihrer letzten 40 Artikel setzt (Spendenboxen, +„Diese Nachricht wurde … gesendet“, Abo-Hinweise), gehören zu keinem Artikel. `GET +/feeds/items//` liefert den Artikel ohne die Absätze, die nur aus solchen Sätzen bestehen; der +Überblick sieht sie gar nicht erst. Je Quelle sechs Stunden zwischengespeichert. + +## Überblick einer Meldung + +`GET /feeds/stories//overview/` – nur für Meldungen aus mindestens zwei Quellen: + +- `summary`: drei Sätze aus allen Berichten, die am meisten von dem sagen, was alle sagen + (LexRank), ohne einander zu wiederholen (MMR); `[{source, text}]`; +- `novel`: je Quelle bis zu zwei Sätze, denen in keinem anderen Bericht etwas ähnelt (tf-idf-Kosinus + < 0,2), ohne die schon im Überblick; `{source: [text]}`; +- `words`: je Quelle ihre eigenen Wörter für das Ereignis (Log-Odds gegen die übrigen Berichte, mit + Dirichlet-Prior; nur großgeschriebene Formen, also Substantive und Namen); `{source: [word]}`; +- `related`: dieselbe Meldung in der anderen Sprache (Deutsch/Englisch) – Meldungen der letzten zwei + Tage, die mindestens drei seltene Namen teilen (nach idf gewichtet, Summe ≥ 17); Story-Objekte wie + in `/feeds/news/`. + +Die App zeigt das über dem Artikel als „Überblick aus N Quellen“, darunter die Meldungen in der +anderen Sprache. Android lädt die Überblicke ungelesener Meldungen mit vor (Repo.kt), damit sie auch +offline da sind. + +## Themen + +Die erste Seite von `GET /feeds/news/` hat `topics: [{label, ids}]`: Meldungen der letzten zwei +Tage, die von derselben Sache handeln, nicht nur vom selben Ereignis. Gruppiert nach Titel und +Anriss (Leader-Clustering gegen die Mittelpunkte, drei Runden). Ein Thema hat mindestens drei +Meldungen aus mindestens zwei Quellen, liegt dicht beieinander (mittlere Ähnlichkeit zum Mittelpunkt +≥ 0,3) und hat ein Etikett aus mindestens zwei großgeschriebenen Wörtern, die die meisten seiner +Meldungen teilen. Mit `category` nur der Anteil der Kategorie. `?ids=1,2,3` liefert genau diese +Meldungen (ein Thema antippen). + +## Verworfen + +Stimmung (SentiWS), steigende Begriffe, Fortsetzungsketten (Nachrichten laufen nach zwei Tagen ab), +Clickbait-Erkennung: im Versuch zu ungenau oder ohne Nutzen. Persönliche Reihenfolge: erst, wenn +genug gelesen ist, um daraus zu lernen. diff --git a/docs/podcasts.md b/docs/podcasts.md new file mode 100644 index 0000000..566f414 --- /dev/null +++ b/docs/podcasts.md @@ -0,0 +1,184 @@ +# Podcasts: Strom, Merkliste, Wiedergabeliste + +The plan agreed on 2026-10-19 (with the user, from their listening history). It replaces +the inbox model ("Neu" to work through, "Folgen" beside it, one queue that is both a store +and the play order). Server: diora-web (podcasts/). The podcast UI is diora-android's +`web-native/static/native-podcast.js`, copied into `static/podcasts/` — built there. + +## Why + +- The subscriptions (about 540) are a self-chosen part of the podcast world: episodes to see + go by and dip into now and then — not a list to work through. "Neu" had 3360 unplayed + episodes; nobody works through that. +- The queue held 74 episodes (100+ hours): a store of "want to hear some day", yet the + player played it in order as if it were the play order. +- In AntennaPod "download automatically" and "add to queue automatically" were one thing + for the user: a regular podcast is one whose new episodes are on the phone. diora split + them (177 feeds downloading, 8 queueing). +- "Silent" subscriptions came from AntennaPod fetching every feed on the phone, slowly. The + server fetches all feeds anyway now (conditional, every 15 minutes). + +## The lists + +**Wiedergabeliste** — short, what plays next. The player goes through it in order and +**stops when it is empty** (it does not take from the Merkliste). Every episode in it is +downloaded. Filled by hand: ▶ on an episode (plays it, puts it first), or "In die +Wiedergabeliste" from the Merkliste or the Strom. + +**Merkliste** — long, "want to hear". It may grow; it **never expires**. Grouped by podcast, +each podcast **collapsible** (newest first inside). New episodes of Stamm subscriptions land +here and are downloaded; so is what the user adds with ＋. An episode played from it moves +to the front of the Wiedergabeliste; finished, it leaves both. + +When an episode leaves both lists (heard, removed), its download is deleted (setting +`delete_dequeue`). Downloads follow the lists, not the subscriptions. + +## Subscriptions: three levels + +| level | new episodes | today's mapping | +|---|---|---| +| **Stamm** | into the Merkliste and downloaded (per feed: or straight into the Wiedergabeliste — an option, used by none yet) | `new_episodes='queue'` (8) plus the regularly heard ones (about 50, list below) | +| **Strom** (default) | in the Strom; nothing to work through, nothing downloaded | `'inbox'`, and the former silent `'ignore'` ones | +| **Auf Abruf** | not in the Strom; on the podcast's page and in search | — (for exceptions) | + +The per-feed "download automatically" switch goes: Stamm means it. + +**Fetch interval** per subscription: *Automatisch* (by publishing rhythm, `interval_days`: +daily publishers hourly, rare ones daily), *stündlich*, *täglich*, *wöchentlich*. The former +silent subscriptions become Strom with *täglich*. + +## The Strom (replaces "Neu" and "Folgen") + +All episodes of all Strom and Stamm subscriptions, by period (Heute, Gestern, Diese Woche, +…). Within a period the podcasts are ordered by **how much the user has listened to them** +(episodes heard): every podcast with at least one heard episode before every one without — +those count for infinitely more (heard in the last 12 months, `heard_365`); among the never-heard ones, rarely publishing first. +Each podcast appears **once per period, always bundled**: its newest episode, and "+n ›" for the others. Episodes new since +they were last seen carry a **small mark** (no counts). Per episode: ▶ (Wiedergabeliste, +first), ＋ (Merkliste), ★. Filters stay: Favoriten, Verlauf, Angefangen, Gehört. + +Tabs: **Wiedergabeliste · Merkliste · Strom · Abos**. + +## Favourite subscriptions + +★ on a subscription (`PodcastFeed.favorite`): a mark and a filter (Abos: only the favourites; +episode lists: `filter=fave_feeds`), whatever its level — it puts nothing on the Merkliste by +itself. Started with the Stamm subscriptions marked. + +## Stöbern (decided 2026-10-19) + +Within the subscriptions: +- **Search** in titles and shownotes: `GET /podcasts/discover/search/?q=…&stream=1` (all + words, the last as a prefix; titles weigh more). SQLite FTS5 index, contentless + (`podcasts/discover.py`), filled after every refresh; `manage.py index_episodes` fills it. +- **Similar episodes** from the other podcasts: `GET /podcasts/discover/similar//`. +- **People** (who talks, who gives a talk), from episode titles only (`podcasts/people.py`): + `GET /podcasts/discover/people/` → `[{name, episodes, feeds, score}]`, the ones in the most + listened podcasts first; `GET /podcasts/discover/person/?name=` → their episodes. +- **Topics** of the Strom's last days, chips as for the news: `GET /podcasts/discover/topics/?days=7` + → `[{label, ids}]`. +- **Forgotten**: `GET /podcasts/discover/forgotten/` → `{loved: [...], rare: [...]}` — unheard + episodes older than 60 days of the podcasts listened to most, and of the rare ones; the same + all day. + +One controlled step out (Apple's episode search and fyyd; no key): +- **People elsewhere**: `GET /podcasts/discover/elsewhere/?name=` → episodes naming the person + in podcasts the user doesn't have (`{source, podcast, feed_url, title, audio_url, guid, + pub_date, duration_seconds, artwork_url, link}`). `POST /podcasts/discover/take/` with such an + item puts the episode on the Merkliste; its podcast becomes an *Auf Abruf* subscription. +- **Recommendations**: `GET /podcasts/discover/recommend/` → podcasts like the most listened ones + (fyyd), recommended for several first: `{title, feed_url, artwork_url, description, link, + because: [titles], count}`. Subscribing: the usual feeds/add/ (then *Auf Abruf* or Strom). + +Show search stays on Apple (best coverage). + +## Podcast-Radio (since 2026-10-25) + +Listening without committing: one button plays the Strom like a station. + +- **The rule**: the radio always plays the **newest** (publication date) Strom episode that is + neither played, nor started (normally), nor done for the radio. So an episode that comes out + while one plays is next (the one playing isn't interrupted); otherwise it goes back in time, + without limit. Auf Abruf podcasts and audiobooks aren't in the Strom, so not in the radio. +- **Mixed**: several episodes of one podcast in a row are mixed with the others waiting — the + next is the newest of *another* podcast than the one before, if there is one (within the + newest 200). +- What comes next is asked anew at every end and skip (so the newest really comes next); the + list fetched before is for "Danach: …" in the player and for going on offline. +- **Leaves no trace**: no progress (no Verlauf, no "Angefangen", no played mark, not counted in + `heard_365`), the Wiedergabeliste and Merkliste unchanged, nothing downloaded (it streams; a + downloaded file plays if there is one), the Strom's marks unchanged. +- **Done for the radio**: run to its end or skipped (⏭, notification, headset, car). Pausing + isn't: the radio keeps its episode and place for itself, ▶ goes on there. +- **Committing**: in the player while the radio plays, ＋ (Merkliste) and "Normal weiterhören" + (first in the Wiedergabeliste at the current place; from then on it plays as usual, with + progress). Playing anything normally switches the radio off (as an audiobook does). +- **Server** (`podcasts/radio.py`): `PodcastRadioHeard(user, episode, at)`, kept apart from + progress and kept for good (pruned, an old episode would come back when the radio goes back + that far). `GET /podcasts/radio/?after=&skip=&limit=5` → the next episodes by + the rule (`{episodes: [...]}`, the rows of the episode lists; `after` isn't among them and its + podcast not first; `skip`: ones the device is done with whose /heard/ may still be on its + way); `POST /podcasts/radio/heard/ {"episode_id"}`. Shared by all devices. Filter `untouched` + (episode lists): neither played nor listened into. +- **Player sync**: a report carries `podcast_radio: true` while the radio plays + (`PlayerState.podcast_radio`); taking it over ("Hier abspielen", a transfer) or adopting it goes + on with the radio on that device. +- **Clients**: the web player (`podcasts-web.js`: an entry with `radio: true`, the radio's own + `podradio` in localStorage — next ones, its place, what it was done with lately; host + `podcastRadio()`, `podcastRadioCommit()`, `podcastRadioLater()`; `playerState().radio = + {next}`) and Android (`PodcastRadio.kt` beside `BookPlay.kt`; RadioService's `save()` / + `markPlayed()` / `followQueue()` leave radio items alone). UI (native-podcast.js): 📻 at the top + of the Strom, "Podcast-Radio · " in the mini player and the player, there a box with + "Danach: …", ＋ Merkliste and "Normal weiterhören"; ⏭ only skips. +- Later perhaps: a radio of one folder, a longest length. + +## Moving over + +- Today's queue (74) becomes the Merkliste; the Wiedergabeliste starts empty. +- Stamm: the 8 automatic subscriptions and the regularly heard ones (≥ 8 episodes a year or + ≥ 3 in 90 days, 2026-10-19): Ones and Tooze, Die Neuen Zwanziger – Der Salon, Surplus, + The Deprogram, junge Welt – Tagesüberblick, Speakeasy-Bar, jwpod – Forschung, RLS + Audiodokumente, The Reaction Pod, Geschichten aus der Geschichte, Märkte Mächte Emissionen, + friendlyjordies, Second Thought, Neben der Spur, Forschungsquartett, GegenStandpunkt – + Artikel, Hörsaal, Hakim, Marxistische Vorträge, Blätter, Kontextfenster, New Left Review, + Kommunisten Kneipe, Jung & Naiv, HealthyGamerGG, Dekarldent, Daily Dose Of Dekarldent, + Eine Stunde History, Übertage, Eigentlich, Zwischen Zwei Deckeln, UNFASSBAR, Über Rechts, + TitosLeftLung, New Books in History, ARTEde, Irgendwas mit ARTE, Zwei wie Pech & Schwafel, + Geschichte der kommenden Welten, Vollbild, Zeitfragen, Zeitfragen. Feature, Die Filmanalyse, + Reaction, Wirtschaft am Mittag. +- Everything else: Strom; the silent ones with *täglich*. + +## Order of work + +1. Wiedergabeliste and Merkliste (server: a second list, moving the queue over, the player's + order; UI: two tabs, grouped Merkliste). +2. The three levels and the fetch interval; downloads following the lists. +3. The Strom replacing "Neu" and "Folgen". +4. Browsing — after more thought. + +## API (server, since 2026-10-19) + +- `PodcastSettings.lists_v2` turns the two lists on. `POST /podcasts/lists/ {"on": true}` + moves the queue into the Merkliste (the Wiedergabeliste starts empty); `{"on": false}` puts + the Merkliste back at the end of the queue. Off, everything works as before (apps not + updated yet). +- **Wiedergabeliste** = the queue as it was: `state.queue`, `/podcasts/queue/add|remove|reorder/`. + The player goes through it and stops at its end. +- **Merkliste** = `PodcastLater`: `state.later` (rows like the queue's, newest first, plus + `added_at`, `auto` — added by a Stamm feed — and `in_queue`), `POST /podcasts/later/add/` and + `/podcasts/later/remove/` `{"episode_ids": [...]}`. Heard (progress at the end, mark-played) + and "Folge löschen" take an episode off it. Episode lists carry `in_later`; filter `later`. +- **Levels**: `PodcastFeed.new_episodes` — `queue` = Stamm into the Merkliste (with lists_v2; + without it: the queue), `play` = Stamm into the Wiedergabeliste, `inbox` = Strom, + `ignore` = Auf Abruf. `auto_queue` stays true for queue and play. +- **Fetch interval**: `PodcastFeed.refresh_interval` — `''` (by rhythm: every `interval_days` + hours, within 15 minutes and 24 hours), `hourly`, `daily`, `weekly`; the cron's + `refresh_feeds` only fetches feeds that are due (`views.refresh_due`). +- **Strom**: `GET /podcasts/episodes/?filter=stream` (all but Auf Abruf; with `folder=`, `sort=`); + episodes carry `discovered_at`. Per feed in `state.feeds`: `heard_365` (episodes heard in the + last 12 months: played, or listened into for more than a minute) and `interval_days`. + `POST /podcasts/stream/seen/` → `{before, now}`; `state.stream_seen_at`. An episode is marked + when its `discovered_at` is after the time before the current look. +- Web host (`static/js/podcasts-web.js`, `DioraPodcastHost`): `podcastLaterAdd(json episodes)`, + `podcastLaterRemove(json ids)`, `podcastListsSwitch(on)`, `podcastStreamSeen()` (keeps the + previous time as `state.stream_seen_before` for the marks). diff --git a/docs/progress-sync.md b/docs/progress-sync.md new file mode 100644 index 0000000..9ab05ac --- /dev/null +++ b/docs/progress-sync.md @@ -0,0 +1,71 @@ +# Reading position: the last place read wins + +Since 2026-10-12 the place to continue a book is **where it was last read, on any device**, not the +furthest place ever reached. The furthest place is still kept, as a hint ("Weiteste Stelle"). +This file is the contract between the server (books/views.py `save_progress`), the web reader +(static/js/app/46-readpos.js, with the reader in 62-reader.js), diora-android (Ops.kt / SyncEngine.kt / native-*.js) and diora-desktop +(src/main/ops.js / sync.js). They must apply the same rules. + +## The state per book + +| field | meaning | +|---|---| +| `scroll_fraction`, `position_anchor` | the continue point (anchor `"block:inner"`, empty for PDFs) | +| `read_at` | when the reader was **at** that place, ms since epoch, **server clock** | +| `device` | who read there (`"Browser"`, `"diora Desktop"`, `"Handy"`, `"tolino"` …), for "auf X weitergelesen" | +| `furthest_fraction`, `furthest_anchor` | the furthest place ever reported (only ever grows) | + +`last_read` in `/books/` is `read_at` as ISO time (it used to be the time of the last save attempt). + +## Writing: `POST /books//progress/` + +```json +{"scroll_fraction": 0.42, "position_anchor": "187:0.62", "read_at": 1760270000000, + "sent_at": 1760270004000, "device": "diora Desktop"} +``` + +* `read_at` is **the moment the reader arrived at this position** (the last time the visible position + changed because of the reader), in the **device's** clock, not the moment of saving. Saving an + unchanged position again — every 10 s, on hide, on close, on unload — therefore carries the old time + and can never beat a newer position from another device. A device that only *shows* a position it + got from the server (just opened, nothing moved) reports the server's `read_at` back unchanged. +* `sent_at` is the device clock **when the request leaves** (an outbox sets it at flush time, not + when queuing). The server corrects the device's clock error with it: + `t = read_at + (server_now − sent_at)`, then `t = min(t, server_now)`. + So a tolino whose clock is hours off still orders correctly, and nothing can win "from the future". +* The server stores the position if `t >= stored read_at` (or nothing is stored, or `force`). + Older writes (an offline device catching up late) are **not** stored as the continue point. +* Every write — stored or not — raises `furthest_*` if it is further (same comparison as before: + anchor block, then inner fraction; `scroll_fraction` when an anchor is missing). +* `force: true` (the explicit "Diese Stelle als Fortschritt setzen") stores regardless, with `t = server_now`. +* **Old clients** (no `read_at`): stored only if **strictly further** than the stored position + (the old rule, but `>` so an idle old client re-saving its place changes nothing), with `t = server_now`. + This keeps not-yet-updated apps from pulling the continue point back. +* The response is the state after the write: + `{"ok": true, "stored": bool, "scroll_fraction", "position_anchor", "read_at", "device", + "furthest_fraction", "furthest_anchor", "server_now"}`. + +## Reading + +`/books/` and `/api/sync/` (`book_progress`) carry all fields of the state, `read_at` in ms (server clock), +plus `server_now` (ms) so a client can estimate its clock offset `server_now − device_now`. + +## Clients + +* **Local apply / outbox** (Android, desktop, web cache): the same rule as the server, in server time: + a local write converts its `read_at` with the last known offset (0 if none), stores it if newer + than or equal to what the local list holds. Two queued writes for the same book: keep the one with + the later `read_at` (force stays sticky). After a pull, a pending write is reapplied with that rule: + newer than the server's → it shows (and is sent); older → the server's position shows (it's still sent, + for `furthest`). +* **Never save before the position is restored** after opening a book, and never save a position the + reader didn't reach by reading: while a jump is tentative ("Zurück zum Text" after a footnote or link, + "Weit gescrollt — Zurückspringen"), the previous position and its `read_at` are what gets saved, until + the reader returns (nothing changes) or carries on reading there (then the jump target counts, with + its own time). +* **Opening a book** uses the newer of server state and local (not yet sent) state, by `read_at`. +* **Coming back to an open book** (window/app in front again, or after a pull): if the server holds a + newer position from another device than the one shown, offer it: "Auf weitergelesen — + Springen". Do not move the page without asking. +* If `furthest_*` is clearly ahead of the continue point when a book opens, offer + "Weiteste Stelle: 63 % — Springen". diff --git a/docs/projects.md b/docs/projects.md new file mode 100644 index 0000000..6616a2b --- /dev/null +++ b/docs/projects.md @@ -0,0 +1,96 @@ +# Projekte + +Projects of a preset: the first is **Forschung** (a research project: sources, citations, +notes; Radar and Schreiben follow). In the navigation (`nav.js`, section and group `projekte`, +flag `side`): a tab beside Home · Lesen · Hören on a phone, at the foot of the sidebar and of +the wide bar above Suche · Abos & Quellen. The search moved out of the phone's tab bar: a +magnifier in the bar on top (`suche` is a footer section; the apps' section bars show it). + +## End-to-end encrypted + +Like the books (docs/library.md): a project's data (`{name, description}`) and every item's +data are AES-GCM blobs with the user's key (`32-crypto.js`, iv hex, ciphertext base64). The +server stores them and knows only: a project exists, its preset, archived, and what an item +points to — `kind` + `ref` (a book's, episode's or article's id, never its title). + +## Items + +| kind | ref | data | +|---|---|---| +| `paper` | – | `{key, csl, note, doi}` (CSL-JSON from doi.org) | +| `book` | book id | `{key, csl, note, type}` (CSL from the book's meta) | +| `episode` | episode id | `{key, csl, note, podcast, feed, position}` (position: seconds, "ab 12:34") | +| `article` | feeds item id | `{key, csl, note}` | +| `note` | – | `{title, body (Markdown), folder}` | + +A book, episode or article is in a project once (adding again answers `existing: true`). +`key` is the citation key for `[@key]` / `[@key, S. 12]` in notes: first author's family name +(or the podcast's), year, first word of the title — unique in the project, editable. + +Notes: Markdown (headings, lists, quotes, bold/italic, links); the view shows citations as +"(Amodio & Jost 2007, S. 3)" and a list of the works cited. Saved as one types (0.9 s). +Export: BibTeX and CSL-JSON of the sources (all or the chosen kind). + +## Linking from elsewhere + +`DioraProjects.link(kind, ref, info)` opens the sheet "Mit Projekt verknüpfen" (the projects, +the ones it is in ticked, ＋ Neues Projekt). Called from the book sheet (library, apps), the +article reader (◇ in its bar), the episode sheet and the player (apps, native-podcast.js). +`DioraProjects.of(kind, ref)` → `[{id, name}]` the projects it's in; `onChange(fn)` after changes. + +## Desktop: a window of its own + +The project's menu (⋯) has "In neuem Fenster öffnen" when `dioraDesktop.openWindow` is there: +the desktop opens the page with `?project=`; `projects.js` then shows only that project +(`html.prj-solo` hides the navigation and the dock). + +## Writing and conflicts + +Every write sends `base_iv` (the IV it changed). Changed elsewhere since: 409 with the current +state. A note's text then goes beside it as a copy "(Konflikt)"; other data reloads. + +## API (signed in) + +- `GET /projects/` → `{projects: [{id, preset, data_ct, data_iv, archived, position, items, …}], links: [{project, kind, ref}], presets}` +- `POST /projects/create/` `{preset, data_ct, data_iv}` → `{project}` +- `GET /projects//` → `{project, items: [{id, kind, ref, data_ct, data_iv, created_at, updated_at}]}` +- `POST /projects//` `{data_ct, data_iv, base_iv}` and/or `{archived}` / `{position}` +- `POST /projects//delete/` +- `POST /projects//items/add/` `{kind, ref?, data_ct, data_iv}` → `{item, existing}` +- `POST /projects/items//` `{data_ct, data_iv, base_iv}` → `{item}` (409 `{item}` when stale) +- `POST /projects/items//delete/` +- `GET /projects/doi/?q=` → `{doi, csl}` — fetched from + doi.org (Crossref, DataCite …) as CSL-JSON, cached a week, nothing else kept. + +## Radar + +A project can have a Radar (`projects/radar.py`, `projects/papers.py`): every week the new papers +from PubMed, arXiv, bioRxiv and medRxiv, sorted by Claude into the config's groups (the first +one — "lesenswert" — with what it's about, how solid it is, the political hook), the week's +news (web search), a basic term explained. One Claude batch (Opus, half price): about 0.40 $ +a week. Ported from the "Neuropolitik Grundlagen" session's `F: +europolitik ools` +(fetch_papers.py, digest_batch.py); the digest is their bundle format (version 1). + +- Config per radar (`Radar.config`, presets in `projects/radar_presets.py`): PubMed queries per + topic, arXiv query, keyword pattern for the preprint servers, groups and their labels, the + prompts, weekday and hour (Europe/Berlin), days. **Plain on the server**: it searches with them. + The papers and digests are public material, plain too (`RadarDigest.bundle`). +- The decisions (gelesen / später / verworfen / in Quellen) are the user's: an encrypted item + of kind `radar` (`{decisions: {paper id: state}}`). "→ Quellen" makes a `paper` source + (citation data from doi.org when there's a DOI, else from the record). +- Runs from the host's crontab every half hour: `docker exec diora-prod python manage.py radar` + (submits what is due, collects finished batches). Needs `ANTHROPIC_API_KEY` in the container. + `radar --setup neuropolitik` gives a project its Radar; `radar --import + ` takes a digest made elsewhere; `radar --seen ` ids reported before. +- API: `GET /projects//radar/` → `{radar: {enabled, title, groups, group_labels, weekday, + hour, days, sources, running, submitted, last_run_at, last_error, next_run, key}, digests: + [{id, generated, bundle}]}` (the latest 12); `POST …/radar/settings/` `{enabled}`; + `POST …/radar/run/` (fetch at the next cron run); `POST …/radar/import/` (a bundle, e.g. + `digest_batch.py --diora-url`). + +## Next + +Papers as books (a paper's PDF into the library, its highlights +into the project's notes), Schreiben (drafts with the bibliography in a citation style), +the projects in the search, more presets. diff --git a/docs/radio.md b/docs/radio.md new file mode 100644 index 0000000..2d6ee05 --- /dev/null +++ b/docs/radio.md @@ -0,0 +1,137 @@ +# Radio, Ambient, Fokus + +The oldest part of diora (it began as a radio player): internet radio with saved stations, +history and track titles, the generative Ambient player, and the web page's Fokus timer. +Written down 2026-10-24 from the code; it is the reference for the three apps. + +## Server: the `radio` app + +Models (`radio/models.py`): + +| model | what | +|---|---| +| `SavedStation` | a user's station: name, url (unique per user), bitrate, country, tags, `favicon_url` (the logo), `is_favorite`, notes | +| `StationPlay` | one listening: station, started/ended; hour, weekday, weekend filled on save (for the recommendations) | +| `TrackHistory` | a track title heard (user, or `session_key` signed out), `scrobbled` | +| `FeaturedStation` | stations picked in the admin for everybody (order, active) | +| `FocusSession` | a finished Fokus round | + +Also: `podcasts.PlayerState` with `kind='radio'` (the player sync between devices) and +`accounts.UserProfile.focus_station_url/_name` (the station the Fokus timer plays). + +Endpoints (`radio/urls.py`): + +- `GET /` — the page (`templates/radio/player.html`). The saved stations, the history (50) and + the featured stations come rendered into it (`#saved-tbody`, `#history-tbody`, + `.featured-section`) and as `INITIAL_SAVED` (with `play_count`), `INITIAL_FEATURED`, + `USER_FOCUS_STATION`. The apps take these parts from it (see below). +- `GET /radio/sse/?url=&station=` — server-sent events with the track title, `data: {"track": …}` when it + changes, `{"error": "stream ended"}` at the end. The server opens the stream itself with + `Icy-MetaData: 1` and reads `StreamTitle` (`radio/icy.py`). Which addresses it opens + (`radio/streamcheck.py`, since 2026-10-24): never one in a private network (loopback, + 10.x, 172.16–31.x, 192.168.x, link-local, the Docker network); **without an account only the + directory's stations** — the page's curated lists (read from `16-stations.js`), the featured + stations, the creamfresh stream, and what radio-browser.info lists (`station=` its station + id, which the page sends for search results; else by the address; kept a day). An address of + one's own is for saving, and saving needs an account: signed out, a pasted address offers + signing in instead of ▶. +- `POST /radio/record/` — a track into the history; `scrobble: true` sends it to Last.fm when the + account has it linked (`radio/lastfm.py`; "Artist - Title", else the station as artist). +- `POST /radio/save/` (by url: create or update), `/radio/remove//`, `/radio/favorite//`, + `/radio/notes//`; `GET /radio/history/` (last 200), `POST /radio/history//delete/`. +- `POST /radio/play/start/` → `play_id`, `/radio/play/stop/`; `GET /radio/recommendations/` — + the five stations most played around this hour on this kind of day, with a German label. +- `POST /radio/import/` — an `.m3u`/`.m3u8` file into the saved stations (`#EXTINF` names) → + `{added, skipped}`. +- `POST /radio/focus/record/`, `GET /radio/focus/stats/`. +- `GET /radio/affiliate/?track=` — artwork/album from iTunes (an Amazon link only with + `AMAZON_AFFILIATE_ENABLED`). +- `GET /radio/creamfresh-stream/`, `POST /radio/creamfresh-feedback/` — the user's own station + (radio.creamfresh.xyz) through the server, which adds its login; 👍/👎 for the DJ. +- `GET /radio/stream-player/` — an old standalone player page; nothing links to it. + +**Station search** is not on the server: the page asks radio-browser.info directly +(`de1.api.radio-browser.info/json/stations/search`, 50, by click count). A pasted `http(s)://` +address is offered to play or save as it is. + +## Web page (`static/js/app/`) + +| file | what | +|---|---| +| `01-core.js` | the globals: `audio` (one `