Buchdateien in eine eigene Tabelle (EBookFile): Änderungen an Titel, Ordner, gelesen schreiben nicht mehr megabyteweise die Datei mit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

books/migrations/0008_ebook_file.py

@@ -0,0 +1,38 @@
+# Generated by Django 6.1 on 2026-10-19 15:02
+# The books' files move out of their rows into a table of their own (docs/database.md):
+# copied in one statement, then the two columns are dropped (ALTER TABLE DROP COLUMN).
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('books', '0007_list_indexes'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='EBookFile',
+            fields=[
+                ('book', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='file', serialize=False, to='books.ebook')),
+                ('data_ct', models.TextField()),
+                ('data_iv', models.CharField(max_length=32)),
+            ],
+        ),
+        migrations.RunSQL(
+            'INSERT INTO books_ebookfile (book_id, data_ct, data_iv) SELECT id, data_ct, data_iv FROM books_ebook',
+            reverse_sql='UPDATE books_ebook SET '
+                        'data_ct = (SELECT f.data_ct FROM books_ebookfile f WHERE f.book_id = books_ebook.id), '
+                        'data_iv = (SELECT f.data_iv FROM books_ebookfile f WHERE f.book_id = books_ebook.id)',
+        ),
+        migrations.RemoveField(
+            model_name='ebook',
+            name='data_ct',
+        ),
+        migrations.RemoveField(
+            model_name='ebook',
+            name='data_iv',
+        ),
+    ]

books/models.py

@@ -2,12 +2,20 @@ from django.db import models
 from django.contrib.auth.models import User
 
 
+class EBookManager(models.Manager):
+    def create(self, data_ct=None, data_iv=None, **kwargs):
+        """A book, and with data_ct/data_iv its file (EBookFile) in the same go."""
+        book = super().create(**kwargs)
+        if data_ct is not None:
+            EBookFile.objects.create(book=book, data_ct=data_ct, data_iv=data_iv or '')
+        return book
+
+
 class EBook(models.Model):
     user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ebooks')
     meta_ct = models.TextField()          # base64 AES-GCM ciphertext of {title, author, filename}
     meta_iv = models.CharField(max_length=32)  # hex IV for metadata
-    data_ct = models.TextField()          # base64 AES-GCM ciphertext of raw EPUB bytes
-    data_iv = models.CharField(max_length=32)  # hex IV for EPUB data
+    # The file itself is in EBookFile (book.file), not here (docs/database.md).
     uploaded_at = models.DateTimeField(auto_now_add=True)
     is_read = models.BooleanField(default=False)
     # The cover, end-to-end encrypted like the rest (docs/library.md): a client that has
@@ -15,11 +23,12 @@ class EBook(models.Model):
     cover_ct = models.TextField(blank=True, default='')
     cover_iv = models.CharField(max_length=32, blank=True, default='')
 
+    objects = EBookManager()
+
     class Meta:
         ordering = ['uploaded_at']
-        # The list reads these from indexes, never from the row: the file (data_ct) stands
-        # before them in it, and SQLite walks through all of a file to reach a column after it
-        # — 37 s for a library of 400 books (docs/database.md). See book_rows().
+        # The list reads these from indexes (book_rows()); they made it fast while the file
+        # was still in this row (docs/database.md), and cost nothing now.
         indexes = [
             models.Index(fields=['user', 'uploaded_at', 'is_read'], name='ebook_list_idx'),
             models.Index(fields=['user', 'cover_iv'], name='ebook_cover_idx'),
@@ -29,6 +38,18 @@ class EBook(models.Model):
         return f"EBook #{self.pk} (user={self.user_id})"
 
 
+class EBookFile(models.Model):
+    """A book's file, encrypted like everything else: base64 AES-GCM of the EPUB/PDF bytes.
+    In a table of its own (docs/database.md): with the file in the book's row, every change of a
+    title, a folder or "gelesen" rewrote megabytes, and the list read past every file."""
+    book = models.OneToOneField(EBook, on_delete=models.CASCADE, primary_key=True, related_name='file')
+    data_ct = models.TextField()
+    data_iv = models.CharField(max_length=32)
+
+    def __str__(self):
+        return f"EBookFile of #{self.book_id}"
+
+
 def book_rows(user):
     """Every book of the user as {id, meta_ct, meta_iv, uploaded_at (ISO), is_read, has_cover},
     oldest first — read without passing a book's file: meta_ct and meta_iv stand before it in

books/tests.py

@@ -225,3 +225,37 @@ class BookRowsReadPastNoFileTests(TestCase):
                     cur.execute('EXPLAIN QUERY PLAN ' + sql.replace('%s', '?'), [user.id] if '?' in sql or '%s' in sql else [])
                     plan = ' '.join(str(r) for r in cur.fetchall())
                     self.assertIn('COVERING INDEX', plan, sql)
+
+
+class EBookFileTests(TestCase):
+    """The file lives in EBookFile; upload, data, replace-data and rekey go there."""
+
+    def setUp(self):
+        from django.contrib.auth.models import User
+        self.user = User.objects.create_user('filer', password='pw-filer-123')
+        self.client.force_login(self.user)
+
+    def test_upload_read_replace_rekey_delete(self):
+        import base64, json
+        from books.models import EBook, EBookFile
+        data = base64.b64encode(b'x' * 100).decode()
+        r = self.client.post('/books/upload/', json.dumps({'meta_ct': 'm', 'meta_iv': 'i', 'data_ct': data, 'data_iv': 'd'}),
+                             content_type='application/json')
+        pk = r.json()['id']
+        self.assertEqual(EBookFile.objects.get(book_id=pk).data_ct, data)
+        self.assertEqual(self.client.get(f'/books/{pk}/data/').json(), {'data_ct': data, 'data_iv': 'd'})
+        self.client.post(f'/books/{pk}/replace-data/', json.dumps({'data_ct': 'bmV3', 'data_iv': 'd2'}), content_type='application/json')
+        self.assertEqual(self.client.get(f'/books/{pk}/data/').json()['data_ct'], 'bmV3')
+        self.client.post(f'/books/{pk}/rekey/', json.dumps({'meta_ct': 'm3', 'meta_iv': 'i3', 'data_ct': 'cmVr', 'data_iv': 'd3'}),
+                         content_type='application/json')
+        self.assertEqual(EBook.objects.get(pk=pk).meta_ct, 'm3')
+        self.assertEqual(EBookFile.objects.get(book_id=pk).data_iv, 'd3')
+        self.client.post(f'/books/{pk}/delete/')
+        self.assertFalse(EBookFile.objects.filter(book_id=pk).exists())
+
+    def test_someone_elses_file_is_not_found(self):
+        from django.contrib.auth.models import User
+        from books.models import EBook
+        other = User.objects.create_user('other-filer', password='pw-other-123')
+        b = EBook.objects.create(user=other, meta_ct='m', meta_iv='i', data_ct='ZA==', data_iv='i')
+        self.assertEqual(self.client.get(f'/books/{b.id}/data/').status_code, 404)

books/views.py

@@ -12,7 +12,7 @@ from django.http import JsonResponse
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_http_methods
 
-from .models import EBook, EBookProgress, EBookHighlights, EBookBookmarks, LibraryPrefs, book_rows
+from .models import EBook, EBookFile, EBookProgress, EBookHighlights, EBookBookmarks, LibraryPrefs, book_rows
 
 
 def _require_auth(request):
@@ -443,14 +443,15 @@ def upload_book(request):
     if raw_size > max_bytes:
         return JsonResponse({'error': f'file too large (max {settings.EBOOK_MAX_BYTES // (1024 * 1024)} MB)'}, status=400)
 
-    book = EBook.objects.create(
-        user=request.user,
-        meta_ct=meta_ct,
-        meta_iv=meta_iv,
-        data_ct=data_ct,
-        data_iv=data_iv,
-    )
-    EBookProgress.objects.create(user=request.user, book=book)
+    with transaction.atomic():
+        book = EBook.objects.create(
+            user=request.user,
+            meta_ct=meta_ct,
+            meta_iv=meta_iv,
+            data_ct=data_ct,
+            data_iv=data_iv,
+        )
+        EBookProgress.objects.create(user=request.user, book=book)
     return JsonResponse({'ok': True, 'id': book.id})
 
 
@@ -460,12 +461,10 @@ def get_book_data(request, pk):
     if err:
         return err
 
-    try:
-        book = EBook.objects.get(pk=pk, user=request.user)
-    except EBook.DoesNotExist:
+    f = EBookFile.objects.filter(book_id=pk, book__user=request.user).first()
+    if f is None:
         return JsonResponse({'error': 'not found'}, status=404)
-
-    return JsonResponse({'data_ct': book.data_ct, 'data_iv': book.data_iv})
+    return JsonResponse({'data_ct': f.data_ct, 'data_iv': f.data_iv})
 
 
 @csrf_exempt
@@ -493,14 +492,12 @@ def replace_book_data(request, pk):
     if not all([data_ct, data_iv]):
         return JsonResponse({'error': 'data_ct and data_iv required'}, status=400)
 
-    update_fields = ['data_ct', 'data_iv']
-    book.data_ct = data_ct
-    book.data_iv = data_iv
-    if meta_ct and meta_iv:
-        book.meta_ct = meta_ct
-        book.meta_iv = meta_iv
-        update_fields += ['meta_ct', 'meta_iv']
-    book.save(update_fields=update_fields)
+    with transaction.atomic():
+        EBookFile.objects.update_or_create(book=book, defaults={'data_ct': data_ct, 'data_iv': data_iv})
+        if meta_ct and meta_iv:
+            book.meta_ct = meta_ct
+            book.meta_iv = meta_iv
+            book.save(update_fields=['meta_ct', 'meta_iv'])
     return JsonResponse({'ok': True})
 
 
@@ -529,11 +526,11 @@ def rekey_book(request, pk):
     if not all([meta_ct, meta_iv, data_ct, data_iv]):
         return JsonResponse({'error': 'meta_ct, meta_iv, data_ct, data_iv required'}, status=400)
 
-    book.meta_ct = meta_ct
-    book.meta_iv = meta_iv
-    book.data_ct = data_ct
-    book.data_iv = data_iv
-    book.save(update_fields=['meta_ct', 'meta_iv', 'data_ct', 'data_iv'])
+    with transaction.atomic():
+        EBookFile.objects.update_or_create(book=book, defaults={'data_ct': data_ct, 'data_iv': data_iv})
+        book.meta_ct = meta_ct
+        book.meta_iv = meta_iv
+        book.save(update_fields=['meta_ct', 'meta_iv'])
     return JsonResponse({'ok': True})
 
 

docs/database.md

@@ -32,5 +32,16 @@ column *after* it, it walks the whole chain: listing 427 books read gigabytes an
 reads `id, meta_ct, meta_iv` from the rows (they stand before the file) and `uploaded_at,
 is_read` / whether there is a cover from the covering indexes `ebook_list_idx` and
 `ebook_cover_idx` — no list query touches a file. `books/tests.py` checks the query plans.
-Writes still rewrite the whole row (file included); moving the files into a table of their own
-would end that too.
+
+## The files in a table of their own (since 2026-10-19)
+
+`books.models.EBookFile` (one row per book, `book_id` its key) holds `data_ct`/`data_iv`;
+`EBook` has only the small fields. Before, every change of a title, a folder or "gelesen"
+rewrote the whole row with its file (megabytes, for a scanned volume tens of them), and every
+`EBook.objects.get()` loaded the file. `EBook.objects.create(data_ct=…, data_iv=…)` still
+works (the manager makes the EBookFile too); the API is unchanged.
+
+Migration `books/0008_ebook_file` copied 430 files in one statement and dropped the two
+columns: about 2 minutes on the server (rehearsed on a copy), the site down meanwhile. The
+pages the old columns used are free now and get reused; `VACUUM` (exclusive, minutes, at a
+quiet time) would give them back to the disk.