From 86a6bb52d7c7fb47b59835fc99812db2225e98b9 Mon Sep 17 00:00:00 2001 From: mrwnslz Date: Mon, 19 Oct 2026 17:11:19 +0200 Subject: [PATCH] =?UTF-8?q?Buchdateien=20in=20eine=20eigene=20Tabelle=20(E?= =?UTF-8?q?BookFile):=20=C3=84nderungen=20an=20Titel,=20Ordner,=20gelesen?= =?UTF-8?q?=20schreiben=20nicht=20mehr=20megabyteweise=20die=20Datei=20mit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- books/migrations/0008_ebook_file.py | 38 +++++++++++++++++++++ books/models.py | 31 +++++++++++++++--- books/tests.py | 34 +++++++++++++++++++ books/views.py | 51 ++++++++++++++--------------- docs/database.md | 15 +++++++-- 5 files changed, 135 insertions(+), 34 deletions(-) create mode 100644 books/migrations/0008_ebook_file.py diff --git a/books/migrations/0008_ebook_file.py b/books/migrations/0008_ebook_file.py new file mode 100644 index 0000000..e014bbc --- /dev/null +++ b/books/migrations/0008_ebook_file.py @@ -0,0 +1,38 @@ +# Generated by Django 6.1 on 2026-10-19 15:02 +# The books' files move out of their rows into a table of their own (docs/database.md): +# copied in one statement, then the two columns are dropped (ALTER TABLE DROP COLUMN). + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('books', '0007_list_indexes'), + ] + + operations = [ + migrations.CreateModel( + name='EBookFile', + fields=[ + ('book', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='file', serialize=False, to='books.ebook')), + ('data_ct', models.TextField()), + ('data_iv', models.CharField(max_length=32)), + ], + ), + migrations.RunSQL( + 'INSERT INTO books_ebookfile (book_id, data_ct, data_iv) SELECT id, data_ct, data_iv FROM books_ebook', + reverse_sql='UPDATE books_ebook SET ' + 'data_ct = (SELECT f.data_ct FROM books_ebookfile f WHERE f.book_id = books_ebook.id), ' + 'data_iv = (SELECT f.data_iv FROM books_ebookfile f WHERE f.book_id = books_ebook.id)', + ), + migrations.RemoveField( + model_name='ebook', + name='data_ct', + ), + migrations.RemoveField( + model_name='ebook', + name='data_iv', + ), + ] diff --git a/books/models.py b/books/models.py index ce958d2..e9e3b16 100644 --- a/books/models.py +++ b/books/models.py @@ -2,12 +2,20 @@ from django.db import models from django.contrib.auth.models import User +class EBookManager(models.Manager): + def create(self, data_ct=None, data_iv=None, **kwargs): + """A book, and with data_ct/data_iv its file (EBookFile) in the same go.""" + book = super().create(**kwargs) + if data_ct is not None: + EBookFile.objects.create(book=book, data_ct=data_ct, data_iv=data_iv or '') + return book + + class EBook(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ebooks') meta_ct = models.TextField() # base64 AES-GCM ciphertext of {title, author, filename} meta_iv = models.CharField(max_length=32) # hex IV for metadata - data_ct = models.TextField() # base64 AES-GCM ciphertext of raw EPUB bytes - data_iv = models.CharField(max_length=32) # hex IV for EPUB data + # The file itself is in EBookFile (book.file), not here (docs/database.md). uploaded_at = models.DateTimeField(auto_now_add=True) is_read = models.BooleanField(default=False) # The cover, end-to-end encrypted like the rest (docs/library.md): a client that has @@ -15,11 +23,12 @@ class EBook(models.Model): cover_ct = models.TextField(blank=True, default='') cover_iv = models.CharField(max_length=32, blank=True, default='') + objects = EBookManager() + class Meta: ordering = ['uploaded_at'] - # The list reads these from indexes, never from the row: the file (data_ct) stands - # before them in it, and SQLite walks through all of a file to reach a column after it - # — 37 s for a library of 400 books (docs/database.md). See book_rows(). + # The list reads these from indexes (book_rows()); they made it fast while the file + # was still in this row (docs/database.md), and cost nothing now. indexes = [ models.Index(fields=['user', 'uploaded_at', 'is_read'], name='ebook_list_idx'), models.Index(fields=['user', 'cover_iv'], name='ebook_cover_idx'), @@ -29,6 +38,18 @@ class EBook(models.Model): return f"EBook #{self.pk} (user={self.user_id})" +class EBookFile(models.Model): + """A book's file, encrypted like everything else: base64 AES-GCM of the EPUB/PDF bytes. + In a table of its own (docs/database.md): with the file in the book's row, every change of a + title, a folder or "gelesen" rewrote megabytes, and the list read past every file.""" + book = models.OneToOneField(EBook, on_delete=models.CASCADE, primary_key=True, related_name='file') + data_ct = models.TextField() + data_iv = models.CharField(max_length=32) + + def __str__(self): + return f"EBookFile of #{self.book_id}" + + def book_rows(user): """Every book of the user as {id, meta_ct, meta_iv, uploaded_at (ISO), is_read, has_cover}, oldest first — read without passing a book's file: meta_ct and meta_iv stand before it in diff --git a/books/tests.py b/books/tests.py index 0be2825..695e9f0 100644 --- a/books/tests.py +++ b/books/tests.py @@ -225,3 +225,37 @@ class BookRowsReadPastNoFileTests(TestCase): cur.execute('EXPLAIN QUERY PLAN ' + sql.replace('%s', '?'), [user.id] if '?' in sql or '%s' in sql else []) plan = ' '.join(str(r) for r in cur.fetchall()) self.assertIn('COVERING INDEX', plan, sql) + + +class EBookFileTests(TestCase): + """The file lives in EBookFile; upload, data, replace-data and rekey go there.""" + + def setUp(self): + from django.contrib.auth.models import User + self.user = User.objects.create_user('filer', password='pw-filer-123') + self.client.force_login(self.user) + + def test_upload_read_replace_rekey_delete(self): + import base64, json + from books.models import EBook, EBookFile + data = base64.b64encode(b'x' * 100).decode() + r = self.client.post('/books/upload/', json.dumps({'meta_ct': 'm', 'meta_iv': 'i', 'data_ct': data, 'data_iv': 'd'}), + content_type='application/json') + pk = r.json()['id'] + self.assertEqual(EBookFile.objects.get(book_id=pk).data_ct, data) + self.assertEqual(self.client.get(f'/books/{pk}/data/').json(), {'data_ct': data, 'data_iv': 'd'}) + self.client.post(f'/books/{pk}/replace-data/', json.dumps({'data_ct': 'bmV3', 'data_iv': 'd2'}), content_type='application/json') + self.assertEqual(self.client.get(f'/books/{pk}/data/').json()['data_ct'], 'bmV3') + self.client.post(f'/books/{pk}/rekey/', json.dumps({'meta_ct': 'm3', 'meta_iv': 'i3', 'data_ct': 'cmVr', 'data_iv': 'd3'}), + content_type='application/json') + self.assertEqual(EBook.objects.get(pk=pk).meta_ct, 'm3') + self.assertEqual(EBookFile.objects.get(book_id=pk).data_iv, 'd3') + self.client.post(f'/books/{pk}/delete/') + self.assertFalse(EBookFile.objects.filter(book_id=pk).exists()) + + def test_someone_elses_file_is_not_found(self): + from django.contrib.auth.models import User + from books.models import EBook + other = User.objects.create_user('other-filer', password='pw-other-123') + b = EBook.objects.create(user=other, meta_ct='m', meta_iv='i', data_ct='ZA==', data_iv='i') + self.assertEqual(self.client.get(f'/books/{b.id}/data/').status_code, 404) diff --git a/books/views.py b/books/views.py index 14a4c47..3c68a17 100644 --- a/books/views.py +++ b/books/views.py @@ -12,7 +12,7 @@ from django.http import JsonResponse from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods -from .models import EBook, EBookProgress, EBookHighlights, EBookBookmarks, LibraryPrefs, book_rows +from .models import EBook, EBookFile, EBookProgress, EBookHighlights, EBookBookmarks, LibraryPrefs, book_rows def _require_auth(request): @@ -443,14 +443,15 @@ def upload_book(request): if raw_size > max_bytes: return JsonResponse({'error': f'file too large (max {settings.EBOOK_MAX_BYTES // (1024 * 1024)} MB)'}, status=400) - book = EBook.objects.create( - user=request.user, - meta_ct=meta_ct, - meta_iv=meta_iv, - data_ct=data_ct, - data_iv=data_iv, - ) - EBookProgress.objects.create(user=request.user, book=book) + with transaction.atomic(): + book = EBook.objects.create( + user=request.user, + meta_ct=meta_ct, + meta_iv=meta_iv, + data_ct=data_ct, + data_iv=data_iv, + ) + EBookProgress.objects.create(user=request.user, book=book) return JsonResponse({'ok': True, 'id': book.id}) @@ -460,12 +461,10 @@ def get_book_data(request, pk): if err: return err - try: - book = EBook.objects.get(pk=pk, user=request.user) - except EBook.DoesNotExist: + f = EBookFile.objects.filter(book_id=pk, book__user=request.user).first() + if f is None: return JsonResponse({'error': 'not found'}, status=404) - - return JsonResponse({'data_ct': book.data_ct, 'data_iv': book.data_iv}) + return JsonResponse({'data_ct': f.data_ct, 'data_iv': f.data_iv}) @csrf_exempt @@ -493,14 +492,12 @@ def replace_book_data(request, pk): if not all([data_ct, data_iv]): return JsonResponse({'error': 'data_ct and data_iv required'}, status=400) - update_fields = ['data_ct', 'data_iv'] - book.data_ct = data_ct - book.data_iv = data_iv - if meta_ct and meta_iv: - book.meta_ct = meta_ct - book.meta_iv = meta_iv - update_fields += ['meta_ct', 'meta_iv'] - book.save(update_fields=update_fields) + with transaction.atomic(): + EBookFile.objects.update_or_create(book=book, defaults={'data_ct': data_ct, 'data_iv': data_iv}) + if meta_ct and meta_iv: + book.meta_ct = meta_ct + book.meta_iv = meta_iv + book.save(update_fields=['meta_ct', 'meta_iv']) return JsonResponse({'ok': True}) @@ -529,11 +526,11 @@ def rekey_book(request, pk): if not all([meta_ct, meta_iv, data_ct, data_iv]): return JsonResponse({'error': 'meta_ct, meta_iv, data_ct, data_iv required'}, status=400) - book.meta_ct = meta_ct - book.meta_iv = meta_iv - book.data_ct = data_ct - book.data_iv = data_iv - book.save(update_fields=['meta_ct', 'meta_iv', 'data_ct', 'data_iv']) + with transaction.atomic(): + EBookFile.objects.update_or_create(book=book, defaults={'data_ct': data_ct, 'data_iv': data_iv}) + book.meta_ct = meta_ct + book.meta_iv = meta_iv + book.save(update_fields=['meta_ct', 'meta_iv']) return JsonResponse({'ok': True}) diff --git a/docs/database.md b/docs/database.md index 6b02b09..3484c04 100644 --- a/docs/database.md +++ b/docs/database.md @@ -32,5 +32,16 @@ column *after* it, it walks the whole chain: listing 427 books read gigabytes an reads `id, meta_ct, meta_iv` from the rows (they stand before the file) and `uploaded_at, is_read` / whether there is a cover from the covering indexes `ebook_list_idx` and `ebook_cover_idx` — no list query touches a file. `books/tests.py` checks the query plans. -Writes still rewrite the whole row (file included); moving the files into a table of their own -would end that too. + +## The files in a table of their own (since 2026-10-19) + +`books.models.EBookFile` (one row per book, `book_id` its key) holds `data_ct`/`data_iv`; +`EBook` has only the small fields. Before, every change of a title, a folder or "gelesen" +rewrote the whole row with its file (megabytes, for a scanned volume tens of them), and every +`EBook.objects.get()` loaded the file. `EBook.objects.create(data_ct=…, data_iv=…)` still +works (the manager makes the EBookFile too); the API is unchanged. + +Migration `books/0008_ebook_file` copied 430 files in one statement and dropped the two +columns: about 2 minutes on the server (rehearsed on a copy), the site down meanwhile. The +pages the old columns used are free now and get reused; `VACUUM` (exclusive, minutes, at a +quiet time) would give them back to the disk.