diff --git a/diora/__init__.py b/diora/__init__.py index e69de29..102c0ca 100644 --- a/diora/__init__.py +++ b/diora/__init__.py @@ -0,0 +1,2 @@ +# SQLite in WAL mode for every connection (diora/sqlite.py). +from . import sqlite # noqa: E402,F401 diff --git a/diora/settings.py b/diora/settings.py index 88ce3af..0306dbe 100644 --- a/diora/settings.py +++ b/diora/settings.py @@ -77,7 +77,9 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': BASE_DIR / 'data' / os.environ.get('DIORA_DB_NAME', 'db.sqlite3'), - 'OPTIONS': {'timeout': 20}, + # WAL (diora/sqlite.py); a transaction takes the write lock when it begins, so one that + # reads first never fails on upgrading it (WAL's busy timeout can't help there). + 'OPTIONS': {'timeout': 20, 'transaction_mode': 'IMMEDIATE'}, } } diff --git a/diora/sqlite.py b/diora/sqlite.py new file mode 100644 index 0000000..873117d --- /dev/null +++ b/diora/sqlite.py @@ -0,0 +1,23 @@ +"""SQLite in write-ahead-log mode (docs/database.md): readers no longer wait for a writer. + +In the old rollback-journal mode a long write (the podcast refresh every 15 minutes, a book of +30 MB going in) kept every reader out until it was done, and requests failed with "database is +locked" — even GET /books/. WAL is set once per database file and stays; synchronous=NORMAL is +the safe pairing for it. A connection that can't switch (another one holds the file) just goes +on as it is; the next one switches. +""" +from django.db.backends.signals import connection_created + + +def _sqlite_wal(sender, connection, **kwargs): + if connection.vendor != 'sqlite': + return + with connection.cursor() as cursor: + try: + cursor.execute('PRAGMA journal_mode=WAL') + except Exception: # noqa: BLE001 - locked right now; another connection will do it + pass + cursor.execute('PRAGMA synchronous=NORMAL') + + +connection_created.connect(_sqlite_wal, dispatch_uid='diora_sqlite_wal') diff --git a/docs/database.md b/docs/database.md new file mode 100644 index 0000000..096f22f --- /dev/null +++ b/docs/database.md @@ -0,0 +1,24 @@ +# The database + +One SQLite file, `data/db.sqlite3` (on the server `/home/marwin/diora-prod-data`, mounted into +`diora-prod` at `/app/data`). The books live in it too, encrypted, so it is large (2.5 GB in +October 2026) and some writes are big (a scanned book of 30 MB). + +## Write-ahead log (since 2026-10-19) + +`diora/sqlite.py` puts every connection in WAL mode (`PRAGMA journal_mode=WAL`, set once per +file, it stays) with `synchronous=NORMAL`; `settings.DATABASES` has `transaction_mode: +IMMEDIATE`, so a transaction takes the write lock when it begins. + +Why: in the rollback-journal mode before, a long write kept all readers out. The podcast +refresh (cron, every 15 minutes) and big uploads made requests fail with "database is +locked", GET /books/ among them — the apps then saw no new books and no changed titles. +With WAL, readers read on while one writer writes; writers still take turns (20 s timeout). + +Next to `db.sqlite3` there are now `db.sqlite3-wal` and `db.sqlite3-shm`. They belong to the +database: a copy of `db.sqlite3` alone may miss the latest writes. To copy it safely: + + docker exec diora-prod python -c "import sqlite3; s=sqlite3.connect('data/db.sqlite3'); d=sqlite3.connect('data/backup.sqlite3'); s.backup(d)" + +or stop the container first. Back to the old mode, if ever needed: +`PRAGMA journal_mode=DELETE` (with no other connection open) and remove `diora/sqlite.py`'s hook.