diff --git a/accounts/migrations/0006_webdav_inbox.py b/accounts/migrations/0006_webdav_inbox.py new file mode 100644 index 0000000..0588bad --- /dev/null +++ b/accounts/migrations/0006_webdav_inbox.py @@ -0,0 +1,33 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0005_webdavsource'), + ] + + operations = [ + migrations.AddField( + model_name='webdavsource', + name='inbox', + field=models.BooleanField(default=False), + ), + migrations.CreateModel( + name='WebDAVSeen', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('path', models.CharField(max_length=1000)), + ('size', models.BigIntegerField(default=0)), + ('modified', models.CharField(blank=True, max_length=100)), + ('claimed_at', models.DateTimeField(blank=True, null=True)), + ('done_at', models.DateTimeField(blank=True, null=True)), + ('result', models.CharField(blank=True, max_length=20)), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='seen', to='accounts.webdavsource')), + ], + options={ + 'unique_together': {('source', 'path')}, + }, + ), + ] diff --git a/accounts/models.py b/accounts/models.py index 38e8229..959ae86 100644 --- a/accounts/models.py +++ b/accounts/models.py @@ -56,6 +56,9 @@ class WebDAVSource(models.Model): username = models.CharField(max_length=200, blank=True) password = models.CharField(max_length=500, blank=True) root_path = models.CharField(max_length=500, blank=True, default='') + # "Eingang" (docs/library.md): new books in it come into the library by themselves — + # a device with the key fetches, encrypts and uploads them (WebDAVSeen keeps which). + inbox = models.BooleanField(default=False) created_at = models.DateTimeField(auto_now_add=True) last_used_at = models.DateTimeField(null=True, blank=True) @@ -89,6 +92,22 @@ class WebDAVSource(models.Model): return url +class WebDAVSeen(models.Model): + """A file of an "Eingang" source a device took (or is taking) into the library: taken + files aren't offered again; a claim keeps two devices from taking the same file at once + and runs out when a device didn't finish.""" + source = models.ForeignKey(WebDAVSource, on_delete=models.CASCADE, related_name='seen') + path = models.CharField(max_length=1000) + size = models.BigIntegerField(default=0) + modified = models.CharField(max_length=100, blank=True) + claimed_at = models.DateTimeField(null=True, blank=True) + done_at = models.DateTimeField(null=True, blank=True) + result = models.CharField(max_length=20, blank=True) # imported | known | too_big | failed + + class Meta: + unique_together = ('source', 'path') + + def _generate_token(): return secrets.token_hex(32) diff --git a/accounts/tests.py b/accounts/tests.py index 6a068b8..edbd610 100644 --- a/accounts/tests.py +++ b/accounts/tests.py @@ -594,3 +594,65 @@ class WebDAVEditTests(TestCase): html = self.client.get('/accounts/settings/').content.decode() self.assertIn(f'/accounts/webdav/{self.source.pk}/edit/', html) self.assertNotIn('app-pw', html) + + +class CloudInboxTests(TestCase): + """An "Eingang" source (docs/library.md): its new books are handed out once, claimed for + one device for ten minutes, and not again once a device took them.""" + + def setUp(self): + self.user = User.objects.create_user('inbox', password='pw12345678') + self.client.force_login(self.user) + self.src = WebDAVSource.objects.create(user=self.user, label='Bücher', base_url='https://b.example.org/dav/', + username='d', password='x', inbox=True) + WebDAVSource.objects.create(user=self.user, label='andere', base_url='https://c.example.org/dav/') + + def listing(self, path=''): + if path == '': + return [{'name': 'Sub', 'path': 'Sub', 'is_dir': True, 'size': 0, 'modified': '', 'is_book': False}, + {'name': 'A - Eins (2020).epub', 'path': 'A - Eins (2020).epub', 'is_dir': False, + 'size': 100, 'modified': 'Mon', 'is_book': True}, + {'name': 'notiz.txt', 'path': 'notiz.txt', 'is_dir': False, 'size': 1, 'modified': '', 'is_book': False}] + return [{'name': 'B.pdf', 'path': 'Sub/B.pdf', 'is_dir': False, 'size': 200, 'modified': '', 'is_book': True}] + + def inbox(self): + with patch('books.webdav.list_directory', side_effect=lambda source, path='': self.listing(path)): + return self.client.get('/books/cloud/inbox/').json() + + def done(self, path, result): + return self.client.post(f'/books/cloud/{self.src.pk}/done/', json.dumps({'path': path, 'result': result}), + content_type='application/json') + + def test_new_books_once_then_claimed_then_done(self): + files = self.inbox()['files'] + self.assertEqual(sorted(f['path'] for f in files), ['A - Eins (2020).epub', 'Sub/B.pdf']) + self.assertEqual(self.inbox()['files'], []) # claimed for this device: no one else gets them now + self.assertEqual(self.done('A - Eins (2020).epub', 'imported').status_code, 200) + self.assertEqual(self.done('Sub/B.pdf', 'failed').status_code, 200) + from accounts.models import WebDAVSeen + WebDAVSeen.objects.filter(path='Sub/B.pdf').update(claimed_at=None) # the claim ran out + self.assertEqual([f['path'] for f in self.inbox()['files']], ['Sub/B.pdf']) + + def test_a_changed_file_counts_as_new(self): + self.inbox() + self.done('A - Eins (2020).epub', 'imported') + self.done('Sub/B.pdf', 'known') + self.assertEqual(self.inbox()['files'], []) + old = self.listing + self.listing = lambda path='': [dict(e, size=999) if e['path'].endswith('.epub') else e for e in old(path)] + self.assertEqual([f['path'] for f in self.inbox()['files']], ['A - Eins (2020).epub']) + + def test_only_own_sources_and_valid_results(self): + other = User.objects.create_user('x', password='pw12345678') + foreign = WebDAVSource.objects.create(user=other, label='f', base_url='https://d.example.org/dav/', inbox=True) + self.assertEqual(self.client.post(f'/books/cloud/{foreign.pk}/done/', '{}', content_type='application/json').status_code, 404) + self.inbox() + self.assertEqual(self.done('A - Eins (2020).epub', 'whatever').status_code, 400) + self.client.logout() + self.assertEqual(self.client.get('/books/cloud/inbox/').status_code, 401) + + def test_settings_switch_the_inbox(self): + r = self.client.post(f'/accounts/webdav/{self.src.pk}/edit/', {'label': 'Bücher', 'base_url': self.src.base_url}) + self.assertEqual(r.status_code, 302) + self.src.refresh_from_db() + self.assertFalse(self.src.inbox) diff --git a/accounts/views.py b/accounts/views.py index 78ee9d7..53a3b67 100644 --- a/accounts/views.py +++ b/accounts/views.py @@ -283,6 +283,7 @@ def webdav_add(request): username=request.POST.get('username', '').strip()[:200], password=request.POST.get('password', '')[:500], root_path=request.POST.get('root_path', '').strip()[:500], + inbox=request.POST.get('inbox') == '1', ) level, message = _probe_source(source) @@ -308,6 +309,7 @@ def webdav_edit(request, pk): source.base_url = base_url[:500] source.username = request.POST.get('username', '').strip()[:200] source.root_path = request.POST.get('root_path', '').strip()[:500] + source.inbox = request.POST.get('inbox') == '1' # The stored password is never sent to the page: empty means keep it. password = request.POST.get('password', '') if password: diff --git a/books/urls.py b/books/urls.py index 557354a..75804a8 100644 --- a/books/urls.py +++ b/books/urls.py @@ -5,6 +5,8 @@ urlpatterns = [ path('', views.book_list, name='book_list'), path('upload/', views.upload_book, name='upload_book'), path('cloud/sources/', webdav.cloud_sources, name='cloud_sources'), + path('cloud/inbox/', webdav.cloud_inbox, name='cloud_inbox'), + path('cloud//done/', webdav.cloud_done, name='cloud_done'), path('cloud//browse/', webdav.cloud_browse, name='cloud_browse'), path('cloud//fetch/', webdav.cloud_fetch, name='cloud_fetch'), path('metadata-lookup/', views.lookup_book_metadata, name='lookup_book_metadata'), diff --git a/books/webdav.py b/books/webdav.py index 11ed64f..d43b72e 100644 --- a/books/webdav.py +++ b/books/webdav.py @@ -12,12 +12,16 @@ browser and only then POSTed to /books/upload/ as ciphertext, exactly like a local upload — nothing readable is ever stored server-side. """ +import json +from datetime import timedelta + from django.conf import settings from django.http import HttpResponse, JsonResponse from django.utils import timezone +from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods -from accounts.models import WebDAVSource +from accounts.models import WebDAVSeen, WebDAVSource from accounts.webdav import WebDAVError, WebDAVInputError, list_directory, fetch_file from .views import _require_auth @@ -34,11 +38,94 @@ def cloud_sources(request): sources = request.user.webdav_sources.all() return JsonResponse({'sources': [ - {'id': s.id, 'label': s.label, 'host': s.base_url} + {'id': s.id, 'label': s.label, 'host': s.base_url, 'inbox': s.inbox} for s in sources ]}) +# ---- "Eingang" (docs/library.md): new books of a source into the library by themselves ---- + +INBOX_DEPTH = 3 # sub-folders looked into +INBOX_MAX = 20 # files handed out per call +INBOX_CLAIM = timedelta(minutes=10) + + +def _inbox_files(source): + """The source's book files, its sub-folders too (INBOX_DEPTH deep).""" + out, todo = [], [('', 0)] + while todo: + path, depth = todo.pop(0) + for e in list_directory(source, path): + if e['is_dir'] and depth < INBOX_DEPTH: + todo.append((e['path'], depth + 1)) + elif e['is_book']: + out.append(e) + return out + + +@require_http_methods(['GET']) +def cloud_inbox(request): + """The files of the user's "Eingang" sources not taken into the library yet, claimed for + this device for ten minutes (another device asking meanwhile doesn't get them): + {files: [{source, label, path, name, size, modified}], errors: [{source, label, error}]}. + A file that changed (other size) counts as new again.""" + unauthorized = _require_auth(request) + if unauthorized: + return unauthorized + now = timezone.now() + files, errors = [], [] + for source in request.user.webdav_sources.filter(inbox=True): + try: + entries = _inbox_files(source) + except (WebDAVError, WebDAVInputError) as exc: + errors.append({'source': source.id, 'label': source.label, 'error': str(exc)}) + continue + seen = {s.path: s for s in source.seen.all()} + for e in entries: + if len(files) >= INBOX_MAX: + break + s = seen.get(e['path']) + if s and s.size == e['size']: + if s.done_at or (s.claimed_at and now - s.claimed_at < INBOX_CLAIM): + continue + WebDAVSeen.objects.update_or_create( + source=source, path=e['path'][:1000], + defaults={'size': e['size'], 'modified': e['modified'][:100], 'claimed_at': now, + 'done_at': None, 'result': ''}) + files.append({'source': source.id, 'label': source.label, 'path': e['path'], + 'name': e['name'].rsplit('/', 1)[-1], 'size': e['size'], 'modified': e['modified']}) + source.last_used_at = now + source.save(update_fields=['last_used_at']) + return JsonResponse({'files': files, 'errors': errors}) + + +@csrf_exempt +@require_http_methods(['POST']) +def cloud_done(request, pk): + """{path, result}: a device finished with an "Eingang" file — imported, known (already in the + library), too_big: not offered again; failed: offered again after the claim ran out.""" + unauthorized = _require_auth(request) + if unauthorized: + return unauthorized + source = _get_source(request, pk) + if not source: + return JsonResponse({'error': 'unknown source'}, status=404) + try: + body = json.loads(request.body or b'{}') + except (json.JSONDecodeError, ValueError): + return JsonResponse({'error': 'invalid JSON'}, status=400) + result = body.get('result') if isinstance(body, dict) else None + if result not in ('imported', 'known', 'too_big', 'failed'): + return JsonResponse({'error': 'result: imported, known, too_big or failed'}, status=400) + seen = WebDAVSeen.objects.filter(source=source, path=str(body.get('path', ''))[:1000]).first() + if not seen: + return JsonResponse({'error': 'unknown file'}, status=404) + seen.result = result + seen.done_at = None if result == 'failed' else timezone.now() + seen.save(update_fields=['result', 'done_at']) + return JsonResponse({'ok': True}) + + @require_http_methods(['GET']) def cloud_browse(request, pk): unauthorized = _require_auth(request) diff --git a/static/js/app/57-cloud-inbox.js b/static/js/app/57-cloud-inbox.js new file mode 100644 index 0000000..b3784c4 --- /dev/null +++ b/static/js/app/57-cloud-inbox.js @@ -0,0 +1,79 @@ +'use strict'; + +// --------------------------------------------------------------------------- +// "Eingang" (docs/library.md): a cloud source marked so brings its new books into the +// library by itself. When the library loads, the server says which files of such sources +// no device has taken yet (and keeps them for this device ten minutes); each is fetched +// through the server, encrypted here like any import, uploaded, and reported done. A book +// already in the library (same title and author) is left out. The folder stays as it is. +// Only where the book key is there; not in diora Reader (e-readers: battery, weak Wi-Fi). +// --------------------------------------------------------------------------- + +const CLOUD_INBOX_EVERY_MS = 5 * 60 * 1000; +let _cloudInboxBusy = false; +let _cloudInboxLast = 0; + +function _cloudInboxKeyReady() { + // getOrCreateEncKey() would make a new key when there is none: never import without the real one. + try { return !!localStorage.getItem(`diora_enc_key_${window.USER_ID || 0}`); } catch (e) { return false; } +} + +async function cloudInboxCheck(force) { + if (typeof IS_AUTHENTICATED === 'undefined' || !IS_AUTHENTICATED) return; + if (document.documentElement.classList.contains('nui-reader-app')) return; + if (_cloudInboxBusy || !_cloudInboxKeyReady()) return; + if (!force && Date.now() - _cloudInboxLast < CLOUD_INBOX_EVERY_MS) return; + _cloudInboxBusy = true; + _cloudInboxLast = Date.now(); + const statusEl = $('book-upload-status'); + const taken = []; + try { + const r = await fetch('/books/cloud/inbox/', {cache: 'no-store'}); + if (!r.ok) return; + const files = (await r.json()).files || []; + for (let i = 0; i < files.length; i++) { + const f = files[i]; + if (statusEl) statusEl.textContent = `Eingang „${f.label}“: ${i + 1} von ${files.length} – ${f.name} …`; + let result = 'failed'; + try { + if (f.size > DIORA_CONFIG.ebookMaxBytes) { + result = 'too_big'; + } else { + const res = await fetch(`/books/cloud/${f.source}/fetch/?path=${encodeURIComponent(f.path)}`); + if (res.ok) { + const out = await _importEbookBuffer(await res.arrayBuffer(), f.name, null, {reload: false, skipKnown: true}); + result = out === 'dup' ? 'known' : out ? 'imported' : 'failed'; + } + } + } catch (e) { + result = 'failed'; + } + if (result === 'imported') taken.push(f); + await fetch(`/books/cloud/${f.source}/done/`, { + method: 'POST', headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({path: f.path, result}), + }).catch(() => {}); + } + } catch (e) { + // Offline or the source unreachable: next time. + } finally { + _cloudInboxBusy = false; + } + if (statusEl) { + const labels = [...new Set(taken.map(f => f.label))].map(l => `„${l}“`).join(', '); + statusEl.textContent = taken.length + ? `${taken.length === 1 ? 'Ein neues Buch' : taken.length + ' neue Bücher'} aus ${labels} übernommen.` : ''; + } + if (taken.length) loadBookList(); +} + +// After the list has loaded (and the key is known), quietly look into the inboxes. +(function () { + const orig = window.loadBookList; + if (typeof orig !== 'function') return; + window.loadBookList = async function () { + const out = await orig.apply(this, arguments); + setTimeout(() => cloudInboxCheck(false), 1500); + return out; + }; +})(); diff --git a/static/js/sw.js b/static/js/sw.js index de43dc5..de1106b 100644 --- a/static/js/sw.js +++ b/static/js/sw.js @@ -2,7 +2,7 @@ * diora service worker — caches the app shell for offline use. */ -const CACHE = 'diora-v146'; +const CACHE = 'diora-v147'; const SHELL = [ '/static/css/app.css', '/static/js/app/01-core.js', @@ -26,6 +26,7 @@ const SHELL = [ '/static/js/app/53-library.js', '/static/js/app/54-issues.js', '/static/js/app/56-cloud-import.js', + '/static/js/app/57-cloud-inbox.js', '/static/js/app/58-pdf.js', '/static/js/app/60-tts.js', '/static/js/app/62-reader.js', diff --git a/templates/accounts/settings.html b/templates/accounts/settings.html index c2d17f7..4ec824f 100644 --- a/templates/accounts/settings.html +++ b/templates/accounts/settings.html @@ -146,7 +146,7 @@ {% for source in webdav_sources %}
  • - {{ source.label }} + {{ source.label }}{% if source.inbox %} · Eingang{% endif %} {{ source.base_url }}{% if source.root_path %} · /{{ source.root_path }}{% endif %}
    @@ -185,6 +185,11 @@ Unterordner (optional, Import startet dort) +
    • @@ -214,6 +219,10 @@ Unterordner (optional) +

    Bei Nextcloud/ownCloud genügt die Server-Adresse — der WebDAV-Pfad wird aus dem Benutzernamen ergänzt. Andere Server brauchen die vollständige WebDAV-URL. diff --git a/templates/radio/player.html b/templates/radio/player.html index ef9b523..6aed593 100644 --- a/templates/radio/player.html +++ b/templates/radio/player.html @@ -381,6 +381,7 @@ +