Bücher: „Eingang“ – eine Cloud-Quelle bringt ihre neuen Bücher von selbst in die Bibliothek (Gerät holt, verschlüsselt, lädt hoch; Server merkt sich, was übernommen ist); sw v147

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

accounts/migrations/0006_webdav_inbox.py

@@ -0,0 +1,33 @@
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0005_webdavsource'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='webdavsource',
+            name='inbox',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.CreateModel(
+            name='WebDAVSeen',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('path', models.CharField(max_length=1000)),
+                ('size', models.BigIntegerField(default=0)),
+                ('modified', models.CharField(blank=True, max_length=100)),
+                ('claimed_at', models.DateTimeField(blank=True, null=True)),
+                ('done_at', models.DateTimeField(blank=True, null=True)),
+                ('result', models.CharField(blank=True, max_length=20)),
+                ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='seen', to='accounts.webdavsource')),
+            ],
+            options={
+                'unique_together': {('source', 'path')},
+            },
+        ),
+    ]

accounts/models.py

@@ -56,6 +56,9 @@ class WebDAVSource(models.Model):
     username = models.CharField(max_length=200, blank=True)
     password = models.CharField(max_length=500, blank=True)
     root_path = models.CharField(max_length=500, blank=True, default='')
+    # "Eingang" (docs/library.md): new books in it come into the library by themselves —
+    # a device with the key fetches, encrypts and uploads them (WebDAVSeen keeps which).
+    inbox = models.BooleanField(default=False)
     created_at = models.DateTimeField(auto_now_add=True)
     last_used_at = models.DateTimeField(null=True, blank=True)
 
@@ -89,6 +92,22 @@ class WebDAVSource(models.Model):
         return url
 
 
+class WebDAVSeen(models.Model):
+    """A file of an "Eingang" source a device took (or is taking) into the library: taken
+    files aren't offered again; a claim keeps two devices from taking the same file at once
+    and runs out when a device didn't finish."""
+    source = models.ForeignKey(WebDAVSource, on_delete=models.CASCADE, related_name='seen')
+    path = models.CharField(max_length=1000)
+    size = models.BigIntegerField(default=0)
+    modified = models.CharField(max_length=100, blank=True)
+    claimed_at = models.DateTimeField(null=True, blank=True)
+    done_at = models.DateTimeField(null=True, blank=True)
+    result = models.CharField(max_length=20, blank=True)   # imported | known | too_big | failed
+
+    class Meta:
+        unique_together = ('source', 'path')
+
+
 def _generate_token():
     return secrets.token_hex(32)
 

accounts/tests.py

@@ -594,3 +594,65 @@ class WebDAVEditTests(TestCase):
         html = self.client.get('/accounts/settings/').content.decode()
         self.assertIn(f'/accounts/webdav/{self.source.pk}/edit/', html)
         self.assertNotIn('app-pw', html)
+
+
+class CloudInboxTests(TestCase):
+    """An "Eingang" source (docs/library.md): its new books are handed out once, claimed for
+    one device for ten minutes, and not again once a device took them."""
+
+    def setUp(self):
+        self.user = User.objects.create_user('inbox', password='pw12345678')
+        self.client.force_login(self.user)
+        self.src = WebDAVSource.objects.create(user=self.user, label='Bücher', base_url='https://b.example.org/dav/',
+                                               username='d', password='x', inbox=True)
+        WebDAVSource.objects.create(user=self.user, label='andere', base_url='https://c.example.org/dav/')
+
+    def listing(self, path=''):
+        if path == '':
+            return [{'name': 'Sub', 'path': 'Sub', 'is_dir': True, 'size': 0, 'modified': '', 'is_book': False},
+                    {'name': 'A - Eins (2020).epub', 'path': 'A - Eins (2020).epub', 'is_dir': False,
+                     'size': 100, 'modified': 'Mon', 'is_book': True},
+                    {'name': 'notiz.txt', 'path': 'notiz.txt', 'is_dir': False, 'size': 1, 'modified': '', 'is_book': False}]
+        return [{'name': 'B.pdf', 'path': 'Sub/B.pdf', 'is_dir': False, 'size': 200, 'modified': '', 'is_book': True}]
+
+    def inbox(self):
+        with patch('books.webdav.list_directory', side_effect=lambda source, path='': self.listing(path)):
+            return self.client.get('/books/cloud/inbox/').json()
+
+    def done(self, path, result):
+        return self.client.post(f'/books/cloud/{self.src.pk}/done/', json.dumps({'path': path, 'result': result}),
+                                content_type='application/json')
+
+    def test_new_books_once_then_claimed_then_done(self):
+        files = self.inbox()['files']
+        self.assertEqual(sorted(f['path'] for f in files), ['A - Eins (2020).epub', 'Sub/B.pdf'])
+        self.assertEqual(self.inbox()['files'], [])   # claimed for this device: no one else gets them now
+        self.assertEqual(self.done('A - Eins (2020).epub', 'imported').status_code, 200)
+        self.assertEqual(self.done('Sub/B.pdf', 'failed').status_code, 200)
+        from accounts.models import WebDAVSeen
+        WebDAVSeen.objects.filter(path='Sub/B.pdf').update(claimed_at=None)   # the claim ran out
+        self.assertEqual([f['path'] for f in self.inbox()['files']], ['Sub/B.pdf'])
+
+    def test_a_changed_file_counts_as_new(self):
+        self.inbox()
+        self.done('A - Eins (2020).epub', 'imported')
+        self.done('Sub/B.pdf', 'known')
+        self.assertEqual(self.inbox()['files'], [])
+        old = self.listing
+        self.listing = lambda path='': [dict(e, size=999) if e['path'].endswith('.epub') else e for e in old(path)]
+        self.assertEqual([f['path'] for f in self.inbox()['files']], ['A - Eins (2020).epub'])
+
+    def test_only_own_sources_and_valid_results(self):
+        other = User.objects.create_user('x', password='pw12345678')
+        foreign = WebDAVSource.objects.create(user=other, label='f', base_url='https://d.example.org/dav/', inbox=True)
+        self.assertEqual(self.client.post(f'/books/cloud/{foreign.pk}/done/', '{}', content_type='application/json').status_code, 404)
+        self.inbox()
+        self.assertEqual(self.done('A - Eins (2020).epub', 'whatever').status_code, 400)
+        self.client.logout()
+        self.assertEqual(self.client.get('/books/cloud/inbox/').status_code, 401)
+
+    def test_settings_switch_the_inbox(self):
+        r = self.client.post(f'/accounts/webdav/{self.src.pk}/edit/', {'label': 'Bücher', 'base_url': self.src.base_url})
+        self.assertEqual(r.status_code, 302)
+        self.src.refresh_from_db()
+        self.assertFalse(self.src.inbox)

accounts/views.py

@@ -283,6 +283,7 @@ def webdav_add(request):
         username=request.POST.get('username', '').strip()[:200],
         password=request.POST.get('password', '')[:500],
         root_path=request.POST.get('root_path', '').strip()[:500],
+        inbox=request.POST.get('inbox') == '1',
     )
 
     level, message = _probe_source(source)
@@ -308,6 +309,7 @@ def webdav_edit(request, pk):
     source.base_url = base_url[:500]
     source.username = request.POST.get('username', '').strip()[:200]
     source.root_path = request.POST.get('root_path', '').strip()[:500]
+    source.inbox = request.POST.get('inbox') == '1'
     # The stored password is never sent to the page: empty means keep it.
     password = request.POST.get('password', '')
     if password:

books/urls.py

@@ -5,6 +5,8 @@ urlpatterns = [
     path('', views.book_list, name='book_list'),
     path('upload/', views.upload_book, name='upload_book'),
     path('cloud/sources/', webdav.cloud_sources, name='cloud_sources'),
+    path('cloud/inbox/', webdav.cloud_inbox, name='cloud_inbox'),
+    path('cloud/<int:pk>/done/', webdav.cloud_done, name='cloud_done'),
     path('cloud/<int:pk>/browse/', webdav.cloud_browse, name='cloud_browse'),
     path('cloud/<int:pk>/fetch/', webdav.cloud_fetch, name='cloud_fetch'),
     path('metadata-lookup/', views.lookup_book_metadata, name='lookup_book_metadata'),

books/webdav.py

@@ -12,12 +12,16 @@ browser and only then POSTed to /books/upload/ as ciphertext, exactly like a
 local upload — nothing readable is ever stored server-side.
 """
 
+import json
+from datetime import timedelta
+
 from django.conf import settings
 from django.http import HttpResponse, JsonResponse
 from django.utils import timezone
+from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_http_methods
 
-from accounts.models import WebDAVSource
+from accounts.models import WebDAVSeen, WebDAVSource
 from accounts.webdav import WebDAVError, WebDAVInputError, list_directory, fetch_file
 from .views import _require_auth
 
@@ -34,11 +38,94 @@ def cloud_sources(request):
 
     sources = request.user.webdav_sources.all()
     return JsonResponse({'sources': [
-        {'id': s.id, 'label': s.label, 'host': s.base_url}
+        {'id': s.id, 'label': s.label, 'host': s.base_url, 'inbox': s.inbox}
         for s in sources
     ]})
 
 
+# ---- "Eingang" (docs/library.md): new books of a source into the library by themselves ----
+
+INBOX_DEPTH = 3          # sub-folders looked into
+INBOX_MAX = 20           # files handed out per call
+INBOX_CLAIM = timedelta(minutes=10)
+
+
+def _inbox_files(source):
+    """The source's book files, its sub-folders too (INBOX_DEPTH deep)."""
+    out, todo = [], [('', 0)]
+    while todo:
+        path, depth = todo.pop(0)
+        for e in list_directory(source, path):
+            if e['is_dir'] and depth < INBOX_DEPTH:
+                todo.append((e['path'], depth + 1))
+            elif e['is_book']:
+                out.append(e)
+    return out
+
+
+@require_http_methods(['GET'])
+def cloud_inbox(request):
+    """The files of the user's "Eingang" sources not taken into the library yet, claimed for
+    this device for ten minutes (another device asking meanwhile doesn't get them):
+    {files: [{source, label, path, name, size, modified}], errors: [{source, label, error}]}.
+    A file that changed (other size) counts as new again."""
+    unauthorized = _require_auth(request)
+    if unauthorized:
+        return unauthorized
+    now = timezone.now()
+    files, errors = [], []
+    for source in request.user.webdav_sources.filter(inbox=True):
+        try:
+            entries = _inbox_files(source)
+        except (WebDAVError, WebDAVInputError) as exc:
+            errors.append({'source': source.id, 'label': source.label, 'error': str(exc)})
+            continue
+        seen = {s.path: s for s in source.seen.all()}
+        for e in entries:
+            if len(files) >= INBOX_MAX:
+                break
+            s = seen.get(e['path'])
+            if s and s.size == e['size']:
+                if s.done_at or (s.claimed_at and now - s.claimed_at < INBOX_CLAIM):
+                    continue
+            WebDAVSeen.objects.update_or_create(
+                source=source, path=e['path'][:1000],
+                defaults={'size': e['size'], 'modified': e['modified'][:100], 'claimed_at': now,
+                          'done_at': None, 'result': ''})
+            files.append({'source': source.id, 'label': source.label, 'path': e['path'],
+                          'name': e['name'].rsplit('/', 1)[-1], 'size': e['size'], 'modified': e['modified']})
+        source.last_used_at = now
+        source.save(update_fields=['last_used_at'])
+    return JsonResponse({'files': files, 'errors': errors})
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def cloud_done(request, pk):
+    """{path, result}: a device finished with an "Eingang" file — imported, known (already in the
+    library), too_big: not offered again; failed: offered again after the claim ran out."""
+    unauthorized = _require_auth(request)
+    if unauthorized:
+        return unauthorized
+    source = _get_source(request, pk)
+    if not source:
+        return JsonResponse({'error': 'unknown source'}, status=404)
+    try:
+        body = json.loads(request.body or b'{}')
+    except (json.JSONDecodeError, ValueError):
+        return JsonResponse({'error': 'invalid JSON'}, status=400)
+    result = body.get('result') if isinstance(body, dict) else None
+    if result not in ('imported', 'known', 'too_big', 'failed'):
+        return JsonResponse({'error': 'result: imported, known, too_big or failed'}, status=400)
+    seen = WebDAVSeen.objects.filter(source=source, path=str(body.get('path', ''))[:1000]).first()
+    if not seen:
+        return JsonResponse({'error': 'unknown file'}, status=404)
+    seen.result = result
+    seen.done_at = None if result == 'failed' else timezone.now()
+    seen.save(update_fields=['result', 'done_at'])
+    return JsonResponse({'ok': True})
+
+
 @require_http_methods(['GET'])
 def cloud_browse(request, pk):
     unauthorized = _require_auth(request)

static/js/app/57-cloud-inbox.js

@@ -0,0 +1,79 @@
+'use strict';
+
+// ---------------------------------------------------------------------------
+// "Eingang" (docs/library.md): a cloud source marked so brings its new books into the
+// library by itself. When the library loads, the server says which files of such sources
+// no device has taken yet (and keeps them for this device ten minutes); each is fetched
+// through the server, encrypted here like any import, uploaded, and reported done. A book
+// already in the library (same title and author) is left out. The folder stays as it is.
+// Only where the book key is there; not in diora Reader (e-readers: battery, weak Wi-Fi).
+// ---------------------------------------------------------------------------
+
+const CLOUD_INBOX_EVERY_MS = 5 * 60 * 1000;
+let _cloudInboxBusy = false;
+let _cloudInboxLast = 0;
+
+function _cloudInboxKeyReady() {
+  // getOrCreateEncKey() would make a new key when there is none: never import without the real one.
+  try { return !!localStorage.getItem(`diora_enc_key_${window.USER_ID || 0}`); } catch (e) { return false; }
+}
+
+async function cloudInboxCheck(force) {
+  if (typeof IS_AUTHENTICATED === 'undefined' || !IS_AUTHENTICATED) return;
+  if (document.documentElement.classList.contains('nui-reader-app')) return;
+  if (_cloudInboxBusy || !_cloudInboxKeyReady()) return;
+  if (!force && Date.now() - _cloudInboxLast < CLOUD_INBOX_EVERY_MS) return;
+  _cloudInboxBusy = true;
+  _cloudInboxLast = Date.now();
+  const statusEl = $('book-upload-status');
+  const taken = [];
+  try {
+    const r = await fetch('/books/cloud/inbox/', {cache: 'no-store'});
+    if (!r.ok) return;
+    const files = (await r.json()).files || [];
+    for (let i = 0; i < files.length; i++) {
+      const f = files[i];
+      if (statusEl) statusEl.textContent = `Eingang „${f.label}“: ${i + 1} von ${files.length} – ${f.name} …`;
+      let result = 'failed';
+      try {
+        if (f.size > DIORA_CONFIG.ebookMaxBytes) {
+          result = 'too_big';
+        } else {
+          const res = await fetch(`/books/cloud/${f.source}/fetch/?path=${encodeURIComponent(f.path)}`);
+          if (res.ok) {
+            const out = await _importEbookBuffer(await res.arrayBuffer(), f.name, null, {reload: false, skipKnown: true});
+            result = out === 'dup' ? 'known' : out ? 'imported' : 'failed';
+          }
+        }
+      } catch (e) {
+        result = 'failed';
+      }
+      if (result === 'imported') taken.push(f);
+      await fetch(`/books/cloud/${f.source}/done/`, {
+        method: 'POST', headers: {'Content-Type': 'application/json'},
+        body: JSON.stringify({path: f.path, result}),
+      }).catch(() => {});
+    }
+  } catch (e) {
+    // Offline or the source unreachable: next time.
+  } finally {
+    _cloudInboxBusy = false;
+  }
+  if (statusEl) {
+    const labels = [...new Set(taken.map(f => f.label))].map(l => `„${l}“`).join(', ');
+    statusEl.textContent = taken.length
+      ? `${taken.length === 1 ? 'Ein neues Buch' : taken.length + ' neue Bücher'} aus ${labels} übernommen.` : '';
+  }
+  if (taken.length) loadBookList();
+}
+
+// After the list has loaded (and the key is known), quietly look into the inboxes.
+(function () {
+  const orig = window.loadBookList;
+  if (typeof orig !== 'function') return;
+  window.loadBookList = async function () {
+    const out = await orig.apply(this, arguments);
+    setTimeout(() => cloudInboxCheck(false), 1500);
+    return out;
+  };
+})();

static/js/sw.js

@@ -2,7 +2,7 @@
  * diora service worker — caches the app shell for offline use.
  */
 
-const CACHE = 'diora-v146';
+const CACHE = 'diora-v147';
 const SHELL = [
   '/static/css/app.css',
   '/static/js/app/01-core.js',
@@ -26,6 +26,7 @@ const SHELL = [
   '/static/js/app/53-library.js',
   '/static/js/app/54-issues.js',
   '/static/js/app/56-cloud-import.js',
+  '/static/js/app/57-cloud-inbox.js',
   '/static/js/app/58-pdf.js',
   '/static/js/app/60-tts.js',
   '/static/js/app/62-reader.js',

templates/accounts/settings.html

@@ -146,7 +146,7 @@
         {% for source in webdav_sources %}
           <li class="webdav-item">
             <div class="webdav-item-info">
-              <strong>{{ source.label }}</strong>
+              <strong>{{ source.label }}</strong>{% if source.inbox %} <span class="muted">· Eingang</span>{% endif %}
               <span class="muted">{{ source.base_url }}{% if source.root_path %} · /{{ source.root_path }}{% endif %}</span>
             </div>
             <div class="webdav-item-actions">
@@ -185,6 +185,11 @@
                 <span>Unterordner <span class="muted">(optional, Import startet dort)</span></span>
                 <input type="text" name="root_path" value="{{ source.root_path }}" placeholder="Buecher">
               </label>
+              <label class="webdav-field webdav-check">
+                <input type="checkbox" name="inbox" value="1"{% if source.inbox %} checked{% endif %}>
+                <span>Eingang: neue Bücher hier kommen von selbst in die Bibliothek
+                  <span class="muted">(beim Öffnen der Bücher; der Ordner bleibt unverändert)</span></span>
+              </label>
               <button type="submit" class="btn">Speichern</button>
             </form>
           </li>
@@ -214,6 +219,10 @@
         <span>Unterordner <span class="muted">(optional)</span></span>
         <input type="text" name="root_path" placeholder="Buecher">
       </label>
+      <label class="webdav-field webdav-check">
+        <input type="checkbox" name="inbox" value="1">
+        <span>Eingang: neue Bücher hier kommen von selbst in die Bibliothek</span>
+      </label>
       <p class="lastfm-description" style="margin:0;">
         Bei Nextcloud/ownCloud genügt die Server-Adresse — der WebDAV-Pfad wird aus dem
         Benutzernamen ergänzt. Andere Server brauchen die vollständige WebDAV-URL.

templates/radio/player.html

@@ -381,6 +381,7 @@
 <script src="/static/js/app/53-library.js"></script>
 <script src="/static/js/app/54-issues.js"></script>
 <script src="/static/js/app/56-cloud-import.js"></script>
+<script src="/static/js/app/57-cloud-inbox.js"></script>
 <script src="/static/js/app/58-pdf.js"></script>
 <script src="/static/js/app/60-tts.js"></script>
 <script src="/static/js/app/62-reader.js"></script>