diff --git a/diora/search.py b/diora/search.py new file mode 100644 index 0000000..2bdb5c7 --- /dev/null +++ b/diora/search.py @@ -0,0 +1,101 @@ +"""The search across diora (docs/search.md): one query, what the server knows of every area — +podcasts and audiobooks one follows, podcast episodes (the full-text index, podcasts/discover.py), +news and blog sources, their articles, saved radio stations. The books aren't here: their titles +are encrypted, the page searches them itself (static/js/search.js). + +Every word of the query has to be there, as the start of a word ("plattf" finds "Plattform"); +case and diacritics don't count.""" +from django.db.models import Q +from django.http import JsonResponse +from django.views.decorators.http import require_http_methods + +from feeds.models import Item, Source +from podcasts import discover +from podcasts.models import PodcastFeed +from radio.models import SavedStation + +LIMIT = 8 # per area; the area's own view has the rest +ARTICLE_SCAN = 400 # articles looked at after the database's rough match + + +def _words(q): + return [w for w in discover._words(q) if w][:8] + + +def _matches(words, *texts): + """Every word as the start of a word in the texts ("plattf" finds "Plattform").""" + have = discover._words(' '.join(t or '' for t in texts)) + return bool(words) and all(any(h.startswith(w) for h in have) for w in words) + + +def _rank(words, title): + """Title starting with the query first, then title words, then the rest.""" + t = discover._words(title) + if t[:len(words)] == words or (t and t[0].startswith(words[0])): + return 0 + return 1 if all(any(h.startswith(w) for h in t) for w in words) else 2 + + +def _feeds(user, words): + out = {'podcasts': [], 'audiobooks': []} + rows = PodcastFeed.objects.filter(user=user).values('id', 'title', 'author', 'artwork_url', 'kind', 'rss_url', + 'favorite', 'new_episodes') + found = [f for f in rows if _matches(words, f['title'], f['author'])] + found.sort(key=lambda f: (_rank(words, f['title']), (f['title'] or '').lower())) + for f in found: + key = 'audiobooks' if f['kind'] == PodcastFeed.AUDIOBOOK else 'podcasts' + out[key].append(f) + return {k: v[:LIMIT] for k, v in out.items()} | {'podcasts_total': len(out['podcasts']), + 'audiobooks_total': len(out['audiobooks'])} + + +def _sources(user, words): + rows = Source.objects.filter(user=user).values('id', 'title', 'url', 'kind', 'icon_url', 'category') + found = [s for s in rows if _matches(words, s['title'], s['url'], s['category'])] + found.sort(key=lambda s: (_rank(words, s['title']), (s['title'] or '').lower())) + return found[:LIMIT] + + +def _articles(user, words): + """Articles whose title or author has every word, the newest first. The database narrows by + the longest word (LIKE, as case-blind as SQLite is), Python decides.""" + longest = max(words, key=len) + rough = Q(title__icontains=longest) | Q(author__icontains=longest) + if longest != longest.capitalize(): + rough |= Q(title__contains=longest.capitalize()) + rows = (Item.objects.filter(user=user, gone=False).filter(rough).order_by('-published') + .values('id', 'title', 'author', 'kind', 'published', 'image_url', 'read_at', 'source__title')[:ARTICLE_SCAN]) + found = [i for i in rows if _matches(words, i['title'], i['author'])] + return [{'id': i['id'], 'title': i['title'], 'author': i['author'], 'kind': i['kind'], + 'published': i['published'].isoformat() if i['published'] else None, 'image_url': i['image_url'], + 'read': i['read_at'] is not None, 'source': i['source__title']} for i in found[:LIMIT * 2]], len(found) + + +def _stations(user, words): + rows = SavedStation.objects.filter(user=user).values('id', 'name', 'url', 'tags', 'country', 'favicon_url', + 'is_favorite') + found = [s for s in rows if _matches(words, s['name'], s['tags'], s['country'])] + found.sort(key=lambda s: (_rank(words, s['name']), not s['is_favorite'], s['name'].lower())) + return found[:LIMIT] + + +def search(user, q): + words = _words(q) + if not words or len(''.join(words)) < 2: + return {'q': q, 'empty': True} + out = {'q': q} + out.update(_feeds(user, words)) + ids = discover.search_ids(user, q, limit=LIMIT * 2) + out['episodes'] = discover._rows(user, ids) + out['sources'] = _sources(user, words) + out['articles'], out['articles_total'] = _articles(user, words) + out['stations'] = _stations(user, words) + return out + + +@require_http_methods(['GET']) +def search_view(request): + """?q=…: what matches in every area (see the module's text).""" + if not request.user.is_authenticated: + return JsonResponse({'error': 'authentication required'}, status=401) + return JsonResponse(search(request.user, request.GET.get('q', '')[:200])) diff --git a/diora/test_search.py b/diora/test_search.py new file mode 100644 index 0000000..863a8a3 --- /dev/null +++ b/diora/test_search.py @@ -0,0 +1,62 @@ +"""The search across diora (diora/search.py).""" +from datetime import datetime, timezone + +from django.contrib.auth.models import User +from django.test import TestCase + +from feeds.models import Item, Source +from podcasts import discover +from podcasts.models import PodcastEpisode, PodcastFeed +from radio.models import SavedStation + + +class SearchTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username='sucher', password='pw12345678') + self.client.force_login(self.user) + other = User.objects.create_user(username='andere', password='pw12345678') + when = datetime(2026, 3, 1, tzinfo=timezone.utc) + self.pod = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/p', title='Ökonomie heute', + author='Peter Decker') + self.book = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/b', title='Das Kapital', + author='Karl Marx', kind=PodcastFeed.AUDIOBOOK) + PodcastFeed.objects.create(user=other, rss_url='https://example.org/p', title='Ökonomie heute') + self.ep = PodcastEpisode.objects.create(feed=self.pod, guid='e1', title='Die Ökonomie der Plattformen', + audio_url='https://example.org/e1.mp3', pub_date=when) + PodcastEpisode.objects.create(feed=self.book, guid='k1', title='Kapitel 1: Die Ware', + audio_url='https://example.org/k1.mp3', pub_date=when) + discover.index_new() + blog = Source.objects.create(user=self.user, url='https://blog.example/feed', title='Plattform-Blog', kind=Source.BLOG) + Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='a', title='Plattformarbeit und Streik', + published=when) + Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='b', title='Gartenarbeit', published=when) + Item.objects.create(source=blog, user=self.user, kind=Source.BLOG, guid_hash='c', title='Plattformen, alt', + published=when, gone=True) + SavedStation.objects.create(user=self.user, name='Radio Plattform', url='https://radio.example/s', tags='jazz') + + def find(self, q): + return self.client.get('/search/', {'q': q}).json() + + def test_every_area(self): + r = self.find('plattf') + self.assertEqual([e['id'] for e in r['episodes']], [self.ep.id]) + self.assertEqual([a['title'] for a in r['articles']], ['Plattformarbeit und Streik']) + self.assertEqual([s['title'] for s in r['sources']], ['Plattform-Blog']) + self.assertEqual([s['name'] for s in r['stations']], ['Radio Plattform']) + + def test_podcasts_and_audiobooks_apart_case_and_accents_blind(self): + r = self.find('okonomie') + self.assertEqual([f['id'] for f in r['podcasts']], [self.pod.id]) + self.assertEqual(r['audiobooks'], []) + r = self.find('marx kap') + self.assertEqual([f['id'] for f in r['audiobooks']], [self.book.id]) + self.assertEqual(r['episodes'], [], 'an audiobook\'s chapters are no podcast episodes') + + def test_every_word_and_nothing_for_too_little(self): + self.assertEqual(self.find('plattform garten')['articles'], []) + self.assertTrue(self.find('a')['empty']) + self.assertEqual(self.client.get('/search/', {'q': 'x'}).status_code, 200) + + def test_only_signed_in(self): + self.client.logout() + self.assertEqual(self.client.get('/search/', {'q': 'plattform'}).status_code, 401) diff --git a/diora/urls.py b/diora/urls.py index 886ee26..3e1d1b7 100644 --- a/diora/urls.py +++ b/diora/urls.py @@ -5,6 +5,7 @@ from django.urls import path, include from django.views.static import serve as serve_static from accounts.sync import sync_snapshot +from diora.search import search_view urlpatterns = [ path('admin/', admin.site.urls), @@ -15,6 +16,7 @@ urlpatterns = [ path('tts/', include('tts.urls')), path('api/2/', include('gpodder.urls')), path('api/sync/', sync_snapshot, name='api_sync'), + path('search/', search_view, name='search'), # Served at the root (not /static/js/sw.js) so its default scope covers # the whole app — a service worker's scope is limited to its own script's # directory unless the server sends Service-Worker-Allowed, so registering diff --git a/docs/search.md b/docs/search.md new file mode 100644 index 0000000..3f248f3 --- /dev/null +++ b/docs/search.md @@ -0,0 +1,32 @@ +# Suche + +One field for all of diora: a section of its own in the navigation (`static/js/nav.js`, +section and group `suche`, flag `side`): + +- phone (the apps' tab bar, the web page's narrow bar): beside Home · Lesen · Hören; +- the apps' sidebar and the web page's wide bar: at the foot, above Abos & Quellen. + +It's never the section the app opens in next time (`diora_nav_section` isn't written for it). + +## What is searched where + +| Area | Where | Opens | +|---|---|---| +| Bücher, Zeitschriften | on the device (`_lastBookListData` / `bookMetaCache`): titles are encrypted, the server never sees them; title, author, series, file name, folders; editions of one work once | `openBook(id)` | +| Hörbücher, Podcasts (Abos) | server: title, author | `__dioraAudiobookOpen(id)` / `__dioraPodcastOpenFeed(id)` | +| Folgen | server: the full-text index (`podcasts/discover.py`, titles and shownotes, podcasts only) | `__dioraPodcastOpenEpisode(row)` if the podcast UI has it, else the podcast's page; ▶ plays (`podcastPlayEpisode`) | +| Artikel | server: titles and authors of news and blog items not expired | `__dioraFeedsOpenItem(id)` | +| Nachrichtenquellen, Blogs | server: title, address, category | the news / blogs | +| Radio | server: saved stations, name, tags, country | `playStation()` | + +Every word has to be there as the start of a word ("plattf" finds "Plattform"); case and +diacritics don't count. "Darüber hinaus" hands the query to Abos & Quellen (new podcasts, +YouTube, feeds) or the radio directory search. The last eight queries (Enter or a hit opened) +are kept on the device. `DioraSearch.open(q)` opens it from anywhere. + +## API + +`GET /search/?q=…` (signed in) → +`{q, podcasts, podcasts_total, audiobooks, audiobooks_total, episodes, sources, articles, +articles_total, stations}`; `{q, empty: true}` for less than two letters. Episodes come as the +episode lists give them (`_episode_list_data`). Eight per area (articles sixteen). diff --git a/e2e/navigation.spec.js b/e2e/navigation.spec.js index 2ec66c4..4e4b286 100644 --- a/e2e/navigation.spec.js +++ b/e2e/navigation.spec.js @@ -14,7 +14,7 @@ test('wide: every section in the bar under its group, each shows its panel and v const wide = page.locator('#tabs .nav-wide'); await expect(wide.locator('.nav-group-label')).toHaveText(['Lesen', 'Hören']); await expect(wide.locator('[data-nav]')).toHaveText( - ['Home', 'Bücher', 'Zeitschriften', 'Nachrichten', 'Blogs', 'Hörbücher', 'Podcasts', 'Radio', 'Ambient', 'Fokus', 'Abos & Quellen']); + ['Home', 'Bücher', 'Zeitschriften', 'Nachrichten', 'Blogs', 'Hörbücher', 'Podcasts', 'Radio', 'Ambient', 'Fokus', 'Suche', 'Abos & Quellen']); await wide.locator('[data-nav="blogs"]').click(); await expect(page).toHaveURL(/#blogs$/); @@ -33,7 +33,7 @@ test('narrow: three groups, the sections as a row on top, each group remembers i await page.setViewportSize({ width: 390, height: 800 }); await page.goto('/#news'); const narrow = page.locator('#tabs .nav-narrow'); - await expect(narrow.locator('[data-nav]')).toHaveText(['Home', 'Lesen', 'Hören', '']); + await expect(narrow.locator('[data-nav]')).toHaveText(['Home', 'Lesen', 'Hören', 'Suche', '']); await expect(narrow.locator('[data-nav="sources"] svg.dnav-ic')).toBeVisible(); await expect(narrow.locator('[data-nav="lesen"]')).toHaveClass(/active/); const segs = page.locator('#nav-segments'); @@ -61,6 +61,34 @@ test('old links and direct panel switches land in their section', async ({ page expect(await visiblePanel(page)).toEqual(['tab-podcasts']); // remembered }); +test('Suche: books from this device, the rest from the server, a hit opens where it lives', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 800 }); + await page.route('**/search/?q=*', route => route.fulfill({json: { + q: 'kapital', podcasts: [{id: 7, title: 'Kapital-Lesekreis', author: 'Rote Hilfe', artwork_url: '', favorite: true}], + podcasts_total: 1, audiobooks: [], audiobooks_total: 0, + episodes: [{id: 70, feed__id: 7, feed__title: 'Kapital-Lesekreis', title: 'Folge 3: Das Kapital, Band 1', duration_seconds: 3600}], + sources: [], articles: [{id: 5, title: 'Kapital und Krise', source: 'Ein Blog', kind: 'blog', published: null, read: false}], + articles_total: 1, stations: []}})); + await page.goto('/#home'); + await page.locator('#tabs .nav-narrow [data-nav="suche"]').click(); + await expect(page).toHaveURL(/#suche$/); + expect(await visiblePanel(page)).toEqual(['tab-suche']); + await expect(page.locator('#tab-suche .sx-q')).toBeFocused(); + await page.evaluate(() => { + _lastBookListData = [{id: 901, title: 'Das Kapital', author: 'Karl Marx', type: 'epub', keyOk: true}, + {id: 902, title: 'Gartenbuch', author: 'Jemand', type: 'epub', keyOk: true}]; + }); + await page.locator('#tab-suche .sx-q').fill('kapital'); + const groups = page.locator('#tab-suche .sx-h'); + await expect(groups).toHaveText([/Bücher/, /Podcasts/, /Folgen/, /Artikel/]); + await expect(page.locator('#tab-suche .sx-group').first().locator('mark')).toHaveText('Kapital'); + await page.locator('#tab-suche .sx-q').press('Enter'); + await page.locator('#tab-suche .sx-q').fill(''); + await expect(page.locator('#tab-suche .sx-recent .sx-chip')).toHaveText(['kapital']); + // Not where the app opens next time. + expect(await page.evaluate(() => localStorage.getItem('diora_nav_section'))).not.toBe('suche'); +}); + test('Abos & Quellen: one page for every subscription, reached from the feeds too', async ({ page }) => { await page.setViewportSize({ width: 1200, height: 800 }); await page.goto('/#news'); diff --git a/static/js/app/20-tabs.js b/static/js/app/20-tabs.js index fb1d079..dec1a2d 100644 --- a/static/js/app/20-tabs.js +++ b/static/js/app/20-tabs.js @@ -4,7 +4,7 @@ // Tabs // --------------------------------------------------------------------------- -const TOP_TABS = ['home', 'radio', 'focus', 'podcasts', 'books', 'feeds', 'ambient', 'audiobooks', 'sources']; +const TOP_TABS = ['home', 'radio', 'focus', 'podcasts', 'books', 'feeds', 'ambient', 'audiobooks', 'sources', 'suche']; const RADIO_SUB_TABS = ['search', 'saved', 'history']; // --------------------------------------------------------------------------- diff --git a/static/js/nav.js b/static/js/nav.js index fd14db3..66640bd 100644 --- a/static/js/nav.js +++ b/static/js/nav.js @@ -1,7 +1,7 @@ /* * The navigation (docs/navigation.md): diora's sections, sorted by what one does — * Home; Lesen: Bücher, Zeitschriften, Nachrichten, Blogs; Hören: Hörbücher, Podcasts, - * Radio, Ambient. One definition for every platform: the web page's #tabs are drawn + * Radio, Ambient; Suche across all of them. One definition for every platform: the web page's #tabs are drawn * here, the apps' tab bar and sidebar (native-radio.js) from DioraNav.groups() and * DioraNav.sections(). A section is a panel (showTab, as before) plus a view in it. * On a phone, Lesen and Hören show their sections as a row on top (#nav-segments). @@ -25,6 +25,9 @@ {id: 'ambient', group: 'hoeren', label: 'Ambient', icon: '∿', panel: 'ambient'}, // The focus timer: the web page's own (the apps never had it). {id: 'focus', group: 'hoeren', label: 'Fokus', icon: '◷', panel: 'focus', webOnly: true}, + // The search across diora (docs/search.md, search.js): beside Home · Lesen · Hören on a + // phone, at the foot of the sidebar (above Abos & Quellen) and of the wide bar ("side"). + {id: 'suche', group: 'suche', label: 'Suche', icon: '⌕', panel: 'suche', account: true, side: true}, // Managing every subscription (docs/sources.md): at the foot of the sidebar, in the app // menu on a phone — not a tab of its own. {id: 'sources', group: 'more', label: 'Abos & Quellen', icon: '⚙', panel: 'sources', account: true, footer: true}, @@ -33,6 +36,7 @@ {id: 'home', label: 'Home', icon: '⌂'}, {id: 'lesen', label: 'Lesen', icon: '▤'}, {id: 'hoeren', label: 'Hören', icon: '◉'}, + {id: 'suche', label: 'Suche', icon: '⌕'}, ]; const byId = {}; SECTIONS.forEach(s => { byId[s.id] = s; }); @@ -59,6 +63,7 @@ '', focus: '', sources: '', + suche: '', menu: '' + '', }; @@ -73,6 +78,8 @@ const nativeUi = () => html.classList.contains('nui-radio'); // the apps' own bars (native-radio.js) const readerApp = () => html.classList.contains('nui-reader-app'); const noRead = () => html.classList.contains('nui-noread'); // TV + // The apps' sidebar: a "side" section stands at its foot, not as a group. + const sidebar = () => html.classList.contains('nui-sidebar'); // Audiobooks need the podcast player: the page has them where it loads audiobooks.js // (diora Reader's build leaves it out). Known before the player's script has run. const podcastHost = () => !!document.querySelector('script[src$="/static/js/audiobooks.js"]'); @@ -89,7 +96,7 @@ function sections() { return SECTIONS.filter(available).map(s => ({id: s.id, group: s.group, label: s.label, icon: s.icon, svg: icon(s.id), - footer: !!s.footer})); + footer: !!s.footer || (!!s.side && sidebar())})); } function groups() { @@ -135,7 +142,7 @@ function setCurrent(id) { current = id; const s = byId[id]; - save('diora_nav_section', id); + if (!s.side) save('diora_nav_section', id); // the app opens where one was, not in the search if (!s.footer) save('diora_nav_last_' + s.group, id); syncHash(); render(); @@ -227,13 +234,16 @@ const gs = groups(); const foot = sections().filter(s => s.footer); if (tabs && !nativeUi()) { - // Wide: every section under its group; narrow: the groups (their sections on top). + // Wide: every section under its group, the "side" ones at the end with the footer's; + // narrow: the groups (their sections on top). + const side = id => !!byId[id].side; + const footWide = sections().filter(s => side(s.id)).concat(foot); tabs.innerHTML = - '