mrwnslz
69442f4a4d
TranscriptionSettings (Token Fernet-verschluesselt wie SiteLogin), TranscriptionJob und EpisodeArticle in podcasts; Source.synthetic in feeds, das Refresh, Aufraeumen, OPML-Export, Kommando und Umstellen auf Nachrichten ueberspringt bzw. verweigert. GET/POST /podcasts/transcription/settings/ und POST /podcasts/transcription/test/ (SSRF-Pruefung ueber chapters._open, jetzt auch fuer POST; eingebaute Instanz aus DIORA_WHISPER_URL ausgenommen). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
594 lines
33 KiB
Python
594 lines
33 KiB
Python
import json
|
||
from datetime import timedelta
|
||
from unittest import mock
|
||
|
||
import feedparser
|
||
from django.contrib.auth.models import User
|
||
from django.core.cache import cache
|
||
from django.test import TestCase
|
||
from django.utils import timezone
|
||
|
||
from podcasts.views import FeedFetch
|
||
|
||
from . import cluster, fetch, logins, refresh, textproc
|
||
from .models import FeedSettings, Item, SiteLogin, Source, Story
|
||
|
||
|
||
def rss(items, title='Testfeed', link='https://example.org/'):
|
||
body = ''.join(
|
||
f'<item><guid>{i["guid"]}</guid><title>{i["title"]}</title><link>{i.get("link", "https://example.org/" + i["guid"])}</link>'
|
||
f'<pubDate>{i.get("date", timezone.now().strftime("%a, %d %b %Y %H:%M:%S +0000"))}</pubDate>'
|
||
f'<description><![CDATA[{i.get("text", "")}]]></description></item>'
|
||
for i in items)
|
||
return f'<?xml version="1.0"?><rss version="2.0"><channel><title>{title}</title><link>{link}</link>{body}</channel></rss>'.encode()
|
||
|
||
|
||
def fetched(content, url='https://example.org/feed'):
|
||
return FeedFetch(200, feedparser.parse(content), 'etag', '', url, url, 'hash')
|
||
|
||
|
||
LONG = '<p>' + 'Ein langer Absatz mit sehr viel Text. ' * 40 + '</p>'
|
||
|
||
|
||
class FetchTest(TestCase):
|
||
def test_relative_links_become_absolute_and_scripts_go(self):
|
||
out = fetch.clean_html('<p>Hi <a href="/a">x</a><img src="i.jpg"><script>bad()</script>'
|
||
'<img data-src="/real.jpg" src="placeholder.gif"></p>', 'https://ex.org/post/1')
|
||
self.assertIn('href="https://ex.org/a"', out)
|
||
self.assertIn('src="https://ex.org/post/i.jpg"', out)
|
||
self.assertIn('src="https://ex.org/real.jpg"', out)
|
||
self.assertNotIn('placeholder', out)
|
||
self.assertNotIn('bad()', out)
|
||
|
||
def test_parse_entry(self):
|
||
e = feedparser.parse(rss([{'guid': 'g1', 'title': 'A & B', 'text': '<p>Kurz <img src="/p.jpg"></p>'}])).entries[0]
|
||
f = fetch.parse_entry(e, 'https://example.org/feed')
|
||
self.assertEqual(f['title'], 'A & B')
|
||
self.assertEqual(f['image_url'], 'https://example.org/p.jpg')
|
||
self.assertTrue(fetch.is_teaser(f['content']))
|
||
|
||
def test_drop_repeated_headline(self):
|
||
self.assertEqual(fetch._drop_title('<img src="x"><h1>Titel</h1><p>Text</p>'), '<img src="x"><p>Text</p>')
|
||
|
||
def test_short_title(self):
|
||
self.assertEqual(fetch.short_title('tagesschau.de - die erste Adresse für Nachrichten'), 'tagesschau.de')
|
||
self.assertEqual(fetch.short_title('Aktuell - FAZ.NET'), 'FAZ.NET')
|
||
self.assertEqual(fetch.short_title('DIE ZEIT | Nachrichten, News, Hintergründe'), 'DIE ZEIT')
|
||
self.assertEqual(fetch.short_title('Golem.de'), 'Golem.de')
|
||
self.assertEqual(fetch.short_title('Kotlin-Blog'), 'Kotlin-Blog')
|
||
|
||
def test_discover(self):
|
||
feeds, icon = fetch.discover('<link rel="alternate" type="application/rss+xml" href="/feed" title="F">'
|
||
'<link rel="icon" href="/fav.png">', 'https://ex.org/')
|
||
self.assertEqual(feeds, [('https://ex.org/feed', 'F')])
|
||
self.assertEqual(icon, 'https://ex.org/fav.png')
|
||
|
||
|
||
class ClusterTest(TestCase):
|
||
def setUp(self):
|
||
self.u = User.objects.create_user('u', password='pw12345678')
|
||
self.a = Source.objects.create(user=self.u, url='https://a/', title='A')
|
||
self.b = Source.objects.create(user=self.u, url='https://b/', title='B')
|
||
self.c = Source.objects.create(user=self.u, url='https://c/', title='C')
|
||
|
||
def item(self, source, title, summary=''):
|
||
return Item.objects.create(source=source, user=self.u, kind='news', guid_hash=title[:60],
|
||
title=title, summary=summary, published=timezone.now())
|
||
|
||
def test_same_event_bundled_other_apart(self):
|
||
c = cluster.Clusterer(self.u, timezone.now())
|
||
s1 = c.add(self.item(self.a, 'Oktoberfest endet mit neuem Besucherrekord in München'))
|
||
s2 = c.add(self.item(self.b, 'Rekord-Wiesn: 7,4 Millionen Besucher auf dem Oktoberfest in München'))
|
||
s3 = c.add(self.item(self.c, 'Bahn führt Familienticket dauerhaft ein'))
|
||
self.assertEqual(s1, s2)
|
||
self.assertNotEqual(s1, s3)
|
||
s1.refresh_from_db()
|
||
self.assertEqual((s1.size, s1.sources), (2, 2))
|
||
|
||
def test_same_source_needs_to_be_very_close(self):
|
||
c = cluster.Clusterer(self.u, timezone.now())
|
||
s1 = c.add(self.item(self.a, 'Nations League: Pressekonferenz vor Irland gegen Israel abgebrochen'))
|
||
s2 = c.add(self.item(self.a, 'Nations League: England schlägt Kroatien deutlich'))
|
||
self.assertNotEqual(s1, s2)
|
||
|
||
|
||
class ApiTest(TestCase):
|
||
def setUp(self):
|
||
self.u = User.objects.create_user('tester', password='pw12345678')
|
||
self.client.force_login(self.u)
|
||
|
||
def post(self, url, data=None):
|
||
return self.client.post(url, json.dumps(data or {}), content_type='application/json')
|
||
|
||
def add(self, kind, feed, url='https://example.org/feed'):
|
||
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed, url)), \
|
||
mock.patch('feeds.fetch.fetch_article', return_value=(LONG, 'https://example.org/img.jpg', False)):
|
||
r = self.post('/feeds/sources/add/', {'url': url, 'kind': kind, 'category': 'Politik'})
|
||
self.assertEqual(r.status_code, 200)
|
||
return r.json()['source']
|
||
|
||
def test_requires_login(self):
|
||
self.client.logout()
|
||
self.assertEqual(self.client.get('/feeds/state/').status_code, 401)
|
||
|
||
def test_add_news_fetches_articles_and_bundles(self):
|
||
s = self.add('news', rss([{'guid': 'a', 'title': 'Merz besucht Kiew und sagt Hilfe zu', 'text': 'Kurz.'},
|
||
{'guid': 'b', 'title': 'Bahn Familienticket dauerhaft', 'text': 'Kurz.'}]))
|
||
self.assertEqual(s['title'], 'Testfeed')
|
||
self.assertEqual(s['unread'], 2)
|
||
items = Item.objects.filter(user=self.u)
|
||
self.assertTrue(all(i.content_full for i in items))
|
||
self.assertEqual(Story.objects.filter(user=self.u).count(), 2)
|
||
news = self.client.get('/feeds/news/').json()
|
||
self.assertEqual(len(news['stories']), 2)
|
||
self.assertEqual(news['stories'][0]['category'], 'Politik')
|
||
b = self.client.get('/feeds/briefing/').json()
|
||
self.assertEqual(b['total'], 2)
|
||
self.assertEqual(b['sections'][0]['category'], 'Politik')
|
||
self.post('/feeds/briefing/seen/')
|
||
self.assertFalse(Item.objects.filter(user=self.u, read_at=None).exists())
|
||
self.assertEqual(self.client.get('/feeds/briefing/').json()['total'], 0)
|
||
|
||
def test_blog_keeps_full_feed_text_and_lists(self):
|
||
self.add('blog', rss([{'guid': 'p1', 'title': 'Ein Artikel', 'text': LONG}]))
|
||
item = Item.objects.get(user=self.u)
|
||
self.assertFalse(item.content_full) # the feed had the whole text
|
||
self.assertIsNone(item.story)
|
||
blogs = self.client.get('/feeds/blogs/?unread=1').json()
|
||
self.assertEqual([i['title'] for i in blogs['items']], ['Ein Artikel'])
|
||
detail = self.client.get(f'/feeds/items/{item.id}/').json()['item']
|
||
self.assertIn('langer Absatz', detail['content'])
|
||
self.post('/feeds/items/read/', {'ids': [item.id]})
|
||
self.assertEqual(self.client.get('/feeds/blogs/?unread=1').json()['items'], [])
|
||
self.post(f'/feeds/items/{item.id}/star/', {'starred': True})
|
||
self.assertEqual(len(self.client.get('/feeds/starred/').json()['items']), 1)
|
||
|
||
def test_blog_folders(self):
|
||
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'x', 'title': 'Essay', 'text': LONG}]))):
|
||
self.post('/feeds/sources/add/', {'url': 'https://a.example/feed', 'kind': 'blog', 'category': 'Philosophie'})
|
||
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'y', 'title': 'Code', 'text': LONG}]))):
|
||
self.post('/feeds/sources/add/', {'url': 'https://b.example/feed', 'kind': 'blog', 'category': ''})
|
||
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Philosophie'])
|
||
self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=Philosophie').json()['items']], ['Essay'])
|
||
self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=').json()['items']], ['Code'])
|
||
self.assertIn('text="Blogs – Philosophie"', self.client.get('/feeds/export.opml').content.decode())
|
||
|
||
def test_old_news_expires_and_is_not_taken_in_again(self):
|
||
old = (timezone.now() - timedelta(days=3)).strftime('%a, %d %b %Y %H:%M:%S +0000')
|
||
feed = rss([{'guid': 'new', 'title': 'Heute passiert', 'text': LONG},
|
||
{'guid': 'old', 'title': 'Vorgestern passiert', 'text': LONG, 'date': old}])
|
||
src = self.add('news', feed)
|
||
self.assertEqual(Item.objects.filter(user=self.u).count(), 1) # too old to take in
|
||
Item.objects.filter(user=self.u).update(published=timezone.now() - timedelta(days=3))
|
||
refresh.expire(self.u)
|
||
self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0)
|
||
self.assertEqual(Story.objects.filter(user=self.u).count(), 0)
|
||
source = Source.objects.get(pk=src['id'])
|
||
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed)):
|
||
refresh.refresh([source])
|
||
self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0)
|
||
|
||
def test_feed_wide_timestamp_means_first_seen(self):
|
||
stamp = (timezone.now() - timedelta(hours=5)).strftime('%a, %d %b %Y %H:%M:%S +0000')
|
||
self.add('news', rss([{'guid': f'g{n}', 'title': f'Meldung {n}', 'text': LONG, 'date': stamp} for n in range(6)]))
|
||
for i in Item.objects.filter(user=self.u):
|
||
self.assertLess((timezone.now() - i.published).total_seconds(), 600) # first fetch: feed order, minutes apart
|
||
|
||
def test_new_blog_without_dates_keeps_order_and_few_unread(self):
|
||
feed = (b'<?xml version="1.0"?><rss version="2.0"><channel><title>Undatiert</title>'
|
||
+ b''.join(b'<item><guid>u%d</guid><title>Post %d</title><description>x</description></item>' % (n, n)
|
||
for n in range(6)) + b'</channel></rss>')
|
||
self.add('blog', feed)
|
||
items = list(Item.objects.filter(user=self.u).order_by('-published'))
|
||
self.assertEqual([i.title for i in items], [f'Post {n}' for n in range(6)])
|
||
self.assertEqual(sum(1 for i in items if i.read_at is None), 3)
|
||
|
||
def test_switching_kind_moves_items(self):
|
||
src = self.add('news', rss([{'guid': 'a', 'title': 'Etwas', 'text': LONG}]))
|
||
self.post(f'/feeds/sources/{src["id"]}/', {'kind': 'blog'})
|
||
item = Item.objects.get(user=self.u)
|
||
self.assertEqual((item.kind, item.story), ('blog', None))
|
||
self.assertEqual(Story.objects.filter(user=self.u).count(), 0)
|
||
|
||
def test_other_users_items_invisible(self):
|
||
self.add('blog', rss([{'guid': 'p1', 'title': 'Privat', 'text': LONG}]))
|
||
item = Item.objects.get(user=self.u)
|
||
other = User.objects.create_user('o', password='pw12345678')
|
||
self.client.force_login(other)
|
||
self.assertEqual(self.client.get(f'/feeds/items/{item.id}/').status_code, 404)
|
||
self.assertEqual(self.client.get('/feeds/blogs/').json()['items'], [])
|
||
self.post('/feeds/items/read/', {'ids': [item.id]})
|
||
item.refresh_from_db()
|
||
self.assertIsNone(item.read_at)
|
||
|
||
def test_opml_export(self):
|
||
self.add('blog', rss([]), url='https://blog.example/feed')
|
||
r = self.client.get('/feeds/export.opml')
|
||
self.assertIn(b'xmlUrl="https://blog.example/feed"', r.content)
|
||
self.assertIn('text="Blogs – Politik"', r.content.decode())
|
||
|
||
def test_settings(self):
|
||
self.post('/feeds/settings/', {'news_keep_days': 5})
|
||
self.assertEqual(FeedSettings.objects.get(user=self.u).news_keep_days, 5)
|
||
|
||
|
||
class FakeResponse:
|
||
def __init__(self, status=302, location='/', text='', url='https://www.jungewelt.de/x', ctype='text/html'):
|
||
self.status_code = status
|
||
self.headers = {'Location': location, 'Content-Type': ctype}
|
||
self.text = text
|
||
self.url = url
|
||
|
||
|
||
PAYWALL = ('<html><body><article><h1>T</h1><p>' + 'Freier Anfang des Artikels. ' * 12 + '</p></article>'
|
||
'<p>Unterstützen Sie die junge Welt mit einem Abonnement und erhalten Sie Zugang zu allen Analysen</p></body></html>')
|
||
FULL = '<html><body><article><h1>T</h1>' + ('<p>' + 'Der ganze Artikel für Abonnenten. ' * 15 + '</p>') * 4 + '</article></body></html>'
|
||
|
||
|
||
class SiteLoginTest(TestCase):
|
||
def setUp(self):
|
||
self.u = User.objects.create_user('tester', password='pw12345678')
|
||
self.client.force_login(self.u)
|
||
self.src = Source.objects.create(user=self.u, url='https://www.jungewelt.de/feeds/newsticker.rss',
|
||
title='junge Welt', kind='news')
|
||
self.item = Item.objects.create(source=self.src, user=self.u, kind='news', guid_hash='a', title='Artikel',
|
||
url='https://www.jungewelt.de/artikel/1.html', summary='Anriss',
|
||
content='<p>Anriss</p>', published=timezone.now())
|
||
logins._sessions.clear()
|
||
|
||
def post(self, url, data):
|
||
return self.client.post(url, json.dumps(data), content_type='application/json')
|
||
|
||
def test_encryption_round_trip_and_not_plain(self):
|
||
token = logins.encrypt('geheim')
|
||
self.assertNotIn('geheim', token)
|
||
self.assertEqual(logins.decrypt(token), 'geheim')
|
||
self.assertIsNone(logins.decrypt('kaputt'))
|
||
|
||
def test_paywalled_page_keeps_free_part_and_waits(self):
|
||
with mock.patch('requests.get', return_value=FakeResponse(200, text=PAYWALL)):
|
||
refresh.load_article(self.item)
|
||
self.item.refresh_from_db()
|
||
self.assertTrue(self.item.paywalled)
|
||
self.assertFalse(self.item.content_full)
|
||
self.assertIn('Freier Anfang', self.item.content)
|
||
self.assertFalse(refresh.needs_article(self.item, self.src)) # no login: don't try again
|
||
|
||
def test_wrong_login_is_refused_and_not_stored(self):
|
||
with mock.patch('requests.Session.post', return_value=FakeResponse(302, 'https://www.jungewelt.de/loginFailed.php?x')):
|
||
r = self.post('/feeds/logins/', {'source': self.src.id, 'username': 'u', 'password': 'falsch'})
|
||
self.assertEqual(r.status_code, 400)
|
||
self.assertIn('falsch', r.json()['error'])
|
||
self.assertFalse(SiteLogin.objects.exists())
|
||
|
||
def test_login_stores_encrypted_and_refetches_full_articles(self):
|
||
Item.objects.filter(pk=self.item.pk).update(paywalled=True)
|
||
|
||
def login_post(session, url, **kw):
|
||
session.cookies.set('jW', 'sitzung')
|
||
return FakeResponse(302, 'https://www.jungewelt.de/')
|
||
|
||
with mock.patch('requests.Session.post', autospec=True, side_effect=login_post), \
|
||
mock.patch('requests.Session.get', return_value=FakeResponse(200, text=FULL)):
|
||
r = self.post('/feeds/logins/', {'source': self.src.id, 'username': 'leser', 'password': 'abo-passwort'})
|
||
self.assertEqual(r.status_code, 200)
|
||
self.assertEqual(r.json()['refetched'], 1)
|
||
login = SiteLogin.objects.get(user=self.u)
|
||
self.assertNotIn('abo-passwort', login.secret)
|
||
self.item.refresh_from_db()
|
||
self.assertTrue(self.item.content_full)
|
||
self.assertFalse(self.item.paywalled)
|
||
self.assertIn('ganze Artikel', self.item.content)
|
||
state = self.client.get('/feeds/state/').json()
|
||
info = state['sources'][0]['login']
|
||
self.assertEqual((info['username'], info['ok'], info['site']), ('leser', True, 'junge Welt'))
|
||
self.assertNotIn('abo-passwort', json.dumps(state))
|
||
self.post('/feeds/logins/delete/', {'source': self.src.id})
|
||
self.assertFalse(SiteLogin.objects.exists())
|
||
|
||
def test_sources_of_other_sites_have_no_login(self):
|
||
other = Source.objects.create(user=self.u, url='https://blog.example/feed', title='B', kind='blog')
|
||
state = self.client.get('/feeds/state/').json()
|
||
by_id = {s['id']: s for s in state['sources']}
|
||
self.assertIsNone(by_id[other.id]['login'])
|
||
self.assertIsNotNone(by_id[self.src.id]['login'])
|
||
|
||
|
||
class BlogFeedTest(TestCase):
|
||
def setUp(self):
|
||
self.u = User.objects.create_user('tester', password='pw12345678')
|
||
self.client.force_login(self.u)
|
||
now = timezone.now()
|
||
self.busy = Source.objects.create(user=self.u, url='https://busy/', title='Busy', kind='blog', folders=['Politik'])
|
||
self.rare = Source.objects.create(user=self.u, url='https://rare/', title='Rare', kind='blog', folders=['Politik'])
|
||
for n in range(10): # a post every two hours
|
||
Item.objects.create(source=self.busy, user=self.u, kind='blog', guid_hash=f'b{n}', title=f'Busy {n}',
|
||
content='<p>' + 'wort ' * 460 + '</p>', words=460, published=now - timedelta(hours=2 * n))
|
||
Item.objects.create(source=self.rare, user=self.u, kind='blog', guid_hash='r', title='Rare essay',
|
||
published=now - timedelta(hours=5))
|
||
|
||
def post(self, url, data):
|
||
return self.client.post(url, json.dumps(data), content_type='application/json')
|
||
|
||
def by_id(self):
|
||
return {s['id']: s for s in self.client.get('/feeds/state/').json()['sources']}
|
||
|
||
def test_interval_new_and_words(self):
|
||
st = self.by_id()
|
||
self.assertLess(st[self.busy.id]['interval_days'], 1)
|
||
self.assertEqual(st[self.rare.id]['interval_days'], 30.0) # one post: counts as rare
|
||
self.assertEqual(st[self.busy.id]['new'], 10)
|
||
items = self.client.get('/feeds/blogs/?unread=1').json()['items']
|
||
self.assertEqual(next(i for i in items if i['title'] == 'Busy 0')['words'], 460)
|
||
|
||
def test_hidden_blog_only_in_its_folder_and_not_counted(self):
|
||
self.post(f'/feeds/sources/{self.busy.id}/', {'hidden': True})
|
||
state = self.client.get('/feeds/state/').json()
|
||
self.assertEqual((state['blogs_unread'], state['blogs_new']), (1, 1))
|
||
main = [i['title'] for i in self.client.get('/feeds/blogs/?unread=1').json()['items']]
|
||
self.assertEqual(main, ['Rare essay'])
|
||
folder = self.client.get('/feeds/blogs/?unread=1&folder=Politik').json()['items']
|
||
self.assertEqual(len(folder), 11)
|
||
|
||
def test_favorites_filter(self):
|
||
self.post(f'/feeds/sources/{self.rare.id}/', {'fav': True})
|
||
self.assertTrue(self.by_id()[self.rare.id]['fav'])
|
||
favs = [i['title'] for i in self.client.get('/feeds/blogs/?fav=1').json()['items']]
|
||
self.assertEqual(favs, ['Rare essay'])
|
||
|
||
|
||
class NotifyTest(TestCase):
|
||
def setUp(self):
|
||
self.u = User.objects.create_user('tester', password='pw12345678')
|
||
self.client.force_login(self.u)
|
||
self.src = Source.objects.create(user=self.u, url='https://example.org/feed', title='Blog', kind='blog', notify=True)
|
||
|
||
def fetch(self, items):
|
||
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss(items))):
|
||
refresh.refresh([self.src])
|
||
|
||
def test_rules(self):
|
||
old = (timezone.now() - timedelta(days=5)).strftime('%a, %d %b %Y %H:%M:%S +0000')
|
||
self.fetch([{'guid': 'a', 'title': 'Schon da', 'text': LONG}]) # first fetch: never
|
||
start = self.client.get('/feeds/notifications/').json()
|
||
self.assertEqual(start['items'], [])
|
||
self.fetch([{'guid': 'a', 'title': 'Schon da', 'text': LONG},
|
||
{'guid': 'b', 'title': 'Neuer Post', 'text': LONG},
|
||
{'guid': 'c', 'title': 'Alter Post', 'text': LONG, 'date': old}])
|
||
got = self.client.get(f"/feeds/notifications/?after={start['last']}").json()
|
||
self.assertEqual([i['title'] for i in got['items']], ['Neuer Post'])
|
||
# Seen up to there: nothing again; read or bell off: nothing either.
|
||
self.assertEqual(self.client.get(f"/feeds/notifications/?after={got['last']}").json()['items'], [])
|
||
Source.objects.filter(pk=self.src.pk).update(notify=False)
|
||
self.assertEqual(self.client.get(f"/feeds/notifications/?after={start['last']}").json()['items'], [])
|
||
|
||
|
||
class FolderAndProgressTest(TestCase):
|
||
def setUp(self):
|
||
self.u = User.objects.create_user('tester', password='pw12345678')
|
||
self.client.force_login(self.u)
|
||
self.a = Source.objects.create(user=self.u, url='https://a/', title='A', kind='blog', folders=['Philosophie'])
|
||
self.b = Source.objects.create(user=self.u, url='https://b/', title='B', kind='blog', folders=['philosophie', 'Rust'])
|
||
self.post_a = Item.objects.create(source=self.a, user=self.u, kind='blog', guid_hash='a', title='Essay',
|
||
published=timezone.now())
|
||
|
||
def post(self, url, data):
|
||
return self.client.post(url, json.dumps(data), content_type='application/json')
|
||
|
||
def test_folders_case_insensitive_rename_dissolve(self):
|
||
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Philosophie', 'Rust'])
|
||
self.post(f'/feeds/sources/{self.a.id}/', {'folders': [' rust ', 'Neu', 'neu']})
|
||
self.a.refresh_from_db()
|
||
self.assertEqual(self.a.folders, ['Rust', 'Neu']) # spelled as the existing one, no doubles
|
||
self.post('/feeds/folders/rename/', {'name': 'rust', 'new': 'Rust & Linux'})
|
||
self.a.refresh_from_db(); self.b.refresh_from_db()
|
||
self.assertEqual(self.a.folders, ['Neu', 'Rust & Linux'])
|
||
self.assertEqual(self.b.folders, ['philosophie', 'Rust & Linux'])
|
||
self.post('/feeds/folders/dissolve/', {'name': 'Rust & Linux'})
|
||
self.b.refresh_from_db()
|
||
self.assertEqual(self.b.folders, ['philosophie'])
|
||
self.assertTrue(Source.objects.filter(pk=self.b.pk).exists())
|
||
titles = [i['title'] for i in self.client.get('/feeds/blogs/?folder=Neu').json()['items']]
|
||
self.assertEqual(titles, ['Essay'])
|
||
|
||
def test_progress_marks_read_at_ninety_percent(self):
|
||
r = self.post(f'/feeds/items/{self.post_a.id}/progress/', {'progress': 0.4}).json()
|
||
self.assertEqual(r, {'progress': 0.4, 'read': False})
|
||
self.assertEqual(self.client.get('/feeds/blogs/').json()['items'][0]['progress'], 0.4)
|
||
self.assertTrue(self.post(f'/feeds/items/{self.post_a.id}/progress/', {'progress': 0.95}).json()['read'])
|
||
|
||
|
||
class DiscoverKindTest(TestCase):
|
||
"""Abos & Quellen: a feed whose entries carry sound is offered as a podcast."""
|
||
|
||
def setUp(self):
|
||
self.u = User.objects.create_user('u', password='pw12345678')
|
||
self.client.force_login(self.u)
|
||
|
||
def discover(self, items):
|
||
xml = ('<?xml version="1.0"?><rss version="2.0"><channel><title>X</title>' + ''.join(
|
||
f'<item><title>Folge {i}</title><guid>g{i}</guid>{extra}</item>' for i, extra in enumerate(items)) +
|
||
'</channel></rss>')
|
||
with mock.patch('feeds.views._probe', lambda url: (('https://ex.org/feed', 'X', feedparser.parse(xml)), None)):
|
||
r = self.client.post('/feeds/discover/', json.dumps({'url': 'https://ex.org/feed'}),
|
||
content_type='application/json')
|
||
return r.json()['feeds'][0]
|
||
|
||
def test_sound_files_make_a_podcast(self):
|
||
enc = '<enclosure url="https://ex.org/a.mp3" type="audio/mpeg" length="1"/>'
|
||
f = self.discover([enc] * 4)
|
||
self.assertEqual(f['suggested_kind'], 'podcast')
|
||
self.assertFalse(f['podcast_subscribed'])
|
||
|
||
def test_text_entries_stay_news_or_a_blog(self):
|
||
self.assertIn(self.discover([''] * 4)['suggested_kind'], ('news', 'blog'))
|
||
|
||
|
||
class FolderOrderTest(TestCase):
|
||
def test_blog_folders_come_in_the_users_order(self):
|
||
u = User.objects.create_user('u', password='pw12345678')
|
||
self.client.force_login(u)
|
||
Source.objects.create(user=u, url='https://a/', title='A', kind='blog', folders=['Kunst', 'Code'])
|
||
Source.objects.create(user=u, url='https://b/', title='B', kind='blog', folders=['Politik'])
|
||
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Code', 'Kunst', 'Politik'])
|
||
r = self.client.post('/feeds/folders/order/', json.dumps({'order': ['Politik', 'kunst']}),
|
||
content_type='application/json')
|
||
self.assertEqual(r.json()['folders'], ['Politik', 'Kunst', 'Code'])
|
||
self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Politik', 'Kunst', 'Code'])
|
||
|
||
|
||
DONATE = 'Unterstützen Sie unabhängigen Journalismus mit einer Spende an unsere Redaktion.'
|
||
FRANCE = [
|
||
('Proteste in Frankreich: Gewalt bei Demonstrationen gegen Macron', 'Die Proteste in Frankreich gegen die Regierung Macron nehmen zu.'),
|
||
('Frankreich: Proteste gegen Sparpläne weiten sich aus', 'In Frankreich wachsen die Proteste gegen die Sparpläne von Macron.'),
|
||
('Macron verurteilt Gewalt bei Protesten in Frankreich', 'Präsident Macron verurteilt die Gewalt bei den Protesten in Frankreich.'),
|
||
]
|
||
|
||
|
||
class TextprocTest(TestCase):
|
||
def test_furniture_is_what_a_source_prints_under_many_articles(self):
|
||
texts = [f'Artikel Nummer {n} über etwas ganz anderes als die anderen. {DONATE}' for n in 'abcd']
|
||
keys = textproc.furniture(texts)
|
||
self.assertEqual(textproc.without_furniture(texts[0], keys), 'Artikel Nummer a über etwas ganz anderes als die anderen.')
|
||
html = f'<p>Der eigentliche Text des Artikels steht hier.</p><p>{DONATE}</p>'
|
||
self.assertEqual(textproc.html_without_furniture(html, keys), '<p>Der eigentliche Text des Artikels steht hier.</p>')
|
||
self.assertEqual(textproc.furniture(texts[:2]), set()) # twice is no pattern
|
||
|
||
def test_overview_of_two_reports(self):
|
||
shared = 'Die Regierung in Paris hat am Montag ein neues Sparpaket von vierzig Milliarden Euro vorgestellt. '
|
||
a = shared + 'Gewerkschaften kündigten daraufhin landesweite Streiks für die kommende Woche an und riefen zu Demonstrationen auf.'
|
||
b = shared + 'Die Börse in Paris reagierte mit deutlichen Kursgewinnen bei den großen Banken des Landes am Nachmittag.'
|
||
docs = [('taz', a + ' Gewerkschaften Gewerkschaften'), ('FAZ', b + ' Börse Börse')]
|
||
self.assertTrue(textproc.summarize(docs, k=2))
|
||
novel = textproc.novel(docs)
|
||
self.assertIn('Gewerkschaften', novel['taz'][0])
|
||
self.assertIn('Börse', novel['FAZ'][0])
|
||
base = ' Die Regierung stellte das Paket vor und die Opposition kritisierte es scharf.' * 3
|
||
words = textproc.word_choice([('taz', base + ' Gewerkschaften streiken.' * 5), ('FAZ', base + ' Börse steigt.' * 5)])
|
||
self.assertEqual(words, {'taz': ['Gewerkschaften'], 'FAZ': ['Börse']})
|
||
|
||
def test_topics_group_one_matter_from_several_sources(self):
|
||
stories = [(n, f'{t}. {s}', {src}) for n, ((t, s), src) in enumerate(zip(FRANCE, ['taz', 'nd', 'tagesschau']))]
|
||
stories.append((9, 'Bahn erhöht die Preise für das Deutschlandticket. Fahrgäste zahlen mehr.', {'taz'}))
|
||
got = textproc.topics(stories)
|
||
self.assertEqual(len(got), 1)
|
||
self.assertEqual(sorted(got[0]['ids']), [0, 1, 2])
|
||
self.assertIn('Frankreich', got[0]['label'])
|
||
self.assertEqual(textproc.topics([(n, t, {'taz'}) for n, t, _ in stories[:3]]), []) # one source: no topic
|
||
|
||
def test_related_story_in_the_other_language(self):
|
||
de = 'Der Papst Leo traf in Beirut den libanesischen Präsidenten Aoun und Ministerpräsident Salam.'
|
||
en = 'Pope Leo met Lebanese President Aoun and Prime Minister Salam in Beirut on Sunday.'
|
||
other = 'Die Bahn erhöht in Berlin die Preise für das Deutschlandticket deutlich.'
|
||
sl = [(1, textproc.language(de), textproc.names(de)), (2, textproc.language(en), textproc.names(en)),
|
||
(3, textproc.language(other), textproc.names(other))]
|
||
self.assertEqual((sl[0][1], sl[1][1]), ('de', 'en'))
|
||
got = textproc.related(sl, min_weight=1.0)
|
||
self.assertEqual([b for b, _ in got[1]], [2])
|
||
self.assertNotIn(3, got)
|
||
|
||
|
||
class NewsTextApiTest(TestCase):
|
||
def setUp(self):
|
||
self.u = User.objects.create_user('reader', password='pw12345678')
|
||
self.client.force_login(self.u)
|
||
cache.clear()
|
||
now = timezone.now()
|
||
self.items = []
|
||
for n, ((title, text), name) in enumerate(zip(FRANCE, ['taz', 'nd', 'tagesschau'])):
|
||
src = Source.objects.create(user=self.u, url=f'https://{name}/feed', title=name, kind='news', category='Welt')
|
||
story = Story.objects.create(user=self.u, first_at=now, last_at=now - timedelta(minutes=n))
|
||
for k in range(4): # older articles of the source, each with the donation box
|
||
Item.objects.create(source=src, user=self.u, kind='news', guid_hash=f'{name}{k}', title=f'Alt {k}',
|
||
published=now - timedelta(days=1, minutes=k),
|
||
content=f'<p>Ein älterer Artikel Nummer {k} von {name}.</p><p>{DONATE}</p>')
|
||
i = Item.objects.create(source=src, user=self.u, kind='news', guid_hash=f'{name}x', title=title,
|
||
summary=text, published=now - timedelta(minutes=n), story=story,
|
||
content=f'<p>{text} Mehr dazu im Laufe des Tages von {name}.</p><p>{DONATE}</p>')
|
||
story.lead = i
|
||
story.save()
|
||
self.items.append(i)
|
||
|
||
def test_first_page_has_topics_and_ids_filter(self):
|
||
r = self.client.get('/feeds/news/').json()
|
||
self.assertEqual(len(r['topics']), 1)
|
||
ids = r['topics'][0]['ids']
|
||
self.assertEqual(len(ids), 3)
|
||
got = self.client.get('/feeds/news/?ids=' + ','.join(map(str, ids[:2]))).json()
|
||
self.assertEqual(sorted(s['id'] for s in got['stories']), sorted(ids[:2]))
|
||
self.assertNotIn('topics', got)
|
||
|
||
def test_unencoded_plus_in_before_is_no_error(self):
|
||
when = timezone.now().isoformat() # +00:00, sent unencoded the '+' becomes a space
|
||
r = self.client.get('/feeds/news/?before=' + when.replace('+', ' '))
|
||
self.assertEqual(r.status_code, 200)
|
||
self.assertEqual(self.client.get('/feeds/news/?before=garbled').status_code, 200)
|
||
|
||
def test_article_comes_without_furniture(self):
|
||
got = self.client.get(f'/feeds/items/{self.items[0].id}/').json()['item']
|
||
self.assertNotIn('Spende', got['content'])
|
||
self.assertIn('Proteste', got['content'])
|
||
|
||
def test_overview_of_a_story_with_several_sources(self):
|
||
a, b = self.items[0], self.items[1]
|
||
b.story = a.story
|
||
b.save()
|
||
got = self.client.get(f'/feeds/stories/{a.story_id}/overview/').json()
|
||
self.assertTrue(got['summary'])
|
||
self.assertFalse(any('Spende' in x['text'] for x in got['summary']))
|
||
self.assertEqual(got['related'], [])
|
||
single = self.client.get(f'/feeds/stories/{self.items[2].story_id}/overview/').json()
|
||
self.assertNotIn('summary', single)
|
||
|
||
|
||
class SyntheticSourceTest(TestCase):
|
||
"""A source diora fills itself ("Podcast-Artikel") is never fetched or pruned."""
|
||
|
||
def setUp(self):
|
||
self.u = User.objects.create_user('tester', password='pw12345678')
|
||
self.client.force_login(self.u)
|
||
self.synth = Source.objects.create(user=self.u, url='diora://podcast-artikel', title='Podcast-Artikel',
|
||
kind='blog', fav=True, synthetic=True)
|
||
self.real = Source.objects.create(user=self.u, url='https://real.example/feed', title='Real', kind='blog')
|
||
|
||
def post(self, url, data):
|
||
return self.client.post(url, json.dumps(data), content_type='application/json')
|
||
|
||
def test_refresh_skips_it(self):
|
||
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([]), 'https://real.example/feed')) as get:
|
||
refresh.refresh([self.synth, self.real])
|
||
self.assertEqual([c.args[0] for c in get.call_args_list], ['https://real.example/feed'])
|
||
|
||
def test_refresh_now_and_command_skip_it(self):
|
||
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([]), 'https://real.example/feed')) as get:
|
||
r = self.post('/feeds/refresh/', {})
|
||
from django.core.management import call_command
|
||
call_command('refresh_newsfeeds', stdout=mock.MagicMock())
|
||
self.assertEqual(r.status_code, 200)
|
||
self.assertTrue(get.call_args_list)
|
||
self.assertTrue(all(c.args[0] == 'https://real.example/feed' for c in get.call_args_list))
|
||
call_command('refresh_newsfeeds', '--source-id', str(self.synth.id), stdout=mock.MagicMock())
|
||
|
||
def test_old_items_are_not_pruned(self):
|
||
old = timezone.now() - timedelta(days=400)
|
||
for source in (self.synth, self.real):
|
||
for n in range(refresh.BLOG_KEEP + 5):
|
||
Item.objects.create(source=source, user=self.u, kind='blog', guid_hash=f'{source.pk}-{n}',
|
||
title='x', published=old - timedelta(minutes=n), read_at=old)
|
||
refresh.expire(self.u)
|
||
self.assertEqual(self.synth.items.count(), refresh.BLOG_KEEP + 5)
|
||
self.assertEqual(self.real.items.count(), refresh.BLOG_KEEP)
|
||
|
||
def test_cannot_become_news_but_can_be_hidden(self):
|
||
r = self.post(f'/feeds/sources/{self.synth.id}/', {'kind': 'news'})
|
||
self.assertEqual(r.status_code, 400)
|
||
r = self.post(f'/feeds/sources/{self.synth.id}/', {'hidden': True, 'notify': True})
|
||
self.assertEqual(r.status_code, 200)
|
||
self.synth.refresh_from_db()
|
||
self.assertEqual((self.synth.kind, self.synth.hidden, self.synth.notify), ('blog', True, True))
|
||
self.assertTrue(r.json()['source']['synthetic'])
|
||
|
||
def test_not_in_opml(self):
|
||
body = self.client.get('/feeds/export.opml').content.decode()
|
||
self.assertIn('real.example', body)
|
||
self.assertNotIn('diora://', body)
|