import json
from datetime import timedelta
from unittest import mock

import feedparser
from django.contrib.auth.models import User
from django.core.cache import cache
from django.test import TestCase
from django.utils import timezone

from podcasts.views import FeedFetch

from . import cluster, fetch, logins, refresh, textproc
from .models import FeedSettings, Item, SiteLogin, Source, Story


def rss(items, title='Testfeed', link='https://example.org/'):
    body = ''.join(
        f'<item><guid>{i["guid"]}</guid><title>{i["title"]}</title><link>{i.get("link", "https://example.org/" + i["guid"])}</link>'
        f'<pubDate>{i.get("date", timezone.now().strftime("%a, %d %b %Y %H:%M:%S +0000"))}</pubDate>'
        f'<description><![CDATA[{i.get("text", "")}]]></description></item>'
        for i in items)
    return f'<?xml version="1.0"?><rss version="2.0"><channel><title>{title}</title><link>{link}</link>{body}</channel></rss>'.encode()


def fetched(content, url='https://example.org/feed'):
    return FeedFetch(200, feedparser.parse(content), 'etag', '', url, url, 'hash')


LONG = '<p>' + 'Ein langer Absatz mit sehr viel Text. ' * 40 + '</p>'


class FetchTest(TestCase):
    def test_relative_links_become_absolute_and_scripts_go(self):
        out = fetch.clean_html('<p>Hi <a href="/a">x</a><img src="i.jpg"><script>bad()</script>'
                               '<img data-src="/real.jpg" src="placeholder.gif"></p>', 'https://ex.org/post/1')
        self.assertIn('href="https://ex.org/a"', out)
        self.assertIn('src="https://ex.org/post/i.jpg"', out)
        self.assertIn('src="https://ex.org/real.jpg"', out)
        self.assertNotIn('placeholder', out)
        self.assertNotIn('bad()', out)

    def test_parse_entry(self):
        e = feedparser.parse(rss([{'guid': 'g1', 'title': 'A &amp; B', 'text': '<p>Kurz <img src="/p.jpg"></p>'}])).entries[0]
        f = fetch.parse_entry(e, 'https://example.org/feed')
        self.assertEqual(f['title'], 'A & B')
        self.assertEqual(f['image_url'], 'https://example.org/p.jpg')
        self.assertTrue(fetch.is_teaser(f['content']))

    def test_drop_repeated_headline(self):
        self.assertEqual(fetch._drop_title('<img src="x"><h1>Titel</h1><p>Text</p>'), '<img src="x"><p>Text</p>')

    def test_short_title(self):
        self.assertEqual(fetch.short_title('tagesschau.de - die erste Adresse für Nachrichten'), 'tagesschau.de')
        self.assertEqual(fetch.short_title('Aktuell - FAZ.NET'), 'FAZ.NET')
        self.assertEqual(fetch.short_title('DIE ZEIT | Nachrichten, News, Hintergründe'), 'DIE ZEIT')
        self.assertEqual(fetch.short_title('Golem.de'), 'Golem.de')
        self.assertEqual(fetch.short_title('Kotlin-Blog'), 'Kotlin-Blog')

    def test_discover(self):
        feeds, icon = fetch.discover('<link rel="alternate" type="application/rss+xml" href="/feed" title="F">'
                                     '<link rel="icon" href="/fav.png">', 'https://ex.org/')
        self.assertEqual(feeds, [('https://ex.org/feed', 'F')])
        self.assertEqual(icon, 'https://ex.org/fav.png')


class ClusterTest(TestCase):
    def setUp(self):
        self.u = User.objects.create_user('u', password='pw12345678')
        self.a = Source.objects.create(user=self.u, url='https://a/', title='A')
        self.b = Source.objects.create(user=self.u, url='https://b/', title='B')
        self.c = Source.objects.create(user=self.u, url='https://c/', title='C')

    def item(self, source, title, summary=''):
        return Item.objects.create(source=source, user=self.u, kind='news', guid_hash=title[:60],
                                   title=title, summary=summary, published=timezone.now())

    def test_same_event_bundled_other_apart(self):
        c = cluster.Clusterer(self.u, timezone.now())
        s1 = c.add(self.item(self.a, 'Oktoberfest endet mit neuem Besucherrekord in München'))
        s2 = c.add(self.item(self.b, 'Rekord-Wiesn: 7,4 Millionen Besucher auf dem Oktoberfest in München'))
        s3 = c.add(self.item(self.c, 'Bahn führt Familienticket dauerhaft ein'))
        self.assertEqual(s1, s2)
        self.assertNotEqual(s1, s3)
        s1.refresh_from_db()
        self.assertEqual((s1.size, s1.sources), (2, 2))

    def test_same_source_needs_to_be_very_close(self):
        c = cluster.Clusterer(self.u, timezone.now())
        s1 = c.add(self.item(self.a, 'Nations League: Pressekonferenz vor Irland gegen Israel abgebrochen'))
        s2 = c.add(self.item(self.a, 'Nations League: England schlägt Kroatien deutlich'))
        self.assertNotEqual(s1, s2)


class ApiTest(TestCase):
    def setUp(self):
        self.u = User.objects.create_user('tester', password='pw12345678')
        self.client.force_login(self.u)

    def post(self, url, data=None):
        return self.client.post(url, json.dumps(data or {}), content_type='application/json')

    def add(self, kind, feed, url='https://example.org/feed'):
        with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed, url)), \
                mock.patch('feeds.fetch.fetch_article', return_value=(LONG, 'https://example.org/img.jpg', False)):
            r = self.post('/feeds/sources/add/', {'url': url, 'kind': kind, 'category': 'Politik'})
        self.assertEqual(r.status_code, 200)
        return r.json()['source']

    def test_requires_login(self):
        self.client.logout()
        self.assertEqual(self.client.get('/feeds/state/').status_code, 401)

    def test_add_news_fetches_articles_and_bundles(self):
        s = self.add('news', rss([{'guid': 'a', 'title': 'Merz besucht Kiew und sagt Hilfe zu', 'text': 'Kurz.'},
                                   {'guid': 'b', 'title': 'Bahn Familienticket dauerhaft', 'text': 'Kurz.'}]))
        self.assertEqual(s['title'], 'Testfeed')
        self.assertEqual(s['unread'], 2)
        items = Item.objects.filter(user=self.u)
        self.assertTrue(all(i.content_full for i in items))
        self.assertEqual(Story.objects.filter(user=self.u).count(), 2)
        news = self.client.get('/feeds/news/').json()
        self.assertEqual(len(news['stories']), 2)
        self.assertEqual(news['stories'][0]['category'], 'Politik')
        b = self.client.get('/feeds/briefing/').json()
        self.assertEqual(b['total'], 2)
        self.assertEqual(b['sections'][0]['category'], 'Politik')
        self.post('/feeds/briefing/seen/')
        self.assertFalse(Item.objects.filter(user=self.u, read_at=None).exists())
        self.assertEqual(self.client.get('/feeds/briefing/').json()['total'], 0)

    def test_blog_keeps_full_feed_text_and_lists(self):
        self.add('blog', rss([{'guid': 'p1', 'title': 'Ein Artikel', 'text': LONG}]))
        item = Item.objects.get(user=self.u)
        self.assertFalse(item.content_full)  # the feed had the whole text
        self.assertIsNone(item.story)
        blogs = self.client.get('/feeds/blogs/?unread=1').json()
        self.assertEqual([i['title'] for i in blogs['items']], ['Ein Artikel'])
        detail = self.client.get(f'/feeds/items/{item.id}/').json()['item']
        self.assertIn('langer Absatz', detail['content'])
        self.post('/feeds/items/read/', {'ids': [item.id]})
        self.assertEqual(self.client.get('/feeds/blogs/?unread=1').json()['items'], [])
        self.post(f'/feeds/items/{item.id}/star/', {'starred': True})
        self.assertEqual(len(self.client.get('/feeds/starred/').json()['items']), 1)

    def test_blog_folders(self):
        with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'x', 'title': 'Essay', 'text': LONG}]))):
            self.post('/feeds/sources/add/', {'url': 'https://a.example/feed', 'kind': 'blog', 'category': 'Philosophie'})
        with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss([{'guid': 'y', 'title': 'Code', 'text': LONG}]))):
            self.post('/feeds/sources/add/', {'url': 'https://b.example/feed', 'kind': 'blog', 'category': ''})
        self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Philosophie'])
        self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=Philosophie').json()['items']], ['Essay'])
        self.assertEqual([i['title'] for i in self.client.get('/feeds/blogs/?folder=').json()['items']], ['Code'])
        self.assertIn('text="Blogs – Philosophie"', self.client.get('/feeds/export.opml').content.decode())

    def test_old_news_expires_and_is_not_taken_in_again(self):
        old = (timezone.now() - timedelta(days=3)).strftime('%a, %d %b %Y %H:%M:%S +0000')
        feed = rss([{'guid': 'new', 'title': 'Heute passiert', 'text': LONG},
                    {'guid': 'old', 'title': 'Vorgestern passiert', 'text': LONG, 'date': old}])
        src = self.add('news', feed)
        self.assertEqual(Item.objects.filter(user=self.u).count(), 1)  # too old to take in
        Item.objects.filter(user=self.u).update(published=timezone.now() - timedelta(days=3))
        refresh.expire(self.u)
        self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0)
        self.assertEqual(Story.objects.filter(user=self.u).count(), 0)
        source = Source.objects.get(pk=src['id'])
        with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed)):
            refresh.refresh([source])
        self.assertEqual(Item.objects.filter(user=self.u, gone=False).count(), 0)

    def test_feed_wide_timestamp_means_first_seen(self):
        stamp = (timezone.now() - timedelta(hours=5)).strftime('%a, %d %b %Y %H:%M:%S +0000')
        self.add('news', rss([{'guid': f'g{n}', 'title': f'Meldung {n}', 'text': LONG, 'date': stamp} for n in range(6)]))
        for i in Item.objects.filter(user=self.u):
            self.assertLess((timezone.now() - i.published).total_seconds(), 600)  # first fetch: feed order, minutes apart

    def test_new_blog_without_dates_keeps_order_and_few_unread(self):
        feed = (b'<?xml version="1.0"?><rss version="2.0"><channel><title>Undatiert</title>'
                + b''.join(b'<item><guid>u%d</guid><title>Post %d</title><description>x</description></item>' % (n, n)
                           for n in range(6)) + b'</channel></rss>')
        self.add('blog', feed)
        items = list(Item.objects.filter(user=self.u).order_by('-published'))
        self.assertEqual([i.title for i in items], [f'Post {n}' for n in range(6)])
        self.assertEqual(sum(1 for i in items if i.read_at is None), 3)

    def test_switching_kind_moves_items(self):
        src = self.add('news', rss([{'guid': 'a', 'title': 'Etwas', 'text': LONG}]))
        self.post(f'/feeds/sources/{src["id"]}/', {'kind': 'blog'})
        item = Item.objects.get(user=self.u)
        self.assertEqual((item.kind, item.story), ('blog', None))
        self.assertEqual(Story.objects.filter(user=self.u).count(), 0)

    def test_other_users_items_invisible(self):
        self.add('blog', rss([{'guid': 'p1', 'title': 'Privat', 'text': LONG}]))
        item = Item.objects.get(user=self.u)
        other = User.objects.create_user('o', password='pw12345678')
        self.client.force_login(other)
        self.assertEqual(self.client.get(f'/feeds/items/{item.id}/').status_code, 404)
        self.assertEqual(self.client.get('/feeds/blogs/').json()['items'], [])
        self.post('/feeds/items/read/', {'ids': [item.id]})
        item.refresh_from_db()
        self.assertIsNone(item.read_at)

    def test_opml_export(self):
        self.add('blog', rss([]), url='https://blog.example/feed')
        r = self.client.get('/feeds/export.opml')
        self.assertIn(b'xmlUrl="https://blog.example/feed"', r.content)
        self.assertIn('text="Blogs – Politik"', r.content.decode())

    def test_settings(self):
        self.post('/feeds/settings/', {'news_keep_days': 5})
        self.assertEqual(FeedSettings.objects.get(user=self.u).news_keep_days, 5)


class FakeResponse:
    def __init__(self, status=302, location='/', text='', url='https://www.jungewelt.de/x', ctype='text/html'):
        self.status_code = status
        self.headers = {'Location': location, 'Content-Type': ctype}
        self.text = text
        self.url = url


PAYWALL = ('<html><body><article><h1>T</h1><p>' + 'Freier Anfang des Artikels. ' * 12 + '</p></article>'
           '<p>Unterstützen Sie die junge Welt mit einem Abonnement und erhalten Sie Zugang zu allen Analysen</p></body></html>')
FULL = '<html><body><article><h1>T</h1>' + ('<p>' + 'Der ganze Artikel für Abonnenten. ' * 15 + '</p>') * 4 + '</article></body></html>'


class SiteLoginTest(TestCase):
    def setUp(self):
        self.u = User.objects.create_user('tester', password='pw12345678')
        self.client.force_login(self.u)
        self.src = Source.objects.create(user=self.u, url='https://www.jungewelt.de/feeds/newsticker.rss',
                                         title='junge Welt', kind='news')
        self.item = Item.objects.create(source=self.src, user=self.u, kind='news', guid_hash='a', title='Artikel',
                                        url='https://www.jungewelt.de/artikel/1.html', summary='Anriss',
                                        content='<p>Anriss</p>', published=timezone.now())
        logins._sessions.clear()

    def post(self, url, data):
        return self.client.post(url, json.dumps(data), content_type='application/json')

    def test_encryption_round_trip_and_not_plain(self):
        token = logins.encrypt('geheim')
        self.assertNotIn('geheim', token)
        self.assertEqual(logins.decrypt(token), 'geheim')
        self.assertIsNone(logins.decrypt('kaputt'))

    def test_paywalled_page_keeps_free_part_and_waits(self):
        with mock.patch('requests.get', return_value=FakeResponse(200, text=PAYWALL)):
            refresh.load_article(self.item)
        self.item.refresh_from_db()
        self.assertTrue(self.item.paywalled)
        self.assertFalse(self.item.content_full)
        self.assertIn('Freier Anfang', self.item.content)
        self.assertFalse(refresh.needs_article(self.item, self.src))  # no login: don't try again

    def test_wrong_login_is_refused_and_not_stored(self):
        with mock.patch('requests.Session.post', return_value=FakeResponse(302, 'https://www.jungewelt.de/loginFailed.php?x')):
            r = self.post('/feeds/logins/', {'source': self.src.id, 'username': 'u', 'password': 'falsch'})
        self.assertEqual(r.status_code, 400)
        self.assertIn('falsch', r.json()['error'])
        self.assertFalse(SiteLogin.objects.exists())

    def test_login_stores_encrypted_and_refetches_full_articles(self):
        Item.objects.filter(pk=self.item.pk).update(paywalled=True)

        def login_post(session, url, **kw):
            session.cookies.set('jW', 'sitzung')
            return FakeResponse(302, 'https://www.jungewelt.de/')

        with mock.patch('requests.Session.post', autospec=True, side_effect=login_post), \
                mock.patch('requests.Session.get', return_value=FakeResponse(200, text=FULL)):
            r = self.post('/feeds/logins/', {'source': self.src.id, 'username': 'leser', 'password': 'abo-passwort'})
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.json()['refetched'], 1)
        login = SiteLogin.objects.get(user=self.u)
        self.assertNotIn('abo-passwort', login.secret)
        self.item.refresh_from_db()
        self.assertTrue(self.item.content_full)
        self.assertFalse(self.item.paywalled)
        self.assertIn('ganze Artikel', self.item.content)
        state = self.client.get('/feeds/state/').json()
        info = state['sources'][0]['login']
        self.assertEqual((info['username'], info['ok'], info['site']), ('leser', True, 'junge Welt'))
        self.assertNotIn('abo-passwort', json.dumps(state))
        self.post('/feeds/logins/delete/', {'source': self.src.id})
        self.assertFalse(SiteLogin.objects.exists())

    def test_sources_of_other_sites_have_no_login(self):
        other = Source.objects.create(user=self.u, url='https://blog.example/feed', title='B', kind='blog')
        state = self.client.get('/feeds/state/').json()
        by_id = {s['id']: s for s in state['sources']}
        self.assertIsNone(by_id[other.id]['login'])
        self.assertIsNotNone(by_id[self.src.id]['login'])


class BlogFeedTest(TestCase):
    def setUp(self):
        self.u = User.objects.create_user('tester', password='pw12345678')
        self.client.force_login(self.u)
        now = timezone.now()
        self.busy = Source.objects.create(user=self.u, url='https://busy/', title='Busy', kind='blog', folders=['Politik'])
        self.rare = Source.objects.create(user=self.u, url='https://rare/', title='Rare', kind='blog', folders=['Politik'])
        for n in range(10):  # a post every two hours
            Item.objects.create(source=self.busy, user=self.u, kind='blog', guid_hash=f'b{n}', title=f'Busy {n}',
                                content='<p>' + 'wort ' * 460 + '</p>', words=460, published=now - timedelta(hours=2 * n))
        Item.objects.create(source=self.rare, user=self.u, kind='blog', guid_hash='r', title='Rare essay',
                            published=now - timedelta(hours=5))

    def post(self, url, data):
        return self.client.post(url, json.dumps(data), content_type='application/json')

    def by_id(self):
        return {s['id']: s for s in self.client.get('/feeds/state/').json()['sources']}

    def test_interval_new_and_words(self):
        st = self.by_id()
        self.assertLess(st[self.busy.id]['interval_days'], 1)
        self.assertEqual(st[self.rare.id]['interval_days'], 30.0)   # one post: counts as rare
        self.assertEqual(st[self.busy.id]['new'], 10)
        items = self.client.get('/feeds/blogs/?unread=1').json()['items']
        self.assertEqual(next(i for i in items if i['title'] == 'Busy 0')['words'], 460)

    def test_hidden_blog_only_in_its_folder_and_not_counted(self):
        self.post(f'/feeds/sources/{self.busy.id}/', {'hidden': True})
        state = self.client.get('/feeds/state/').json()
        self.assertEqual((state['blogs_unread'], state['blogs_new']), (1, 1))
        main = [i['title'] for i in self.client.get('/feeds/blogs/?unread=1').json()['items']]
        self.assertEqual(main, ['Rare essay'])
        folder = self.client.get('/feeds/blogs/?unread=1&folder=Politik').json()['items']
        self.assertEqual(len(folder), 11)

    def test_favorites_filter(self):
        self.post(f'/feeds/sources/{self.rare.id}/', {'fav': True})
        self.assertTrue(self.by_id()[self.rare.id]['fav'])
        favs = [i['title'] for i in self.client.get('/feeds/blogs/?fav=1').json()['items']]
        self.assertEqual(favs, ['Rare essay'])


class NotifyTest(TestCase):
    def setUp(self):
        self.u = User.objects.create_user('tester', password='pw12345678')
        self.client.force_login(self.u)
        self.src = Source.objects.create(user=self.u, url='https://example.org/feed', title='Blog', kind='blog', notify=True)

    def fetch(self, items):
        with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(rss(items))):
            refresh.refresh([self.src])

    def test_rules(self):
        old = (timezone.now() - timedelta(days=5)).strftime('%a, %d %b %Y %H:%M:%S +0000')
        self.fetch([{'guid': 'a', 'title': 'Schon da', 'text': LONG}])           # first fetch: never
        start = self.client.get('/feeds/notifications/').json()
        self.assertEqual(start['items'], [])
        self.fetch([{'guid': 'a', 'title': 'Schon da', 'text': LONG},
                    {'guid': 'b', 'title': 'Neuer Post', 'text': LONG},
                    {'guid': 'c', 'title': 'Alter Post', 'text': LONG, 'date': old}])
        got = self.client.get(f"/feeds/notifications/?after={start['last']}").json()
        self.assertEqual([i['title'] for i in got['items']], ['Neuer Post'])
        # Seen up to there: nothing again; read or bell off: nothing either.
        self.assertEqual(self.client.get(f"/feeds/notifications/?after={got['last']}").json()['items'], [])
        Source.objects.filter(pk=self.src.pk).update(notify=False)
        self.assertEqual(self.client.get(f"/feeds/notifications/?after={start['last']}").json()['items'], [])


class FolderAndProgressTest(TestCase):
    def setUp(self):
        self.u = User.objects.create_user('tester', password='pw12345678')
        self.client.force_login(self.u)
        self.a = Source.objects.create(user=self.u, url='https://a/', title='A', kind='blog', folders=['Philosophie'])
        self.b = Source.objects.create(user=self.u, url='https://b/', title='B', kind='blog', folders=['philosophie', 'Rust'])
        self.post_a = Item.objects.create(source=self.a, user=self.u, kind='blog', guid_hash='a', title='Essay',
                                          published=timezone.now())

    def post(self, url, data):
        return self.client.post(url, json.dumps(data), content_type='application/json')

    def test_folders_case_insensitive_rename_dissolve(self):
        self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Philosophie', 'Rust'])
        self.post(f'/feeds/sources/{self.a.id}/', {'folders': ['  rust ', 'Neu', 'neu']})
        self.a.refresh_from_db()
        self.assertEqual(self.a.folders, ['Rust', 'Neu'])          # spelled as the existing one, no doubles
        self.post('/feeds/folders/rename/', {'name': 'rust', 'new': 'Rust & Linux'})
        self.a.refresh_from_db(); self.b.refresh_from_db()
        self.assertEqual(self.a.folders, ['Neu', 'Rust & Linux'])
        self.assertEqual(self.b.folders, ['philosophie', 'Rust & Linux'])
        self.post('/feeds/folders/dissolve/', {'name': 'Rust & Linux'})
        self.b.refresh_from_db()
        self.assertEqual(self.b.folders, ['philosophie'])
        self.assertTrue(Source.objects.filter(pk=self.b.pk).exists())
        titles = [i['title'] for i in self.client.get('/feeds/blogs/?folder=Neu').json()['items']]
        self.assertEqual(titles, ['Essay'])

    def test_progress_marks_read_at_ninety_percent(self):
        r = self.post(f'/feeds/items/{self.post_a.id}/progress/', {'progress': 0.4}).json()
        self.assertEqual(r, {'progress': 0.4, 'read': False})
        self.assertEqual(self.client.get('/feeds/blogs/').json()['items'][0]['progress'], 0.4)
        self.assertTrue(self.post(f'/feeds/items/{self.post_a.id}/progress/', {'progress': 0.95}).json()['read'])


class DiscoverKindTest(TestCase):
    """Abos & Quellen: a feed whose entries carry sound is offered as a podcast."""

    def setUp(self):
        self.u = User.objects.create_user('u', password='pw12345678')
        self.client.force_login(self.u)

    def discover(self, items):
        xml = ('<?xml version="1.0"?><rss version="2.0"><channel><title>X</title>' + ''.join(
            f'<item><title>Folge {i}</title><guid>g{i}</guid>{extra}</item>' for i, extra in enumerate(items)) +
            '</channel></rss>')
        with mock.patch('feeds.views._probe', lambda url: (('https://ex.org/feed', 'X', feedparser.parse(xml)), None)):
            r = self.client.post('/feeds/discover/', json.dumps({'url': 'https://ex.org/feed'}),
                                 content_type='application/json')
        return r.json()['feeds'][0]

    def test_sound_files_make_a_podcast(self):
        enc = '<enclosure url="https://ex.org/a.mp3" type="audio/mpeg" length="1"/>'
        f = self.discover([enc] * 4)
        self.assertEqual(f['suggested_kind'], 'podcast')
        self.assertFalse(f['podcast_subscribed'])

    def test_text_entries_stay_news_or_a_blog(self):
        self.assertIn(self.discover([''] * 4)['suggested_kind'], ('news', 'blog'))


class FolderOrderTest(TestCase):
    def test_blog_folders_come_in_the_users_order(self):
        u = User.objects.create_user('u', password='pw12345678')
        self.client.force_login(u)
        Source.objects.create(user=u, url='https://a/', title='A', kind='blog', folders=['Kunst', 'Code'])
        Source.objects.create(user=u, url='https://b/', title='B', kind='blog', folders=['Politik'])
        self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Code', 'Kunst', 'Politik'])
        r = self.client.post('/feeds/folders/order/', json.dumps({'order': ['Politik', 'kunst']}),
                             content_type='application/json')
        self.assertEqual(r.json()['folders'], ['Politik', 'Kunst', 'Code'])
        self.assertEqual(self.client.get('/feeds/state/').json()['folders'], ['Politik', 'Kunst', 'Code'])


DONATE = 'Unterstützen Sie unabhängigen Journalismus mit einer Spende an unsere Redaktion.'
FRANCE = [
    ('Proteste in Frankreich: Gewalt bei Demonstrationen gegen Macron', 'Die Proteste in Frankreich gegen die Regierung Macron nehmen zu.'),
    ('Frankreich: Proteste gegen Sparpläne weiten sich aus', 'In Frankreich wachsen die Proteste gegen die Sparpläne von Macron.'),
    ('Macron verurteilt Gewalt bei Protesten in Frankreich', 'Präsident Macron verurteilt die Gewalt bei den Protesten in Frankreich.'),
]


class TextprocTest(TestCase):
    def test_furniture_is_what_a_source_prints_under_many_articles(self):
        texts = [f'Artikel Nummer {n} über etwas ganz anderes als die anderen. {DONATE}' for n in 'abcd']
        keys = textproc.furniture(texts)
        self.assertEqual(textproc.without_furniture(texts[0], keys), 'Artikel Nummer a über etwas ganz anderes als die anderen.')
        html = f'<p>Der eigentliche Text des Artikels steht hier.</p><p>{DONATE}</p>'
        self.assertEqual(textproc.html_without_furniture(html, keys), '<p>Der eigentliche Text des Artikels steht hier.</p>')
        self.assertEqual(textproc.furniture(texts[:2]), set())   # twice is no pattern

    def test_overview_of_two_reports(self):
        shared = 'Die Regierung in Paris hat am Montag ein neues Sparpaket von vierzig Milliarden Euro vorgestellt. '
        a = shared + 'Gewerkschaften kündigten daraufhin landesweite Streiks für die kommende Woche an und riefen zu Demonstrationen auf.'
        b = shared + 'Die Börse in Paris reagierte mit deutlichen Kursgewinnen bei den großen Banken des Landes am Nachmittag.'
        docs = [('taz', a + ' Gewerkschaften Gewerkschaften'), ('FAZ', b + ' Börse Börse')]
        self.assertTrue(textproc.summarize(docs, k=2))
        novel = textproc.novel(docs)
        self.assertIn('Gewerkschaften', novel['taz'][0])
        self.assertIn('Börse', novel['FAZ'][0])
        base = ' Die Regierung stellte das Paket vor und die Opposition kritisierte es scharf.' * 3
        words = textproc.word_choice([('taz', base + ' Gewerkschaften streiken.' * 5), ('FAZ', base + ' Börse steigt.' * 5)])
        self.assertEqual(words, {'taz': ['Gewerkschaften'], 'FAZ': ['Börse']})

    def test_topics_group_one_matter_from_several_sources(self):
        stories = [(n, f'{t}. {s}', {src}) for n, ((t, s), src) in enumerate(zip(FRANCE, ['taz', 'nd', 'tagesschau']))]
        stories.append((9, 'Bahn erhöht die Preise für das Deutschlandticket. Fahrgäste zahlen mehr.', {'taz'}))
        got = textproc.topics(stories)
        self.assertEqual(len(got), 1)
        self.assertEqual(sorted(got[0]['ids']), [0, 1, 2])
        self.assertIn('Frankreich', got[0]['label'])
        self.assertEqual(textproc.topics([(n, t, {'taz'}) for n, t, _ in stories[:3]]), [])  # one source: no topic

    def test_related_story_in_the_other_language(self):
        de = 'Der Papst Leo traf in Beirut den libanesischen Präsidenten Aoun und Ministerpräsident Salam.'
        en = 'Pope Leo met Lebanese President Aoun and Prime Minister Salam in Beirut on Sunday.'
        other = 'Die Bahn erhöht in Berlin die Preise für das Deutschlandticket deutlich.'
        sl = [(1, textproc.language(de), textproc.names(de)), (2, textproc.language(en), textproc.names(en)),
              (3, textproc.language(other), textproc.names(other))]
        self.assertEqual((sl[0][1], sl[1][1]), ('de', 'en'))
        got = textproc.related(sl, min_weight=1.0)
        self.assertEqual([b for b, _ in got[1]], [2])
        self.assertNotIn(3, got)


class NewsTextApiTest(TestCase):
    def setUp(self):
        self.u = User.objects.create_user('reader', password='pw12345678')
        self.client.force_login(self.u)
        cache.clear()
        now = timezone.now()
        self.items = []
        for n, ((title, text), name) in enumerate(zip(FRANCE, ['taz', 'nd', 'tagesschau'])):
            src = Source.objects.create(user=self.u, url=f'https://{name}/feed', title=name, kind='news', category='Welt')
            story = Story.objects.create(user=self.u, first_at=now, last_at=now - timedelta(minutes=n))
            for k in range(4):   # older articles of the source, each with the donation box
                Item.objects.create(source=src, user=self.u, kind='news', guid_hash=f'{name}{k}', title=f'Alt {k}',
                                    published=now - timedelta(days=1, minutes=k),
                                    content=f'<p>Ein älterer Artikel Nummer {k} von {name}.</p><p>{DONATE}</p>')
            i = Item.objects.create(source=src, user=self.u, kind='news', guid_hash=f'{name}x', title=title,
                                    summary=text, published=now - timedelta(minutes=n), story=story,
                                    content=f'<p>{text} Mehr dazu im Laufe des Tages von {name}.</p><p>{DONATE}</p>')
            story.lead = i
            story.save()
            self.items.append(i)

    def test_first_page_has_topics_and_ids_filter(self):
        r = self.client.get('/feeds/news/').json()
        self.assertEqual(len(r['topics']), 1)
        ids = r['topics'][0]['ids']
        self.assertEqual(len(ids), 3)
        got = self.client.get('/feeds/news/?ids=' + ','.join(map(str, ids[:2]))).json()
        self.assertEqual(sorted(s['id'] for s in got['stories']), sorted(ids[:2]))
        self.assertNotIn('topics', got)

    def test_unencoded_plus_in_before_is_no_error(self):
        when = timezone.now().isoformat()   # +00:00, sent unencoded the '+' becomes a space
        r = self.client.get('/feeds/news/?before=' + when.replace('+', ' '))
        self.assertEqual(r.status_code, 200)
        self.assertEqual(self.client.get('/feeds/news/?before=garbled').status_code, 200)

    def test_article_comes_without_furniture(self):
        got = self.client.get(f'/feeds/items/{self.items[0].id}/').json()['item']
        self.assertNotIn('Spende', got['content'])
        self.assertIn('Proteste', got['content'])

    def test_overview_of_a_story_with_several_sources(self):
        a, b = self.items[0], self.items[1]
        b.story = a.story
        b.save()
        got = self.client.get(f'/feeds/stories/{a.story_id}/overview/').json()
        self.assertTrue(got['summary'])
        self.assertFalse(any('Spende' in x['text'] for x in got['summary']))
        self.assertEqual(got['related'], [])
        single = self.client.get(f'/feeds/stories/{self.items[2].story_id}/overview/').json()
        self.assertNotIn('summary', single)