Feeds: Abo-Anmeldung für Seiten mit Bezahlschranke (junge Welt)
- In den Einstellungen einer Quelle: Zugangsdaten der Seite; der Server prüft die Anmeldung sofort, speichert das Passwort verschlüsselt (Fernet, Schlüssel aus SECRET_KEY) und gibt es nie wieder heraus - Artikel werden mit der angemeldeten Sitzung geholt; hinter der Bezahlschranke bleibt der freie Teil (paywalled) und wird nach der Anmeldung neu geholt - Adapter je Seite (feeds/logins.py), zuerst jungewelt.de - sw v92 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
c8e03f343e
commit
bc1b7cb25e
11 changed files with 454 additions and 22 deletions
|
|
@ -154,19 +154,24 @@ def _drop_title(html_text):
|
|||
return re.sub(r'^\s*(<img[^>]*>\s*)?<h1>.*?</h1>', lambda m: m.group(1) or '', html_text, count=1, flags=re.S)
|
||||
|
||||
|
||||
def fetch_article(url):
|
||||
"""The article on `url` as (clean HTML, image URL), or None. trafilatura
|
||||
finds the main text and leaves out navigation, ads, comments."""
|
||||
def fetch_article(url, http=None):
|
||||
"""The article on `url` as (clean HTML, image URL, paywalled), or None.
|
||||
trafilatura finds the main text and leaves out navigation, ads, comments.
|
||||
`http`: a logged-in session for a paywalled site (logins.py); paywalled
|
||||
says the page showed only the free part anyway."""
|
||||
import trafilatura # heavy import, only where articles are fetched
|
||||
from .logins import adapter_for
|
||||
|
||||
try:
|
||||
r = requests.get(url, timeout=PAGE_TIMEOUT, headers={'User-Agent': USER_AGENT})
|
||||
r = (http or requests).get(url, timeout=PAGE_TIMEOUT, headers={'User-Agent': USER_AGENT})
|
||||
except requests.RequestException as e:
|
||||
log.info('article %s: %s', url, type(e).__name__)
|
||||
return None
|
||||
if r.status_code >= 400 or 'html' not in r.headers.get('Content-Type', 'text/html'):
|
||||
return None
|
||||
page = r.text
|
||||
adapter = adapter_for(r.url)
|
||||
paywalled = bool(adapter and adapter.paywalled(page))
|
||||
try:
|
||||
body = trafilatura.extract(page, url=r.url, output_format='html', include_images=True,
|
||||
include_links=True, include_tables=True, favor_precision=True)
|
||||
|
|
@ -177,7 +182,7 @@ def fetch_article(url):
|
|||
return None
|
||||
body = re.sub(r'^\s*<html>\s*<body>|</body>\s*</html>\s*$', '', body.strip())
|
||||
content = clean_html(_drop_title(body), r.url)
|
||||
if len(plain(content)) < 200:
|
||||
if len(plain(content)) < 200 and not paywalled:
|
||||
return None
|
||||
image = ''
|
||||
try:
|
||||
|
|
@ -185,7 +190,7 @@ def fetch_article(url):
|
|||
image = (meta.image if meta else '') or ''
|
||||
except Exception:
|
||||
pass
|
||||
return content, image
|
||||
return content, image, paywalled
|
||||
|
||||
|
||||
# ---- finding a site's feed
|
||||
|
|
|
|||
167
feeds/logins.py
Normal file
167
feeds/logins.py
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
"""Subscriber logins for sites behind a paywall (junge Welt …).
|
||||
|
||||
The account holder enters the site's username and password in the feed's
|
||||
settings; the server logs in with them and fetches the articles as a
|
||||
subscriber. The password is stored encrypted (Fernet, key derived from
|
||||
SECRET_KEY — a copy of the database alone doesn't reveal it) and never
|
||||
handed out again. Each supported site is an adapter: how to log in, how a
|
||||
paywalled page looks.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
TIMEOUT = 15
|
||||
SESSION_SECONDS = 6 * 3600 # a logged-in session is reused this long, then logs in again
|
||||
|
||||
|
||||
class LoginFailed(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class JungeWelt:
|
||||
domain = 'jungewelt.de'
|
||||
name = 'junge Welt'
|
||||
|
||||
def login(self, http, username, password):
|
||||
r = http.post('https://www.jungewelt.de/login.php', timeout=TIMEOUT, allow_redirects=False,
|
||||
data={'username': username, 'password': password, 'ref': '/', 'submit': 'Login'})
|
||||
where = r.headers.get('Location', '')
|
||||
if 'loginFailed' in where:
|
||||
raise LoginFailed('Benutzername oder Passwort falsch')
|
||||
if r.status_code not in (301, 302, 303) or not http.cookies.get('jW'):
|
||||
raise LoginFailed(f'Anmeldung fehlgeschlagen ({r.status_code})')
|
||||
|
||||
def paywalled(self, page):
|
||||
return 'mit einem Abonnement und erhalten Sie Zugang' in page
|
||||
|
||||
|
||||
ADAPTERS = {a.domain: a for a in (JungeWelt(),)}
|
||||
|
||||
|
||||
def adapter_for(url):
|
||||
host = (urlparse(url or '').hostname or '').lower()
|
||||
for domain, adapter in ADAPTERS.items():
|
||||
if host == domain or host.endswith('.' + domain):
|
||||
return adapter
|
||||
return None
|
||||
|
||||
|
||||
def _fernet():
|
||||
key = hashlib.sha256(b'diora feeds site login\0' + settings.SECRET_KEY.encode()).digest()
|
||||
return Fernet(base64.urlsafe_b64encode(key))
|
||||
|
||||
|
||||
def encrypt(text):
|
||||
return _fernet().encrypt(text.encode()).decode()
|
||||
|
||||
|
||||
def decrypt(token):
|
||||
"""The plain text, or None when it can't be read (SECRET_KEY changed)."""
|
||||
try:
|
||||
return _fernet().decrypt(token.encode()).decode()
|
||||
except (InvalidToken, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def new_http():
|
||||
from .fetch import USER_AGENT
|
||||
s = requests.Session()
|
||||
s.headers['User-Agent'] = USER_AGENT
|
||||
return s
|
||||
|
||||
|
||||
# Logged-in sessions in this process: (user id, domain) -> (session, logged in at).
|
||||
_sessions = {}
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def forget(user_id, domain):
|
||||
with _lock:
|
||||
_sessions.pop((user_id, domain), None)
|
||||
|
||||
|
||||
def check_and_store(user, url, username, password):
|
||||
"""Log in once; on success store the login (encrypted). Raises LoginFailed."""
|
||||
from .models import SiteLogin
|
||||
adapter = adapter_for(url)
|
||||
if adapter is None:
|
||||
raise LoginFailed('Für diese Seite gibt es keine Anmeldung.')
|
||||
http = new_http()
|
||||
try:
|
||||
adapter.login(http, username, password)
|
||||
except requests.RequestException:
|
||||
raise LoginFailed(f'{adapter.name} ist nicht erreichbar.')
|
||||
SiteLogin.objects.update_or_create(user=user, domain=adapter.domain, defaults={
|
||||
'username': username, 'secret': encrypt(password), 'ok': True, 'last_error': '',
|
||||
'checked_at': timezone.now()})
|
||||
with _lock:
|
||||
_sessions[(user.id, adapter.domain)] = (http, time.monotonic())
|
||||
return adapter
|
||||
|
||||
|
||||
def session_for(user_id, url, fresh=False):
|
||||
"""A logged-in requests session for the article's site, or None (no login
|
||||
stored, or it no longer works — then the login is marked as failing)."""
|
||||
from .models import SiteLogin
|
||||
adapter = adapter_for(url)
|
||||
if adapter is None:
|
||||
return None
|
||||
key = (user_id, adapter.domain)
|
||||
with _lock:
|
||||
cached = _sessions.get(key)
|
||||
if cached and not fresh and time.monotonic() - cached[1] < SESSION_SECONDS:
|
||||
return cached[0]
|
||||
login = SiteLogin.objects.filter(user_id=user_id, domain=adapter.domain).first()
|
||||
if login is None:
|
||||
return None
|
||||
password = decrypt(login.secret)
|
||||
if password is None:
|
||||
login.ok, login.last_error = False, 'Bitte die Zugangsdaten neu eingeben.'
|
||||
login.save(update_fields=['ok', 'last_error'])
|
||||
return None
|
||||
http = new_http()
|
||||
try:
|
||||
adapter.login(http, login.username, password)
|
||||
except (LoginFailed, requests.RequestException) as e:
|
||||
login.ok = False
|
||||
login.last_error = str(e) if isinstance(e, LoginFailed) else f'{adapter.name} nicht erreichbar'
|
||||
login.checked_at = timezone.now()
|
||||
login.save(update_fields=['ok', 'last_error', 'checked_at'])
|
||||
log.info('site login %s for user %s failed: %s', adapter.domain, user_id, e)
|
||||
return None
|
||||
if not login.ok:
|
||||
login.ok, login.last_error = True, ''
|
||||
login.save(update_fields=['ok', 'last_error'])
|
||||
with _lock:
|
||||
_sessions[key] = (http, time.monotonic())
|
||||
return http
|
||||
|
||||
|
||||
def has_login(user_id, url):
|
||||
from .models import SiteLogin
|
||||
adapter = adapter_for(url)
|
||||
return bool(adapter) and SiteLogin.objects.filter(user_id=user_id, domain=adapter.domain, ok=True).exists()
|
||||
|
||||
|
||||
def info(user_id, url):
|
||||
"""What the feed settings show about the login for this source's site."""
|
||||
from .models import SiteLogin
|
||||
adapter = adapter_for(url)
|
||||
if adapter is None:
|
||||
return None
|
||||
login = SiteLogin.objects.filter(user_id=user_id, domain=adapter.domain).first()
|
||||
return {'site': adapter.name, 'domain': adapter.domain,
|
||||
'username': login.username if login else '', 'ok': bool(login and login.ok),
|
||||
'error': login.last_error if login else ''}
|
||||
37
feeds/migrations/0002_item_paywalled_sitelogin.py
Normal file
37
feeds/migrations/0002_item_paywalled_sitelogin.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Generated by Django 6.1 on 2026-10-04 19:22
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('feeds', '0001_initial'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='item',
|
||||
name='paywalled',
|
||||
field=models.BooleanField(default=False),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='SiteLogin',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('domain', models.CharField(max_length=200)),
|
||||
('username', models.CharField(max_length=300)),
|
||||
('secret', models.TextField()),
|
||||
('ok', models.BooleanField(default=True)),
|
||||
('last_error', models.CharField(blank=True, max_length=300)),
|
||||
('checked_at', models.DateTimeField(blank=True, null=True)),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='feed_site_logins', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'unique_together': {('user', 'domain')},
|
||||
},
|
||||
),
|
||||
]
|
||||
|
|
@ -78,6 +78,9 @@ class Item(models.Model):
|
|||
content_full = models.BooleanField(default=False)
|
||||
# Fetching the page failed or gave nothing: don't try on every open.
|
||||
fulltext_failed = models.BooleanField(default=False)
|
||||
# The page showed only the free part (paywall): fetched again once there's
|
||||
# a subscriber login for the site (logins.py).
|
||||
paywalled = models.BooleanField(default=False)
|
||||
image_url = models.URLField(max_length=2000, blank=True)
|
||||
read_at = models.DateTimeField(null=True, blank=True)
|
||||
starred = models.BooleanField(default=False)
|
||||
|
|
@ -98,6 +101,20 @@ class Item(models.Model):
|
|||
return self.title
|
||||
|
||||
|
||||
class SiteLogin(models.Model):
|
||||
"""A subscriber login for a paywalled site (logins.py); the password encrypted."""
|
||||
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_site_logins')
|
||||
domain = models.CharField(max_length=200)
|
||||
username = models.CharField(max_length=300)
|
||||
secret = models.TextField() # the password, Fernet-encrypted
|
||||
ok = models.BooleanField(default=True)
|
||||
last_error = models.CharField(max_length=300, blank=True)
|
||||
checked_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = ('user', 'domain')
|
||||
|
||||
|
||||
class FeedSettings(models.Model):
|
||||
user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='feed_settings')
|
||||
# The last time the news briefing was seen; what came after is "Seit letztem Mal".
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from django.utils import timezone
|
|||
|
||||
from podcasts.views import FeedFetchError, _clear_error, _error_text, _fetch_feed
|
||||
|
||||
from . import fetch
|
||||
from . import fetch, logins
|
||||
from .cluster import Clusterer
|
||||
from .models import FeedSettings, Item, Source, Story
|
||||
|
||||
|
|
@ -115,12 +115,21 @@ def needs_article(item, source):
|
|||
return False
|
||||
if source.fulltext == Source.FULLTEXT_NEVER:
|
||||
return False
|
||||
if item.paywalled:
|
||||
# Only the free part came: again once there's a working subscriber login.
|
||||
return logins.has_login(item.user_id, item.url)
|
||||
return source.fulltext == Source.FULLTEXT_ALWAYS or fetch.is_teaser(item.content)
|
||||
|
||||
|
||||
def load_article(item):
|
||||
"""Fetch the article for one item now (when opened); saves it."""
|
||||
got = fetch.fetch_article(item.url)
|
||||
http = logins.session_for(item.user_id, item.url)
|
||||
got = fetch.fetch_article(item.url, http)
|
||||
if got and got[2] and http:
|
||||
# Logged in, still the paywall: the session ran out, log in anew once.
|
||||
http = logins.session_for(item.user_id, item.url, fresh=True)
|
||||
if http:
|
||||
got = fetch.fetch_article(item.url, http)
|
||||
_store_article(item, got)
|
||||
return item
|
||||
|
||||
|
|
@ -130,24 +139,46 @@ def _store_article(item, got):
|
|||
item.fulltext_failed = True
|
||||
item.save(update_fields=['fulltext_failed'])
|
||||
return
|
||||
content, image = got
|
||||
if len(fetch.plain(content)) <= len(fetch.plain(item.content)):
|
||||
content, image, paywalled = got
|
||||
longer = len(fetch.plain(content)) > len(fetch.plain(item.content))
|
||||
if paywalled:
|
||||
# The free part, if it's more than the feed's teaser; the rest waits for a login.
|
||||
item.paywalled = True
|
||||
if longer:
|
||||
item.content = content
|
||||
if not item.image_url and image:
|
||||
item.image_url = image[:2000]
|
||||
item.save(update_fields=['paywalled', 'content', 'image_url'])
|
||||
return
|
||||
if not longer:
|
||||
item.fulltext_failed = True # the feed had more
|
||||
item.save(update_fields=['fulltext_failed'])
|
||||
return
|
||||
item.content = content
|
||||
item.content_full = True
|
||||
item.paywalled = False
|
||||
if not item.image_url and image:
|
||||
item.image_url = image[:2000]
|
||||
item.save(update_fields=['content', 'content_full', 'image_url'])
|
||||
item.save(update_fields=['content', 'content_full', 'paywalled', 'image_url'])
|
||||
|
||||
|
||||
def _articles(items, deadline):
|
||||
todo = [i for i in items if needs_article(i, i.source)][:ARTICLES_PER_RUN]
|
||||
if not todo:
|
||||
return
|
||||
# Logged-in sessions for paywalled sites, one per account and site (made here, not in the threads).
|
||||
sessions = {}
|
||||
for i in todo:
|
||||
adapter = logins.adapter_for(i.url)
|
||||
if adapter and (i.user_id, adapter.domain) not in sessions:
|
||||
sessions[(i.user_id, adapter.domain)] = logins.session_for(i.user_id, i.url)
|
||||
|
||||
def http_for(i):
|
||||
adapter = logins.adapter_for(i.url)
|
||||
return sessions.get((i.user_id, adapter.domain)) if adapter else None
|
||||
|
||||
pool = ThreadPoolExecutor(ARTICLE_WORKERS)
|
||||
futures = {pool.submit(fetch.fetch_article, i.url): i for i in todo}
|
||||
futures = {pool.submit(fetch.fetch_article, i.url, http_for(i)): i for i in todo}
|
||||
pending = set(futures)
|
||||
try:
|
||||
while pending:
|
||||
|
|
@ -156,10 +187,15 @@ def _articles(items, deadline):
|
|||
break
|
||||
done, pending = wait(pending, timeout=left, return_when=FIRST_COMPLETED)
|
||||
for f in done:
|
||||
i = futures[f]
|
||||
try:
|
||||
_store_article(futures[f], f.result())
|
||||
got = f.result()
|
||||
if got and got[2] and http_for(i) is not None:
|
||||
# Logged in and still the paywall: next time with a fresh login.
|
||||
logins.forget(i.user_id, logins.adapter_for(i.url).domain)
|
||||
_store_article(i, got)
|
||||
except Exception:
|
||||
log.exception('article for item %s', futures[f].pk)
|
||||
log.exception('article for item %s', i.pk)
|
||||
finally:
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ from django.utils import timezone
|
|||
|
||||
from podcasts.views import FeedFetch
|
||||
|
||||
from . import cluster, fetch, refresh
|
||||
from .models import FeedSettings, Item, Source, Story
|
||||
from . import cluster, fetch, logins, refresh
|
||||
from .models import FeedSettings, Item, SiteLogin, Source, Story
|
||||
|
||||
|
||||
def rss(items, title='Testfeed', link='https://example.org/'):
|
||||
|
|
@ -101,7 +101,7 @@ class ApiTest(TestCase):
|
|||
|
||||
def add(self, kind, feed, url='https://example.org/feed'):
|
||||
with mock.patch('feeds.refresh._fetch_feed', return_value=fetched(feed, url)), \
|
||||
mock.patch('feeds.fetch.fetch_article', return_value=(LONG, 'https://example.org/img.jpg')):
|
||||
mock.patch('feeds.fetch.fetch_article', return_value=(LONG, 'https://example.org/img.jpg', False)):
|
||||
r = self.post('/feeds/sources/add/', {'url': url, 'kind': kind, 'category': 'Politik'})
|
||||
self.assertEqual(r.status_code, 200)
|
||||
return r.json()['source']
|
||||
|
|
@ -209,3 +209,85 @@ class ApiTest(TestCase):
|
|||
def test_settings(self):
|
||||
self.post('/feeds/settings/', {'news_keep_days': 5})
|
||||
self.assertEqual(FeedSettings.objects.get(user=self.u).news_keep_days, 5)
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, status=302, location='/', text='', url='https://www.jungewelt.de/x', ctype='text/html'):
|
||||
self.status_code = status
|
||||
self.headers = {'Location': location, 'Content-Type': ctype}
|
||||
self.text = text
|
||||
self.url = url
|
||||
|
||||
|
||||
PAYWALL = ('<html><body><article><h1>T</h1><p>' + 'Freier Anfang des Artikels. ' * 12 + '</p></article>'
|
||||
'<p>Unterstützen Sie die junge Welt mit einem Abonnement und erhalten Sie Zugang zu allen Analysen</p></body></html>')
|
||||
FULL = '<html><body><article><h1>T</h1>' + ('<p>' + 'Der ganze Artikel für Abonnenten. ' * 15 + '</p>') * 4 + '</article></body></html>'
|
||||
|
||||
|
||||
class SiteLoginTest(TestCase):
|
||||
def setUp(self):
|
||||
self.u = User.objects.create_user('tester', password='pw12345678')
|
||||
self.client.force_login(self.u)
|
||||
self.src = Source.objects.create(user=self.u, url='https://www.jungewelt.de/feeds/newsticker.rss',
|
||||
title='junge Welt', kind='news')
|
||||
self.item = Item.objects.create(source=self.src, user=self.u, kind='news', guid_hash='a', title='Artikel',
|
||||
url='https://www.jungewelt.de/artikel/1.html', summary='Anriss',
|
||||
content='<p>Anriss</p>', published=timezone.now())
|
||||
logins._sessions.clear()
|
||||
|
||||
def post(self, url, data):
|
||||
return self.client.post(url, json.dumps(data), content_type='application/json')
|
||||
|
||||
def test_encryption_round_trip_and_not_plain(self):
|
||||
token = logins.encrypt('geheim')
|
||||
self.assertNotIn('geheim', token)
|
||||
self.assertEqual(logins.decrypt(token), 'geheim')
|
||||
self.assertIsNone(logins.decrypt('kaputt'))
|
||||
|
||||
def test_paywalled_page_keeps_free_part_and_waits(self):
|
||||
with mock.patch('requests.get', return_value=FakeResponse(200, text=PAYWALL)):
|
||||
refresh.load_article(self.item)
|
||||
self.item.refresh_from_db()
|
||||
self.assertTrue(self.item.paywalled)
|
||||
self.assertFalse(self.item.content_full)
|
||||
self.assertIn('Freier Anfang', self.item.content)
|
||||
self.assertFalse(refresh.needs_article(self.item, self.src)) # no login: don't try again
|
||||
|
||||
def test_wrong_login_is_refused_and_not_stored(self):
|
||||
with mock.patch('requests.Session.post', return_value=FakeResponse(302, 'https://www.jungewelt.de/loginFailed.php?x')):
|
||||
r = self.post('/feeds/logins/', {'source': self.src.id, 'username': 'u', 'password': 'falsch'})
|
||||
self.assertEqual(r.status_code, 400)
|
||||
self.assertIn('falsch', r.json()['error'])
|
||||
self.assertFalse(SiteLogin.objects.exists())
|
||||
|
||||
def test_login_stores_encrypted_and_refetches_full_articles(self):
|
||||
Item.objects.filter(pk=self.item.pk).update(paywalled=True)
|
||||
|
||||
def login_post(session, url, **kw):
|
||||
session.cookies.set('jW', 'sitzung')
|
||||
return FakeResponse(302, 'https://www.jungewelt.de/')
|
||||
|
||||
with mock.patch('requests.Session.post', autospec=True, side_effect=login_post), \
|
||||
mock.patch('requests.Session.get', return_value=FakeResponse(200, text=FULL)):
|
||||
r = self.post('/feeds/logins/', {'source': self.src.id, 'username': 'leser', 'password': 'abo-passwort'})
|
||||
self.assertEqual(r.status_code, 200)
|
||||
self.assertEqual(r.json()['refetched'], 1)
|
||||
login = SiteLogin.objects.get(user=self.u)
|
||||
self.assertNotIn('abo-passwort', login.secret)
|
||||
self.item.refresh_from_db()
|
||||
self.assertTrue(self.item.content_full)
|
||||
self.assertFalse(self.item.paywalled)
|
||||
self.assertIn('ganze Artikel', self.item.content)
|
||||
state = self.client.get('/feeds/state/').json()
|
||||
info = state['sources'][0]['login']
|
||||
self.assertEqual((info['username'], info['ok'], info['site']), ('leser', True, 'junge Welt'))
|
||||
self.assertNotIn('abo-passwort', json.dumps(state))
|
||||
self.post('/feeds/logins/delete/', {'source': self.src.id})
|
||||
self.assertFalse(SiteLogin.objects.exists())
|
||||
|
||||
def test_sources_of_other_sites_have_no_login(self):
|
||||
other = Source.objects.create(user=self.u, url='https://blog.example/feed', title='B', kind='blog')
|
||||
state = self.client.get('/feeds/state/').json()
|
||||
by_id = {s['id']: s for s in state['sources']}
|
||||
self.assertIsNone(by_id[other.id]['login'])
|
||||
self.assertIsNotNone(by_id[self.src.id]['login'])
|
||||
|
|
|
|||
|
|
@ -21,4 +21,6 @@ urlpatterns = [
|
|||
path('items/<int:pk>/', views.item, name='feeds_item'),
|
||||
path('items/<int:pk>/star/', views.star, name='feeds_star'),
|
||||
path('settings/', views.settings, name='feeds_settings'),
|
||||
path('logins/', views.save_login, name='feeds_save_login'),
|
||||
path('logins/delete/', views.delete_login, name='feeds_delete_login'),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ Items: GET /feeds/items/<id>/ the article (fetches the page if t
|
|||
POST /feeds/items/<id>/star/ {starred}
|
||||
GET /feeds/starred/
|
||||
Settings: POST /feeds/settings/ {news_keep_days}
|
||||
Logins: POST /feeds/logins/ {source, username, password} subscriber login for its site
|
||||
POST /feeds/logins/delete/ {source}
|
||||
"""
|
||||
|
||||
import json
|
||||
|
|
@ -34,8 +36,8 @@ from django.utils.dateparse import parse_datetime
|
|||
from django.views.decorators.csrf import csrf_exempt
|
||||
from django.views.decorators.http import require_http_methods
|
||||
|
||||
from . import fetch, refresh
|
||||
from .models import Item, Source, Story
|
||||
from . import fetch, logins, refresh
|
||||
from .models import Item, SiteLogin, Source, Story
|
||||
|
||||
PAGE = 40
|
||||
BRIEFING_TOP = 8
|
||||
|
|
@ -66,6 +68,7 @@ def _source_data(s, unread=0):
|
|||
'id': s.id, 'url': s.url, 'title': s.title, 'site_url': s.site_url, 'icon_url': s.icon_url,
|
||||
'kind': s.kind, 'category': s.category, 'fulltext': s.fulltext, 'unread': unread,
|
||||
'last_refreshed_at': _iso(s.last_refreshed_at), 'last_error': s.last_error, 'error_count': s.error_count,
|
||||
'login': logins.info(s.user_id, s.url),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -79,6 +82,7 @@ def _item_data(i, content=False):
|
|||
if content:
|
||||
d['content'] = i.content
|
||||
d['content_full'] = i.content_full
|
||||
d['paywalled'] = i.paywalled
|
||||
return d
|
||||
|
||||
|
||||
|
|
@ -435,6 +439,46 @@ def star(request, pk):
|
|||
return JsonResponse({'updated': n})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
@require_http_methods(['POST'])
|
||||
def save_login(request):
|
||||
"""Check a subscriber login for the source's site and keep it; then the
|
||||
articles that came only in part are fetched again."""
|
||||
if (err := _auth(request)):
|
||||
return err
|
||||
body = _body(request) or {}
|
||||
source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
|
||||
username = (body.get('username') or '').strip()
|
||||
password = body.get('password') or ''
|
||||
if source is None or not username or not password:
|
||||
return JsonResponse({'error': 'Benutzername und Passwort nötig'}, status=400)
|
||||
try:
|
||||
adapter = logins.check_and_store(request.user, source.url, username, password)
|
||||
except logins.LoginFailed as e:
|
||||
return JsonResponse({'error': str(e)}, status=400)
|
||||
# Whatever came in part (or not at all) from this site: fetch again, the newest now.
|
||||
items = Item.objects.filter(user=request.user, gone=False, content_full=False, url__icontains=adapter.domain)
|
||||
items.update(fulltext_failed=False)
|
||||
todo = list(items.select_related('source').order_by('-published')[:40])
|
||||
refresh._articles(todo, deadline=time.monotonic() + 30)
|
||||
done = Item.objects.filter(pk__in=[i.pk for i in todo], content_full=True).count()
|
||||
return JsonResponse({'ok': True, 'login': logins.info(request.user.id, source.url), 'refetched': done})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
@require_http_methods(['POST'])
|
||||
def delete_login(request):
|
||||
if (err := _auth(request)):
|
||||
return err
|
||||
body = _body(request) or {}
|
||||
source = Source.objects.filter(user=request.user, pk=body.get('source')).first()
|
||||
adapter = logins.adapter_for(source.url) if source else None
|
||||
if adapter:
|
||||
SiteLogin.objects.filter(user=request.user, domain=adapter.domain).delete()
|
||||
logins.forget(request.user.id, adapter.domain)
|
||||
return JsonResponse({'ok': True})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
@require_http_methods(['POST'])
|
||||
def settings(request):
|
||||
|
|
|
|||
|
|
@ -21,3 +21,5 @@ piper-tts==1.7.0
|
|||
imageio-ffmpeg==0.6.0
|
||||
# The feed reader takes the article out of a news page (main text without menus, ads, comments).
|
||||
trafilatura==2.3.0
|
||||
# Subscriber logins of the feed reader are kept encrypted (feeds/logins.py).
|
||||
cryptography==50.0.1
|
||||
|
|
|
|||
|
|
@ -330,7 +330,29 @@
|
|||
'</select></label></div>' +
|
||||
'<div class="fd-buttons"><button class="btn btn-primary" type="submit">Speichern</button>' +
|
||||
'<button class="btn" type="button" data-act="cancel-edit">Abbrechen</button>' +
|
||||
'<button class="btn btn-danger" type="button" data-act="delete" data-id="' + x.id + '">Abbestellen</button></div></form>';
|
||||
'<button class="btn btn-danger" type="button" data-act="delete" data-id="' + x.id + '">Abbestellen</button></div></form>' +
|
||||
loginForm(x);
|
||||
}
|
||||
|
||||
// A paywalled site (junge Welt …): the subscriber login, kept encrypted on the
|
||||
// server, which then fetches whole articles. The password never comes back.
|
||||
function loginForm(x) {
|
||||
const l = x.login;
|
||||
if (!l) return '';
|
||||
let h = '<form class="fd-edit fd-login" data-form="login" data-id="' + x.id + '">' +
|
||||
'<h4>Abo-Anmeldung bei ' + esc(l.site) + '</h4>';
|
||||
if (l.username) {
|
||||
h += '<p>' + (l.ok ? '✓ Angemeldet als ' : '⚠ Anmeldung klappt nicht: ') + esc(l.ok ? l.username : (l.error || l.username)) + '</p>' +
|
||||
'<div class="fd-buttons"><button class="btn" type="button" data-act="login-delete" data-id="' + x.id + '">Zugangsdaten löschen</button></div>';
|
||||
if (l.ok) return h + '</form>';
|
||||
h += '<p class="fd-note">Neu eingeben:</p>';
|
||||
}
|
||||
return h +
|
||||
'<label>Benutzername <input class="search-input" name="username" autocomplete="off" autocapitalize="none" required></label>' +
|
||||
'<label>Passwort <input class="search-input" name="password" type="password" autocomplete="new-password" required></label>' +
|
||||
'<p class="fd-note">Das Passwort liegt verschlüsselt auf dem diora-Server und wird nie wieder angezeigt. ' +
|
||||
'Damit holt der Server die ganzen Artikel statt nur des freien Teils.</p>' +
|
||||
'<div class="fd-buttons"><button class="btn btn-primary" type="submit">Anmelden</button></div></form>';
|
||||
}
|
||||
|
||||
function renderOpml() {
|
||||
|
|
@ -449,7 +471,10 @@
|
|||
const content = i.content || '';
|
||||
if (i.image_url && content.indexOf(i.image_url) < 0) h += '<img class="fd-art-img" alt="" src="' + esc(i.image_url) + '">';
|
||||
h += '<div class="fd-art-body">' + (content || '<p>' + esc(i.summary) + '</p>') + '</div>';
|
||||
if (!i.content_full && i.url && (content.replace(/<[^>]+>/g, '').length < 700)) {
|
||||
if (i.paywalled) {
|
||||
h += '<p class="fd-note">Nur der freie Teil des Artikels. Mit deiner Abo-Anmeldung (⚙ → ' + esc(i.source_title) +
|
||||
') holt diora den ganzen.</p>';
|
||||
} else if (!i.content_full && i.url && (content.replace(/<[^>]+>/g, '').length < 700)) {
|
||||
h += '<p class="fd-note">Der Feed bringt nur einen Anriss, der ganze Artikel ließ sich nicht laden.</p>';
|
||||
}
|
||||
if (story && story.items.length > 1) {
|
||||
|
|
@ -591,6 +616,11 @@
|
|||
await api('/feeds/items/read/', {ids, read: true}).catch(() => {});
|
||||
return load();
|
||||
}
|
||||
if (act === 'login-delete') {
|
||||
await api('/feeds/logins/delete/', {source: parseInt(t.dataset.id, 10)}).catch(() => {});
|
||||
await loadState().catch(() => {});
|
||||
return render();
|
||||
}
|
||||
if (act === 'cancel-edit') {
|
||||
st.editing = null;
|
||||
return render();
|
||||
|
|
@ -678,6 +708,14 @@
|
|||
category: f.category.value});
|
||||
found.subscribed = true;
|
||||
await loadState();
|
||||
} else if (f.dataset.form === 'login') {
|
||||
if (btn) btn.textContent = 'Meldet an …';
|
||||
const r = await api('/feeds/logins/', {source: parseInt(f.dataset.id, 10), username: f.username.value, password: f.password.value});
|
||||
f.password.value = '';
|
||||
await loadState();
|
||||
if (typeof customAlert === 'function') {
|
||||
customAlert('Angemeldet.' + (r.refetched ? ' ' + r.refetched + ' Artikel jetzt vollständig.' : ''));
|
||||
}
|
||||
} else if (f.dataset.form === 'edit') {
|
||||
await api('/feeds/sources/' + f.dataset.id + '/', {title: f.title.value, kind: f.kind.value,
|
||||
category: f.category.value, fulltext: f.fulltext.value});
|
||||
|
|
@ -836,6 +874,8 @@
|
|||
'#tab-feeds .fd-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 8px; }',
|
||||
'#tab-feeds .fd-grid label, #tab-feeds .fd-edit > label { display: flex; flex-direction: column; gap: 4px; font-size: 12px; color: var(--nui-muted, #9a9a9a); }',
|
||||
'#tab-feeds .fd-buttons { display: flex; flex-wrap: wrap; gap: 8px; }',
|
||||
'#tab-feeds .fd-login h4 { margin: 0; font-size: 13px; }',
|
||||
'#tab-feeds .fd-login p { margin: 0; font-size: 13px; }',
|
||||
'#tab-feeds .fd-src { display: flex; flex-direction: column; gap: 2px; width: 100%; padding: 10px 0; background: none; border: none; border-bottom: 1px solid var(--border, #2a2a2a); color: inherit; font: inherit; text-align: left; cursor: pointer; }',
|
||||
'#tab-feeds .fd-opml { margin-top: 12px; display: flex; flex-direction: column; gap: 8px; }',
|
||||
'#tab-feeds .fd-opml-row { display: flex; justify-content: space-between; align-items: center; gap: 8px; }',
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* diora service worker — caches the app shell for offline use.
|
||||
*/
|
||||
|
||||
const CACHE = 'diora-v91';
|
||||
const CACHE = 'diora-v92';
|
||||
const SHELL = [
|
||||
'/static/css/app.css',
|
||||
'/static/js/app.js',
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue