Compare commits
No commits in common. "578b0e478bdfbfc8b2b3234c40102aff7ff7aaa3" and "f9b3a0ce04355015d23c4d6e2dbb3ae9a6af2518" have entirely different histories.
578b0e478b
...
f9b3a0ce04
11 changed files with 70 additions and 2570 deletions
|
|
@ -1,39 +1,30 @@
|
|||
"""Podcast chapters: Podlove Simple Chapters from the feed, Podcasting 2.0
|
||||
JSON chapters fetched from the URL a feed points to, and the CHAP frames of
|
||||
the ID3v2 tag at the start of the audio file.
|
||||
"""Podcast chapters: Podlove Simple Chapters from the feed, and Podcasting 2.0
|
||||
JSON chapters fetched from the URL a feed points to.
|
||||
|
||||
Both end up in one shape, a list sorted by start time:
|
||||
[{"start": 62.5, "title": "…", "url": "https://…", "img": "https://…"}]
|
||||
url and img only when present.
|
||||
|
||||
The JSON file and the audio file live on servers the feed's author chose, so
|
||||
fetching them is an SSRF surface like the WebDAV import, and it reuses that
|
||||
module's guards: public addresses only (checked before connecting and again
|
||||
on the connected socket), redirects followed by hand with each hop
|
||||
re-checked, and a size cap. Transcripts (transcripts.py) are fetched the same
|
||||
way, through `fetch_bytes`.
|
||||
The JSON file lives on a server the feed's author chose, so fetching it is an
|
||||
SSRF surface like the WebDAV import, and it reuses that module's guards:
|
||||
public addresses only (checked before connecting and again on the connected
|
||||
socket), redirects followed by hand with each hop re-checked, and a size cap.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import requests
|
||||
|
||||
from accounts.webdav import WebDAVError, WebDAVInputError, _assert_peer_is_safe, _read_capped, assert_safe_url
|
||||
|
||||
from . import id3
|
||||
|
||||
MAX_CHAPTERS = 500
|
||||
MAX_TITLE = 500
|
||||
MAX_URL = 1000
|
||||
CHAPTERS_TIMEOUT = 10 # seconds
|
||||
CHAPTERS_MAX_BYTES = 1024 * 1024
|
||||
CHAPTERS_MAX_REDIRECTS = 3
|
||||
ID3_MAX_BYTES = 4 * 1024 * 1024 # of the audio file's start; a larger tag is read this far
|
||||
ID3_DEADLINE = 30 # seconds for reading it, however slowly it trickles in
|
||||
ID3_CHUNK = 64 * 1024
|
||||
|
||||
_SCHEMES = ('http://', 'https://')
|
||||
|
||||
|
|
@ -123,14 +114,8 @@ class ChaptersInvalid(Exception):
|
|||
"""The chapters URL is unusable for good (blocked, missing, not chapters)."""
|
||||
|
||||
|
||||
# Generic names for the two kinds of failure, for other fetches (transcripts).
|
||||
FetchUnavailable = ChaptersUnavailable
|
||||
FetchInvalid = ChaptersInvalid
|
||||
|
||||
|
||||
def _open(url, headers):
|
||||
"""GET `url` with every redirect hop checked; the streamed response of
|
||||
the final hop, which the caller has to close.
|
||||
def fetch_json(url):
|
||||
"""Download and parse the JSON at `url`.
|
||||
|
||||
Raises ChaptersInvalid for permanent problems and ChaptersUnavailable for
|
||||
ones that may go away (network trouble, server errors).
|
||||
|
|
@ -141,7 +126,10 @@ def _open(url, headers):
|
|||
except WebDAVInputError as e:
|
||||
raise ChaptersInvalid(str(e))
|
||||
try:
|
||||
r = requests.get(url, timeout=CHAPTERS_TIMEOUT, allow_redirects=False, stream=True, headers=headers)
|
||||
r = requests.get(
|
||||
url, timeout=CHAPTERS_TIMEOUT, allow_redirects=False, stream=True,
|
||||
headers={'User-Agent': 'diora podcast chapters', 'Accept': 'application/json'},
|
||||
)
|
||||
except requests.RequestException as e:
|
||||
raise ChaptersUnavailable(e.__class__.__name__)
|
||||
try:
|
||||
|
|
@ -151,94 +139,28 @@ def _open(url, headers):
|
|||
if not location:
|
||||
raise ChaptersInvalid('redirect without a location')
|
||||
url = urljoin(url, location)
|
||||
r.close()
|
||||
continue
|
||||
if r.status_code >= 500 or r.status_code == 429:
|
||||
raise ChaptersUnavailable(f'HTTP {r.status_code}')
|
||||
if r.status_code >= 400:
|
||||
raise ChaptersInvalid(f'HTTP {r.status_code}')
|
||||
body = _read_capped(r, CHAPTERS_MAX_BYTES, lambda: ChaptersInvalid('too large'))
|
||||
except WebDAVInputError as e:
|
||||
r.close()
|
||||
raise ChaptersInvalid(str(e))
|
||||
except Exception:
|
||||
except WebDAVError as e: # transfer broke off in _read_capped
|
||||
raise ChaptersUnavailable(str(e))
|
||||
finally:
|
||||
r.close()
|
||||
raise
|
||||
return r
|
||||
try:
|
||||
return json.loads(body)
|
||||
except ValueError:
|
||||
raise ChaptersInvalid('not JSON')
|
||||
raise ChaptersInvalid('too many redirects')
|
||||
|
||||
|
||||
def fetch_bytes(url, max_bytes, accept='*/*', agent='diora podcast chapters'):
|
||||
"""The body at `url`, at most `max_bytes` of it (more is ChaptersInvalid)."""
|
||||
r = _open(url, {'User-Agent': agent, 'Accept': accept})
|
||||
try:
|
||||
return _read_capped(r, max_bytes, lambda: ChaptersInvalid('too large'))
|
||||
except WebDAVError as e: # transfer broke off in _read_capped
|
||||
raise ChaptersUnavailable(str(e))
|
||||
finally:
|
||||
r.close()
|
||||
|
||||
|
||||
def fetch_json(url):
|
||||
"""Download and parse the JSON at `url`."""
|
||||
body = fetch_bytes(url, CHAPTERS_MAX_BYTES, accept='application/json')
|
||||
try:
|
||||
return json.loads(body)
|
||||
except ValueError:
|
||||
raise ChaptersInvalid('not JSON')
|
||||
|
||||
|
||||
def fetch(url):
|
||||
"""Normalised chapters from a Podcasting 2.0 chapters URL."""
|
||||
try:
|
||||
return from_json(fetch_json(url))
|
||||
except ValueError:
|
||||
raise ChaptersInvalid('not a chapters document')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ID3v2 chapters (CHAP frames at the start of the audio file)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def read_id3_tag(url):
|
||||
"""The ID3v2 tag at the start of the file at `url` (at most ID3_MAX_BYTES
|
||||
of it), or b'' if the file doesn't start with one.
|
||||
|
||||
Asks for just that part with a Range request. Servers that ignore Range
|
||||
and send the whole file are read only as far as needed, then dropped.
|
||||
"""
|
||||
r = _open(url, {
|
||||
'User-Agent': 'diora podcast chapters',
|
||||
'Range': f'bytes=0-{ID3_MAX_BYTES - 1}',
|
||||
'Accept-Encoding': 'identity',
|
||||
})
|
||||
data = bytearray()
|
||||
need = None # bytes of the tag, known once its header is in
|
||||
deadline = time.monotonic() + ID3_DEADLINE
|
||||
try:
|
||||
for chunk in r.iter_content(chunk_size=ID3_CHUNK):
|
||||
data += chunk
|
||||
if need is None and len(data) >= id3.HEADER_SIZE:
|
||||
size = id3.tag_size(bytes(data[:id3.HEADER_SIZE]))
|
||||
if size is None:
|
||||
return b''
|
||||
need = min(size, ID3_MAX_BYTES)
|
||||
if need is not None and len(data) >= need:
|
||||
break
|
||||
if time.monotonic() > deadline:
|
||||
raise ChaptersUnavailable('too slow')
|
||||
except requests.RequestException as e:
|
||||
raise ChaptersUnavailable(e.__class__.__name__)
|
||||
finally:
|
||||
r.close()
|
||||
if need is None:
|
||||
return b'' # shorter than a tag header
|
||||
return bytes(data[:need]) # may be less if the file ended early
|
||||
|
||||
|
||||
def fetch_id3(url):
|
||||
"""Normalised chapters from the ID3v2 tag of the audio file at `url`;
|
||||
[] if it has none."""
|
||||
return _finish(
|
||||
_chapter(start_ms / 1000, title, link)
|
||||
for start_ms, _end_ms, title, link in id3.chapters(read_id3_tag(url))
|
||||
)
|
||||
|
|
|
|||
281
podcasts/id3.py
281
podcasts/id3.py
|
|
@ -1,281 +0,0 @@
|
|||
"""Chapters embedded in the ID3v2 tag at the start of an MP3 file.
|
||||
|
||||
Written from the ID3v2.3.0 and ID3v2.4.0 specifications and the ID3v2
|
||||
Chapter Frame Addendum (CHAP and CTOC frames). Only what chapters need is
|
||||
read: the tag header, an optional extended header (skipped), and the frames.
|
||||
Of those, CHAP frames give the chapters, with the title from an embedded TIT2
|
||||
frame and a link from WXXX or another W*** frame; CTOC frames, when there
|
||||
are any, give the order (which matters only for chapters starting at the
|
||||
same time, as callers sort by start). Pictures (APIC) and everything else are
|
||||
skipped.
|
||||
|
||||
Pure parsing, no I/O: the caller hands over the first bytes of the file
|
||||
(`tag_size` says how many are needed).
|
||||
"""
|
||||
|
||||
import re
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
HEADER_SIZE = 10
|
||||
MAX_FRAME_BYTES = 4 * 1024 * 1024 # a decompressed frame can't grow past this
|
||||
|
||||
_FRAME_ID = re.compile(rb'[A-Z0-9]{4}\Z')
|
||||
|
||||
# Header flags
|
||||
_UNSYNC = 0x80
|
||||
_EXTENDED = 0x40
|
||||
_FOOTER = 0x10
|
||||
|
||||
|
||||
def _synchsafe(raw):
|
||||
"""Integer from bytes that carry 7 bits each (the high bit is always 0)."""
|
||||
value = 0
|
||||
for b in raw:
|
||||
value = (value << 7) | (b & 0x7F)
|
||||
return value
|
||||
|
||||
|
||||
def tag_size(header):
|
||||
"""Bytes the ID3v2 tag at the start of a file takes up, header and footer
|
||||
included, from the file's first 10 bytes; None if there's no such tag."""
|
||||
if len(header) < HEADER_SIZE or header[:3] != b'ID3':
|
||||
return None
|
||||
major, revision, flags = header[3], header[4], header[5]
|
||||
if major == 0xFF or revision == 0xFF or any(b & 0x80 for b in header[6:10]):
|
||||
return None
|
||||
size = HEADER_SIZE + _synchsafe(header[6:10])
|
||||
if major >= 4 and flags & _FOOTER:
|
||||
size += HEADER_SIZE
|
||||
return size
|
||||
|
||||
|
||||
def _deunsync(data):
|
||||
"""Undo unsynchronisation: every $FF $00 was an $FF before."""
|
||||
return data.replace(b'\xff\x00', b'\xff')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Frames
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _plausible_frame_start(buf, at):
|
||||
"""True if a frame (or padding, or the end of `buf`) can begin at `at`."""
|
||||
if at == len(buf):
|
||||
return True
|
||||
if at > len(buf):
|
||||
return False
|
||||
head = buf[at:at + 4]
|
||||
return bool(_FRAME_ID.match(head)) or not head.strip(b'\x00')
|
||||
|
||||
|
||||
def _v24_frame_size(buf, pos, raw):
|
||||
"""Size of a v2.4 frame. The spec says synchsafe; some writers (older
|
||||
iTunes among them) store a plain integer, so where the two differ, take
|
||||
the one after which the next frame starts."""
|
||||
plain = int.from_bytes(raw, 'big')
|
||||
if any(b & 0x80 for b in raw):
|
||||
return plain # can't be synchsafe
|
||||
safe = _synchsafe(raw)
|
||||
if safe == plain or _plausible_frame_start(buf, pos + HEADER_SIZE + safe):
|
||||
return safe
|
||||
if _plausible_frame_start(buf, pos + HEADER_SIZE + plain):
|
||||
return plain
|
||||
return safe
|
||||
|
||||
|
||||
def _frame_payload(data, fmt, major, unsync_all):
|
||||
"""The frame's content after its format flags are dealt with, or None if
|
||||
it can't be read (encrypted, broken compression)."""
|
||||
if major == 3:
|
||||
compressed, encrypted = fmt & 0x80, fmt & 0x40
|
||||
# Added after the frame header, in flag order: decompressed size (4),
|
||||
# encryption method (1), group id (1).
|
||||
skip = (4 if compressed else 0) + (1 if encrypted else 0) + (1 if fmt & 0x20 else 0)
|
||||
unsync = False # v2.3 unsynchronises the whole tag, undone already
|
||||
else:
|
||||
compressed, encrypted = fmt & 0x08, fmt & 0x04
|
||||
# Group id (1), encryption method (1), data length indicator (4).
|
||||
skip = (1 if fmt & 0x40 else 0) + (1 if encrypted else 0) + (4 if fmt & 0x01 else 0)
|
||||
unsync = unsync_all or fmt & 0x02
|
||||
if encrypted:
|
||||
return None
|
||||
data = data[skip:]
|
||||
if unsync:
|
||||
data = _deunsync(data)
|
||||
if compressed:
|
||||
try:
|
||||
data = zlib.decompressobj().decompress(data, MAX_FRAME_BYTES)
|
||||
except zlib.error:
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def _frames(buf, pos, major, unsync_all=False):
|
||||
"""(frame id, content) for each complete frame in buf[pos:]."""
|
||||
while pos + HEADER_SIZE <= len(buf):
|
||||
frame_id = buf[pos:pos + 4]
|
||||
if not _FRAME_ID.match(frame_id):
|
||||
return # padding, or something we can't make sense of
|
||||
raw_size = buf[pos + 4:pos + 8]
|
||||
size = _v24_frame_size(buf, pos, raw_size) if major >= 4 else int.from_bytes(raw_size, 'big')
|
||||
fmt = buf[pos + 9]
|
||||
start = pos + HEADER_SIZE
|
||||
if start + size > len(buf):
|
||||
return # cut off (the caller read only part of a large tag)
|
||||
payload = _frame_payload(buf[start:start + size], fmt, major, unsync_all)
|
||||
if payload is not None:
|
||||
yield frame_id.decode('ascii'), payload
|
||||
pos = start + size
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Frame contents
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_CODECS = {0: 'latin-1', 1: 'utf-16', 2: 'utf-16-be', 3: 'utf-8'}
|
||||
|
||||
|
||||
def _decode(raw, encoding):
|
||||
codec = _CODECS.get(encoding)
|
||||
if codec is None:
|
||||
return ''
|
||||
if encoding in (1, 2) and len(raw) % 2:
|
||||
raw = raw[:-1]
|
||||
return raw.decode(codec, errors='replace')
|
||||
|
||||
|
||||
def _text_frame(data):
|
||||
"""The (first) string of a T*** frame."""
|
||||
if not data:
|
||||
return ''
|
||||
text = _decode(data[1:], data[0])
|
||||
# v2.4 allows several strings, separated by NUL; the first is the title.
|
||||
return text.split('\x00')[0].replace('', '').strip()
|
||||
|
||||
|
||||
def _wxxx_frame(data):
|
||||
"""The URL of a WXXX frame: encoding, description, NUL, URL (latin-1)."""
|
||||
if not data:
|
||||
return ''
|
||||
encoding, rest = data[0], data[1:]
|
||||
if encoding in (1, 2): # the description ends with a two-byte NUL
|
||||
i = 0
|
||||
while i + 1 < len(rest) and not (rest[i] == 0 and rest[i + 1] == 0):
|
||||
i += 2
|
||||
url = rest[i + 2:]
|
||||
else:
|
||||
i = rest.find(b'\x00')
|
||||
url = rest[i + 1:] if i >= 0 else b''
|
||||
return url.split(b'\x00')[0].decode('latin-1').strip()
|
||||
|
||||
|
||||
def _url_frame(data):
|
||||
"""The URL of any other W*** frame, which is nothing but the URL."""
|
||||
return data.split(b'\x00')[0].decode('latin-1').strip()
|
||||
|
||||
|
||||
def _chap(data, major):
|
||||
"""(element id, start ms, end ms, title, url) of a CHAP frame, or None."""
|
||||
nul = data.find(b'\x00')
|
||||
if nul < 0 or len(data) < nul + 17:
|
||||
return None
|
||||
element_id = data[:nul]
|
||||
start_ms, end_ms, _start_offset, _end_offset = struct.unpack('>IIII', data[nul + 1:nul + 17])
|
||||
title = url = ''
|
||||
# Embedded frames, in the same format as the tag's own. Unsynchronisation
|
||||
# of the whole tag has already been undone on `data`.
|
||||
for frame_id, payload in _frames(data, nul + 17, major):
|
||||
if frame_id == 'TIT2' and not title:
|
||||
title = _text_frame(payload)
|
||||
elif frame_id == 'WXXX' and not url:
|
||||
url = _wxxx_frame(payload)
|
||||
elif frame_id.startswith('W') and not url:
|
||||
url = _url_frame(payload)
|
||||
return element_id, start_ms, end_ms, title, url
|
||||
|
||||
|
||||
def _ctoc(data):
|
||||
"""(element id, top level?, child element ids) of a CTOC frame, or None."""
|
||||
nul = data.find(b'\x00')
|
||||
if nul < 0 or len(data) < nul + 3:
|
||||
return None
|
||||
element_id, flags, count = data[:nul], data[nul + 1], data[nul + 2]
|
||||
children, pos = [], nul + 3
|
||||
for _ in range(count):
|
||||
end = data.find(b'\x00', pos)
|
||||
if end < 0:
|
||||
break
|
||||
children.append(data[pos:end])
|
||||
pos = end + 1
|
||||
return element_id, bool(flags & 0x02), children
|
||||
|
||||
|
||||
def _toc_order(chaps, tocs):
|
||||
"""Element ids of the chapters the table of contents lists, in its order
|
||||
(top-level CTOCs first, nested ones walked depth first); [] if there's no
|
||||
usable table of contents."""
|
||||
if not tocs:
|
||||
return []
|
||||
nested = {child for _, _, children in tocs for child in children}
|
||||
roots = [t for t in tocs if t[1]] or [t for t in tocs if t[0] not in nested]
|
||||
by_id = {t[0]: t for t in tocs}
|
||||
order, seen = [], set()
|
||||
|
||||
def walk(toc):
|
||||
if toc[0] in seen:
|
||||
return # a loop
|
||||
seen.add(toc[0])
|
||||
for child in toc[2]:
|
||||
if child in chaps:
|
||||
if child not in order:
|
||||
order.append(child)
|
||||
elif child in by_id:
|
||||
walk(by_id[child])
|
||||
|
||||
for root in roots:
|
||||
walk(root)
|
||||
return order
|
||||
|
||||
|
||||
def chapters(data):
|
||||
"""[(start ms, end ms, title, url)] of the chapters in the ID3v2 tag at
|
||||
the start of `data`, in table-of-contents order where there is one, else
|
||||
in tag order.
|
||||
[] if there is no tag or it has no chapters; a tag cut short yields the
|
||||
chapters in the part that's there."""
|
||||
size = tag_size(data)
|
||||
if size is None:
|
||||
return []
|
||||
major, flags = data[3], data[5]
|
||||
if major not in (3, 4):
|
||||
return [] # v2.2 has no chapter frames
|
||||
body = data[HEADER_SIZE:min(len(data), size)]
|
||||
unsync = bool(flags & _UNSYNC)
|
||||
if major == 3 and unsync:
|
||||
body = _deunsync(body)
|
||||
pos = 0
|
||||
if flags & _EXTENDED and len(body) >= 4:
|
||||
if major == 3: # size of what follows the size field
|
||||
pos = 4 + int.from_bytes(body[:4], 'big')
|
||||
else: # synchsafe size of the whole extended header
|
||||
pos = _synchsafe(body[:4])
|
||||
|
||||
chaps, order, tocs = {}, [], []
|
||||
for frame_id, payload in _frames(body, pos, major, unsync_all=(major == 4 and unsync)):
|
||||
if frame_id == 'CHAP':
|
||||
chap = _chap(payload, major)
|
||||
if chap and chap[0] not in chaps:
|
||||
chaps[chap[0]] = chap
|
||||
order.append(chap[0])
|
||||
elif frame_id == 'CTOC':
|
||||
toc = _ctoc(payload)
|
||||
if toc:
|
||||
tocs.append(toc)
|
||||
|
||||
# The table of contents' order first; chapters it doesn't list are kept
|
||||
# (after it) rather than dropped, since writers don't always list all.
|
||||
toc = _toc_order(chaps, tocs)
|
||||
listed = set(toc)
|
||||
order = toc + [e for e in order if e not in listed]
|
||||
return [chaps[e][1:] for e in order]
|
||||
|
|
@ -1,15 +1,16 @@
|
|||
import sys
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils import timezone
|
||||
|
||||
from podcasts.models import PodcastFeed
|
||||
from podcasts.views import _in_backoff_q, _refresh_feeds
|
||||
from podcasts.views import _refresh_feeds
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Refresh podcast feeds (fetch new episodes from RSS)'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID (even when backing off)')
|
||||
parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID')
|
||||
parser.add_argument('--limit', type=int, default=0, help='Max number of feeds to refresh')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
|
|
@ -17,10 +18,6 @@ class Command(BaseCommand):
|
|||
|
||||
if options['feed_id']:
|
||||
qs = qs.filter(pk=options['feed_id'])
|
||||
else:
|
||||
# A feed failing 3+ times in a row is tried every 2, 4, 8, 16,
|
||||
# at most 24 hours instead of every hour.
|
||||
qs = qs.exclude(_in_backoff_q(timezone.now()))
|
||||
|
||||
if options['limit']:
|
||||
qs = qs[: options['limit']]
|
||||
|
|
|
|||
|
|
@ -1,74 +0,0 @@
|
|||
from django.db import migrations, models
|
||||
|
||||
|
||||
def auto_queue_to_new_episodes(apps, schema_editor):
|
||||
"""Feeds that queued their new episodes keep doing so."""
|
||||
PodcastFeed = apps.get_model('podcasts', 'PodcastFeed')
|
||||
PodcastFeed.objects.filter(auto_queue=True).update(new_episodes='queue')
|
||||
|
||||
|
||||
def new_episodes_to_auto_queue(apps, schema_editor):
|
||||
PodcastFeed = apps.get_model('podcasts', 'PodcastFeed')
|
||||
PodcastFeed.objects.filter(new_episodes='queue').update(auto_queue=True)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('podcasts', '0006_episodeprogress_favorite'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='exclude_words',
|
||||
field=models.CharField(blank=True, max_length=500),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='include_words',
|
||||
field=models.CharField(blank=True, max_length=500),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='min_duration',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='new_episodes',
|
||||
field=models.CharField(
|
||||
choices=[('inbox', 'In „Neu“'), ('queue', 'In die Warteschlange'), ('ignore', 'Nicht anzeigen')],
|
||||
default='inbox', max_length=10),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='playback_speed',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='skip_intro',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='skip_outro',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='volume',
|
||||
field=models.FloatField(default=1.0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastsettings',
|
||||
name='keep_sorted',
|
||||
field=models.CharField(
|
||||
blank=True,
|
||||
choices=[('', 'Aus'), ('new', 'Neueste zuerst'), ('old', 'Älteste zuerst'),
|
||||
('short', 'Kürzeste zuerst'), ('long', 'Längste zuerst'), ('feed', 'Nach Podcast')],
|
||||
default='', max_length=10),
|
||||
),
|
||||
migrations.RunPython(auto_queue_to_new_episodes, new_episodes_to_auto_queue),
|
||||
]
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
# Generated by Django 6.1 on 2026-10-02 12:00
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('podcasts', '0007_feed_settings'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='http_etag',
|
||||
field=models.CharField(blank=True, max_length=500),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='http_last_modified',
|
||||
field=models.CharField(blank=True, max_length=100),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='last_error',
|
||||
field=models.CharField(blank=True, max_length=500),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='last_error_at',
|
||||
field=models.DateTimeField(blank=True, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='podcastfeed',
|
||||
name='error_count',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
# Generated by Django 6.1 on 2026-10-02 07:44
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('podcasts', '0008_feed_http_cache'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='EpisodeTranscript',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('sources', models.JSONField(default=list)),
|
||||
('data', models.JSONField(blank=True, null=True)),
|
||||
('error', models.CharField(blank=True, max_length=20)),
|
||||
('fetched_at', models.DateTimeField(auto_now=True)),
|
||||
('episode', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='transcript_cache', to='podcasts.podcastepisode')),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
|
@ -3,18 +3,6 @@ from django.db import models
|
|||
|
||||
|
||||
class PodcastFeed(models.Model):
|
||||
# What happens to a new episode of this feed (one that passed the filter).
|
||||
NEW_INBOX = 'inbox'
|
||||
NEW_QUEUE = 'queue'
|
||||
NEW_IGNORE = 'ignore'
|
||||
NEW_EPISODE_ACTIONS = [
|
||||
(NEW_INBOX, 'In „Neu“'),
|
||||
(NEW_QUEUE, 'In die Warteschlange'),
|
||||
(NEW_IGNORE, 'Nicht anzeigen'),
|
||||
]
|
||||
# Volume factors: below 1 quieter, above 1 louder (the apps boost those).
|
||||
VOLUMES = (0.2, 0.5, 1.0, 1.5, 2.0, 2.5)
|
||||
|
||||
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='podcast_feeds')
|
||||
rss_url = models.URLField(max_length=1000)
|
||||
title = models.CharField(max_length=300)
|
||||
|
|
@ -24,30 +12,7 @@ class PodcastFeed(models.Model):
|
|||
link = models.URLField(max_length=1000, blank=True)
|
||||
last_refreshed_at = models.DateTimeField(null=True, blank=True)
|
||||
added_at = models.DateTimeField(auto_now_add=True)
|
||||
# Always the same as new_episodes == 'queue' (see save()); kept for the
|
||||
# web UI's toggle and older clients.
|
||||
auto_queue = models.BooleanField(default=False)
|
||||
# HTTP validators of the last full (200) response, sent back as
|
||||
# If-None-Match / If-Modified-Since so an unchanged feed costs a 304.
|
||||
http_etag = models.CharField(max_length=500, blank=True)
|
||||
http_last_modified = models.CharField(max_length=100, blank=True)
|
||||
# The last failed refresh (network, HTTP status, not a feed); cleared on
|
||||
# success. error_count counts failures in a row and drives the backoff.
|
||||
last_error = models.CharField(max_length=500, blank=True)
|
||||
last_error_at = models.DateTimeField(null=True, blank=True)
|
||||
error_count = models.PositiveIntegerField(default=0)
|
||||
|
||||
# Playback, for every device: null = the device's own speed.
|
||||
playback_speed = models.FloatField(null=True, blank=True)
|
||||
skip_intro = models.PositiveIntegerField(default=0) # seconds
|
||||
skip_outro = models.PositiveIntegerField(default=0) # seconds
|
||||
volume = models.FloatField(default=1.0) # one of VOLUMES
|
||||
# New episodes: what happens to them, and which ones count at all.
|
||||
new_episodes = models.CharField(max_length=10, choices=NEW_EPISODE_ACTIONS, default=NEW_INBOX)
|
||||
# Comma-separated words, case-insensitive, matched against the title.
|
||||
include_words = models.CharField(max_length=500, blank=True)
|
||||
exclude_words = models.CharField(max_length=500, blank=True)
|
||||
min_duration = models.PositiveIntegerField(default=0) # minutes, 0 = any
|
||||
|
||||
class Meta:
|
||||
unique_together = ('user', 'rss_url')
|
||||
|
|
@ -56,58 +21,6 @@ class PodcastFeed(models.Model):
|
|||
def __str__(self):
|
||||
return self.title
|
||||
|
||||
@classmethod
|
||||
def from_db(cls, *args, **kwargs):
|
||||
obj = super().from_db(*args, **kwargs)
|
||||
obj._remember_sync()
|
||||
return obj
|
||||
|
||||
def refresh_from_db(self, *args, **kwargs):
|
||||
super().refresh_from_db(*args, **kwargs)
|
||||
self._remember_sync()
|
||||
|
||||
def _remember_sync(self):
|
||||
self._synced = (self.__dict__.get('auto_queue'), self.__dict__.get('new_episodes'))
|
||||
|
||||
def _sync_auto_queue(self):
|
||||
"""Keep auto_queue == (new_episodes == 'queue'). Whichever of the two
|
||||
was changed since loading (or, for a new feed, set away from its
|
||||
default) wins; new_episodes if both were."""
|
||||
auto_before, new_before = getattr(self, '_synced', (False, self.NEW_INBOX))
|
||||
if self.new_episodes != new_before or self.auto_queue == auto_before:
|
||||
self.auto_queue = self.new_episodes == self.NEW_QUEUE
|
||||
elif self.auto_queue:
|
||||
self.new_episodes = self.NEW_QUEUE
|
||||
elif self.new_episodes == self.NEW_QUEUE:
|
||||
self.new_episodes = self.NEW_INBOX
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
update_fields = kwargs.get('update_fields')
|
||||
if update_fields is not None and not {'auto_queue', 'new_episodes'} & set(update_fields):
|
||||
return super().save(*args, **kwargs)
|
||||
self._sync_auto_queue()
|
||||
if update_fields is not None:
|
||||
kwargs['update_fields'] = set(update_fields) | {'auto_queue', 'new_episodes'}
|
||||
super().save(*args, **kwargs)
|
||||
self._remember_sync()
|
||||
|
||||
def matches_filter(self, title, duration_seconds):
|
||||
"""Whether a new episode passes this feed's filter: an include word
|
||||
(if any are given), no exclude word, and long enough (if known)."""
|
||||
title = (title or '').casefold()
|
||||
include = _words(self.include_words)
|
||||
if include and not any(w in title for w in include):
|
||||
return False
|
||||
if any(w in title for w in _words(self.exclude_words)):
|
||||
return False
|
||||
if self.min_duration and duration_seconds and duration_seconds < self.min_duration * 60:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _words(raw):
|
||||
return [w.strip().casefold() for w in (raw or '').split(',') if w.strip()]
|
||||
|
||||
|
||||
class PodcastEpisode(models.Model):
|
||||
feed = models.ForeignKey(PodcastFeed, on_delete=models.CASCADE, related_name='episodes')
|
||||
|
|
@ -124,11 +37,9 @@ class PodcastEpisode(models.Model):
|
|||
# Podcasting 2.0 <podcast:chapters url=…>: a JSON file, loaded on demand.
|
||||
chapters_url = models.URLField(max_length=1000, blank=True)
|
||||
# [{start: seconds, title, url?, img?}], from Podlove Simple Chapters in
|
||||
# the feed, or once fetched: the JSON at chapters_url, else (or if that
|
||||
# has none) the CHAP frames of the audio file's ID3v2 tag.
|
||||
# the feed, or the JSON at chapters_url once it has been fetched.
|
||||
chapters = models.JSONField(default=list, blank=True)
|
||||
# When chapters were fetched into `chapters` (from chapters_url or the
|
||||
# audio file); null = not yet.
|
||||
# When chapters_url was fetched into `chapters`; null = not yet.
|
||||
chapters_fetched_at = models.DateTimeField(null=True, blank=True)
|
||||
# [{url, type, language}] from <podcast:transcript>.
|
||||
transcripts = models.JSONField(default=list, blank=True)
|
||||
|
|
@ -141,27 +52,6 @@ class PodcastEpisode(models.Model):
|
|||
return self.title
|
||||
|
||||
|
||||
class EpisodeTranscript(models.Model):
|
||||
"""An episode's transcript, fetched and parsed once (transcripts.py).
|
||||
|
||||
A table of its own rather than fields on PodcastEpisode: a parsed
|
||||
transcript runs to hundreds of KB, and episode lists load whole rows.
|
||||
"""
|
||||
episode = models.OneToOneField(PodcastEpisode, on_delete=models.CASCADE, related_name='transcript_cache')
|
||||
# URLs of the transcripts it was made from, best first. When the feed
|
||||
# offers different ones, the cache is stale.
|
||||
sources = models.JSONField(default=list)
|
||||
# {"segments": [...], "source": {...}}, or null with an error.
|
||||
data = models.JSONField(null=True, blank=True)
|
||||
# '' when data is there; else why there's none for good: 'unsupported'
|
||||
# (no format with timing), 'invalid' (missing, broken, too large).
|
||||
error = models.CharField(max_length=20, blank=True)
|
||||
fetched_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
def __str__(self):
|
||||
return f'Transcript of {self.episode}'
|
||||
|
||||
|
||||
# An episode counts as played this close to its end (AntennaPod's default),
|
||||
# the same rule as the apps' "als gehört zählen".
|
||||
PLAYED_BEFORE_END_SECONDS = 30
|
||||
|
|
@ -204,8 +94,8 @@ class PodcastQueue(models.Model):
|
|||
|
||||
class PodcastSettings(models.Model):
|
||||
"""A user's podcast preferences. Where episodes go in the queue: when added
|
||||
by hand (also when played from outside the queue), and when a feed that
|
||||
queues its new episodes brings some; and whether the queue stays sorted."""
|
||||
by hand (also when played from outside the queue), and when a feed with
|
||||
auto_queue brings new ones."""
|
||||
BACK = 'back'
|
||||
FRONT = 'front'
|
||||
AFTER_CURRENT = 'after_current'
|
||||
|
|
@ -215,22 +105,10 @@ class PodcastSettings(models.Model):
|
|||
(AFTER_CURRENT, 'Nach der laufenden Folge'),
|
||||
]
|
||||
|
||||
# Keep the queue sorted: after every change the whole queue is sorted
|
||||
# again by this rule ('' = off, the user's own order).
|
||||
SORT_ORDERS = [
|
||||
('', 'Aus'),
|
||||
('new', 'Neueste zuerst'),
|
||||
('old', 'Älteste zuerst'),
|
||||
('short', 'Kürzeste zuerst'),
|
||||
('long', 'Längste zuerst'),
|
||||
('feed', 'Nach Podcast'),
|
||||
]
|
||||
|
||||
user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='podcast_settings')
|
||||
enqueue_position = models.CharField(max_length=20, choices=POSITIONS, default=BACK)
|
||||
# front = how auto-queue always worked before this setting existed
|
||||
auto_queue_position = models.CharField(max_length=20, choices=POSITIONS, default=FRONT)
|
||||
keep_sorted = models.CharField(max_length=10, choices=SORT_ORDERS, default='', blank=True)
|
||||
|
||||
@classmethod
|
||||
def for_user(cls, user):
|
||||
|
|
|
|||
1186
podcasts/tests.py
1186
podcasts/tests.py
File diff suppressed because it is too large
Load diff
|
|
@ -1,282 +0,0 @@
|
|||
"""Podcast transcripts (<podcast:transcript>): pick the best one an episode
|
||||
offers, fetch it with the same guards as the chapters, and turn it into
|
||||
readable segments:
|
||||
|
||||
{"segments": [{"start": 1.5, "end": 9.25, "speaker": "Anna" | None, "text": "…"}],
|
||||
"source": {"url": "…", "type": "text/vtt", "language": "de"}}
|
||||
|
||||
Formats: Podcasting 2.0 JSON, WebVTT and SubRip (SRT). HTML transcripts have
|
||||
no timing and aren't supported. Cues are often a few words each; consecutive
|
||||
ones of the same speaker are merged up to a sentence end or about 30 s.
|
||||
"""
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
|
||||
from . import chapters as fetching
|
||||
|
||||
MAX_BYTES = 2 * 1024 * 1024
|
||||
MAX_SEGMENTS = 20_000
|
||||
MAX_TEXT = 5_000 # characters of one segment
|
||||
MAX_SPEAKER = 200
|
||||
|
||||
# Merging
|
||||
MERGE_MAX_SECONDS = 30 # a segment never spans more than this
|
||||
MERGE_MIN_SECONDS = 5 # ... and ends at a sentence end once it's this long
|
||||
MERGE_MAX_GAP = 5 # silence that always starts a new segment
|
||||
|
||||
JSON, VTT, SRT, HTML = 'json', 'vtt', 'srt', 'html'
|
||||
_KIND_RANK = {JSON: 0, VTT: 1, SRT: 2}
|
||||
_ACCEPT = {JSON: 'application/json', VTT: 'text/vtt', SRT: 'application/x-subrip'}
|
||||
|
||||
|
||||
class Unsupported(Exception):
|
||||
"""The transcript is in a format without timing (HTML) or an unknown one."""
|
||||
|
||||
|
||||
def kind(transcript):
|
||||
"""json, vtt, srt, html or '' (unknown) from the type, else the URL."""
|
||||
t = (transcript.get('type') or '').lower().split(';')[0].strip()
|
||||
if 'json' in t:
|
||||
return JSON
|
||||
if 'vtt' in t:
|
||||
return VTT
|
||||
if 'srt' in t or 'subrip' in t:
|
||||
return SRT
|
||||
if 'html' in t:
|
||||
return HTML
|
||||
path = (transcript.get('url') or '').lower().split('?')[0].split('#')[0]
|
||||
for ext, k in (('.json', JSON), ('.vtt', VTT), ('.srt', SRT), ('.html', HTML), ('.htm', HTML)):
|
||||
if path.endswith(ext):
|
||||
return k
|
||||
return ''
|
||||
|
||||
|
||||
def _language_rank(language):
|
||||
lang = (language or '').lower().split('-')[0].split('_')[0]
|
||||
if lang in ('de', ''): # no language given: most likely the episode's own
|
||||
return 0
|
||||
if lang == 'en':
|
||||
return 1
|
||||
return 2
|
||||
|
||||
|
||||
def candidates(transcripts):
|
||||
"""The episode's transcripts in a format we read, best first: German (or
|
||||
no language given) before English before others, then JSON before VTT
|
||||
before SRT."""
|
||||
usable = [(i, t) for i, t in enumerate(transcripts or []) if kind(t) in _KIND_RANK]
|
||||
usable.sort(key=lambda it: (_language_rank(it[1].get('language')), _KIND_RANK[kind(it[1])], it[0]))
|
||||
return [t for _, t in usable]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parsers: each returns fragments [(start, end, speaker, text)]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _decode(body):
|
||||
if body.startswith((b'\xff\xfe', b'\xfe\xff')):
|
||||
return body.decode('utf-16', errors='replace')
|
||||
try:
|
||||
return body.decode('utf-8-sig')
|
||||
except UnicodeDecodeError:
|
||||
return body.decode('cp1252', errors='replace')
|
||||
|
||||
|
||||
def _clean(text):
|
||||
text = re.sub(r'<[^>]*>', ' ', text)
|
||||
return ' '.join(html.unescape(text).split())
|
||||
|
||||
|
||||
def _seconds(value):
|
||||
try:
|
||||
value = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if value != value or value < 0 or value == float('inf'):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def parse_json(text):
|
||||
"""Podcasting 2.0 JSON: {"segments": [{"startTime", "endTime", "speaker", "body"}]}."""
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except ValueError:
|
||||
raise fetching.FetchInvalid('not JSON')
|
||||
segments = data.get('segments') if isinstance(data, dict) else None
|
||||
if not isinstance(segments, list):
|
||||
raise fetching.FetchInvalid('no segments list')
|
||||
out = []
|
||||
for s in segments:
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
start = _seconds(s.get('startTime'))
|
||||
if start is None:
|
||||
continue
|
||||
end = _seconds(s.get('endTime'))
|
||||
speaker = s.get('speaker')
|
||||
speaker = ' '.join(speaker.split()) if isinstance(speaker, str) else None
|
||||
body = s.get('body')
|
||||
out.append((start, end, speaker or None, _clean(body) if isinstance(body, str) else ''))
|
||||
return out
|
||||
|
||||
|
||||
# 00:01:02.500 / 01:02.500 (VTT), 00:01:02,500 (SRT)
|
||||
_TIME = r'(?:(\d+):)?(\d{1,2}):(\d{1,2})(?:[.,](\d{1,3}))?'
|
||||
_TIMING = re.compile(r'^\s*' + _TIME + r'\s*-->\s*' + _TIME + r'(?:\s|$)')
|
||||
_VOICE = re.compile(r'<v(?:\.[^\s>]*)?(?:\s+([^>]*))?>', re.IGNORECASE)
|
||||
|
||||
|
||||
def _time(hours, minutes, seconds, fraction):
|
||||
return int(hours or 0) * 3600 + int(minutes) * 60 + int(seconds) + (int(fraction.ljust(3, '0')) / 1000 if fraction else 0)
|
||||
|
||||
|
||||
def _cue_fragments(start, end, lines):
|
||||
"""Split a cue's text at its voice tags (<v Anna>…) into fragments."""
|
||||
text = '\n'.join(lines)
|
||||
out = []
|
||||
speaker, pos = None, 0
|
||||
for m in _VOICE.finditer(text):
|
||||
out.append((start, end, speaker, _clean(text[pos:m.start()])))
|
||||
speaker = ' '.join((m.group(1) or '').split())[:MAX_SPEAKER] or None
|
||||
pos = m.end()
|
||||
out.append((start, end, speaker, _clean(text[pos:])))
|
||||
return [f for f in out if f[3]]
|
||||
|
||||
|
||||
def parse_cues(text):
|
||||
"""WebVTT or SRT. Lines before a cue's timing line (VTT cue ids, SRT
|
||||
numbers) are dropped, as is everything outside cues (the WEBVTT header,
|
||||
NOTE, STYLE and REGION blocks)."""
|
||||
cues, current = [], None
|
||||
for line in text.replace('\r\n', '\n').replace('\r', '\n').split('\n'):
|
||||
m = _TIMING.match(line)
|
||||
if m:
|
||||
if current is not None and current[2] and current[2][-1].strip().isdigit():
|
||||
current[2].pop() # an SRT number with no blank line before it
|
||||
g = m.groups()
|
||||
current = (_time(*g[:4]), _time(*g[4:]), [])
|
||||
cues.append(current)
|
||||
elif not line.strip():
|
||||
current = None
|
||||
elif current is not None:
|
||||
current[2].append(line)
|
||||
out = []
|
||||
for start, end, lines in cues:
|
||||
out.extend(_cue_fragments(start, end, lines))
|
||||
return out
|
||||
|
||||
|
||||
def parse(body, declared_kind):
|
||||
"""Fragments from a downloaded transcript; the content decides the format
|
||||
when it's obvious, the declared type otherwise."""
|
||||
text = _decode(body)
|
||||
head = text.lstrip()[:200].lower()
|
||||
if head.startswith(('<!doctype', '<html', '<?xml', '<body', '<p', '<div')):
|
||||
raise Unsupported('HTML')
|
||||
if head.startswith(('{', '[')):
|
||||
return parse_json(text)
|
||||
if head.startswith('webvtt') or declared_kind in (VTT, SRT):
|
||||
return parse_cues(text)
|
||||
if declared_kind == JSON:
|
||||
return parse_json(text)
|
||||
raise Unsupported('unknown format')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Merging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_SENTENCE_END = re.compile(r'[.!?…]["\'”’»)\]]*$')
|
||||
|
||||
|
||||
def merge(fragments):
|
||||
"""Readable segments from cue fragments: consecutive ones of the same
|
||||
speaker are joined until a sentence ends (once the segment is at least
|
||||
MERGE_MIN_SECONDS long), MERGE_MAX_SECONDS are reached, or there's a
|
||||
pause of more than MERGE_MAX_GAP."""
|
||||
frags = sorted((f for f in fragments if f[3]), key=lambda f: f[0])[:MAX_SEGMENTS * 10]
|
||||
# Fragments without a usable end run until the next one starts.
|
||||
fixed = []
|
||||
for i, (start, end, speaker, text) in enumerate(frags):
|
||||
if end is None or end <= start:
|
||||
nxt = frags[i + 1][0] if i + 1 < len(frags) else start
|
||||
end = max(nxt, start)
|
||||
fixed.append((start, end, speaker, text))
|
||||
|
||||
segments = []
|
||||
for start, end, speaker, text in fixed:
|
||||
last = segments[-1] if segments else None
|
||||
if (
|
||||
last is not None
|
||||
and last['speaker'] == speaker
|
||||
and start - last['end'] <= MERGE_MAX_GAP
|
||||
and end - last['start'] <= MERGE_MAX_SECONDS
|
||||
and len(last['text']) + len(text) < MAX_TEXT
|
||||
and not (_SENTENCE_END.search(last['text']) and last['end'] - last['start'] >= MERGE_MIN_SECONDS)
|
||||
):
|
||||
last['text'] += ' ' + text
|
||||
last['end'] = max(last['end'], end)
|
||||
continue
|
||||
if len(segments) >= MAX_SEGMENTS:
|
||||
break
|
||||
segments.append({'start': start, 'end': end, 'speaker': speaker, 'text': text[:MAX_TEXT]})
|
||||
for s in segments:
|
||||
s['start'], s['end'] = round(s['start'], 3), round(s['end'], 3)
|
||||
return segments
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fetching
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def fetch(transcript):
|
||||
"""The parsed transcript {"segments", "source"} from one of the episode's
|
||||
<podcast:transcript> entries. Raises Unsupported, FetchInvalid or
|
||||
FetchUnavailable."""
|
||||
k = kind(transcript)
|
||||
if k not in _KIND_RANK:
|
||||
raise Unsupported(transcript.get('type') or 'unknown type')
|
||||
body = fetching.fetch_bytes(
|
||||
transcript['url'], MAX_BYTES, accept=f'{_ACCEPT[k]}, */*;q=0.1', agent='diora podcast transcripts',
|
||||
)
|
||||
segments = merge(parse(body, k))
|
||||
if not segments:
|
||||
raise fetching.FetchInvalid('no segments')
|
||||
return {
|
||||
'segments': segments,
|
||||
'source': {
|
||||
'url': transcript['url'],
|
||||
'type': transcript.get('type') or '',
|
||||
'language': transcript.get('language') or '',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load(transcripts):
|
||||
"""(parsed transcript, final) from the best candidate that works.
|
||||
|
||||
final is False when a better candidate failed for now only, so the
|
||||
result shouldn't be kept for good. Raises FetchUnavailable when nothing
|
||||
worked and something may work later, Unsupported when no candidate is in
|
||||
a usable format, FetchInvalid otherwise.
|
||||
"""
|
||||
retry_later = False
|
||||
unsupported_only = True
|
||||
for t in candidates(transcripts):
|
||||
try:
|
||||
return fetch(t), not retry_later
|
||||
except fetching.FetchUnavailable:
|
||||
retry_later = True
|
||||
unsupported_only = False
|
||||
except Unsupported:
|
||||
pass
|
||||
except fetching.FetchInvalid:
|
||||
unsupported_only = False
|
||||
if retry_later:
|
||||
raise fetching.FetchUnavailable('try again later')
|
||||
if unsupported_only:
|
||||
raise Unsupported('no usable transcript')
|
||||
raise fetching.FetchInvalid('no usable transcript')
|
||||
|
|
@ -12,11 +12,9 @@ urlpatterns = [
|
|||
path('state/', views.podcast_state, name='podcast_state'),
|
||||
path('feeds/<int:pk>/remove/', views.remove_feed, name='podcast_remove_feed'),
|
||||
path('feeds/<int:pk>/set-auto-queue/', views.set_auto_queue, name='podcast_set_auto_queue'),
|
||||
path('feeds/<int:pk>/settings/', views.feed_settings, name='podcast_feed_settings'),
|
||||
path('feeds/<int:pk>/episodes/', views.feed_episodes, name='podcast_feed_episodes'),
|
||||
path('episodes/', views.episode_list, name='podcast_episode_list'),
|
||||
path('episodes/<int:pk>/', views.episode_detail, name='podcast_episode_detail'),
|
||||
path('episodes/<int:pk>/transcript/', views.episode_transcript, name='podcast_episode_transcript'),
|
||||
path('episodes/<int:pk>/favorite/', views.set_favorite, name='podcast_set_favorite'),
|
||||
path('stats/', views.podcast_stats, name='podcast_stats'),
|
||||
path('queue/', views.queue_get, name='podcast_queue_get'),
|
||||
|
|
|
|||
|
|
@ -2,14 +2,12 @@ import json
|
|||
import logging
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import feedparser
|
||||
import requests
|
||||
from django.conf import settings
|
||||
from django.db import IntegrityError, transaction
|
||||
from django.db.models import (
|
||||
Case, Count, Exists, F, FilteredRelation, IntegerField, Max, OuterRef, Q, Sum, Value, When,
|
||||
)
|
||||
|
|
@ -21,10 +19,7 @@ from django.views.decorators.http import require_http_methods
|
|||
|
||||
from . import chapters as chapters_mod
|
||||
from . import feedext
|
||||
from . import transcripts as transcripts_mod
|
||||
from .models import (
|
||||
EpisodeProgress, EpisodeTranscript, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings, counts_as_played,
|
||||
)
|
||||
from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings, counts_as_played
|
||||
from .shownotes import extract_timestamps, sanitize_html
|
||||
|
||||
feedext.install()
|
||||
|
|
@ -43,7 +38,6 @@ MAX_TRANSCRIPTS = 10
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
QUEUE_POSITIONS = {key for key, _ in PodcastSettings.POSITIONS}
|
||||
KEEP_SORTED = {key for key, _ in PodcastSettings.SORT_ORDERS}
|
||||
|
||||
|
||||
def _queue_order(user):
|
||||
|
|
@ -108,58 +102,9 @@ def _enqueue(user, episode_ids, where, current_hint=None):
|
|||
)
|
||||
_write_queue_order(user, queued[:at] + new + queued[at:])
|
||||
_leave_inbox(user, new)
|
||||
_keep_sorted(user)
|
||||
return new
|
||||
|
||||
|
||||
def _sort_key(rule):
|
||||
"""Sort key for one queue row ({pub_date, duration, position, feed_title})
|
||||
by a PodcastSettings.SORT_ORDERS rule. Ties keep their current order
|
||||
(the sort is stable)."""
|
||||
def ts(row):
|
||||
return row['pub_date'].timestamp() if row['pub_date'] else None
|
||||
|
||||
def remaining(row):
|
||||
return max((row['duration'] or 0) - (row['position'] or 0), 0)
|
||||
|
||||
if rule == 'new': # no date: last
|
||||
return lambda r: (ts(r) is None, -(ts(r) or 0))
|
||||
if rule == 'old': # no date: last as well
|
||||
return lambda r: (ts(r) is None, ts(r) or 0)
|
||||
if rule == 'short':
|
||||
return remaining
|
||||
if rule == 'long':
|
||||
return lambda r: -remaining(r)
|
||||
if rule == 'feed': # by podcast title, then newest first
|
||||
return lambda r: ((r['feed_title'] or '').casefold(), ts(r) is None, -(ts(r) or 0))
|
||||
return None
|
||||
|
||||
|
||||
def _keep_sorted(user, rule=None):
|
||||
"""Sort the user's whole queue by their keep_sorted rule (or `rule`), if
|
||||
there is one. Returns True if it is kept sorted."""
|
||||
if rule is None:
|
||||
rule = PodcastSettings.objects.filter(user=user).values_list('keep_sorted', flat=True).first() or ''
|
||||
key = _sort_key(rule)
|
||||
if key is None:
|
||||
return False
|
||||
rows = list(
|
||||
PodcastQueue.objects.filter(user=user)
|
||||
.order_by('position', 'added_at', 'id')
|
||||
.values('episode_id', pub_date=F('episode__pub_date'), duration=F('episode__duration_seconds'),
|
||||
feed_title=F('episode__feed__title'))
|
||||
)
|
||||
positions = dict(
|
||||
EpisodeProgress.objects.filter(user=user, episode_id__in=[r['episode_id'] for r in rows])
|
||||
.values_list('episode_id', 'position_seconds')
|
||||
)
|
||||
for r in rows:
|
||||
r['position'] = positions.get(r['episode_id'], 0)
|
||||
rows.sort(key=key)
|
||||
_write_queue_order(user, [r['episode_id'] for r in rows])
|
||||
return True
|
||||
|
||||
|
||||
def _leave_inbox(user, episode_ids):
|
||||
"""Queued or started episodes are no longer new (AntennaPod's rule): out
|
||||
of the inbox, without counting as played."""
|
||||
|
|
@ -234,10 +179,8 @@ def _entry_transcripts(entry):
|
|||
def _apply_extras(episode, description, chapters_url, psc_chapters, transcripts):
|
||||
"""Copy what the feed says now onto `episode`; True if anything changed.
|
||||
|
||||
Inline (Podlove) chapters win. Without them, fetched chapters (from
|
||||
chapters_url, or from the audio file when there's no URL) stay cached for
|
||||
as long as the URL stays the same; inline chapters going away means a
|
||||
fresh fetch.
|
||||
Inline (Podlove) chapters win. Without them, chapters fetched from
|
||||
chapters_url stay cached for as long as the URL stays the same.
|
||||
"""
|
||||
changed = []
|
||||
if episode.description != description:
|
||||
|
|
@ -247,8 +190,8 @@ def _apply_extras(episode, description, chapters_url, psc_chapters, transcripts)
|
|||
episode.transcripts = transcripts
|
||||
changed.append('transcripts')
|
||||
if psc_chapters:
|
||||
chapters, fetched_at = psc_chapters, None
|
||||
elif chapters_url == episode.chapters_url and episode.chapters_fetched_at:
|
||||
chapters, fetched_at = psc_chapters, episode.chapters_fetched_at
|
||||
elif chapters_url and chapters_url == episode.chapters_url and episode.chapters_fetched_at:
|
||||
chapters, fetched_at = episode.chapters, episode.chapters_fetched_at
|
||||
else:
|
||||
chapters, fetched_at = [], None
|
||||
|
|
@ -271,141 +214,11 @@ EXTRA_FIELDS = ['description', 'transcripts', 'chapters_url', 'chapters', 'chapt
|
|||
# Shared feed refresh helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class FeedFetchError(Exception):
|
||||
"""A feed that could not be fetched or is not a feed; str() is the short
|
||||
message stored in PodcastFeed.last_error ("HTTP 404", "timeout", …)."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class FeedFetch:
|
||||
"""What _fetch_feed got. status 200: `parsed` is the feedparser result and
|
||||
etag/last_modified are the new validators ('' when the server sent none).
|
||||
status 304: unchanged since the validators that were sent, parsed is None.
|
||||
final_url is where the content came from after all redirects;
|
||||
permanent_url is the requested URL after following only the 301/308 hops
|
||||
at the start of the redirect chain (== the requested URL without one)."""
|
||||
status: int
|
||||
parsed: object = None
|
||||
etag: str = ''
|
||||
last_modified: str = ''
|
||||
final_url: str = ''
|
||||
permanent_url: str = ''
|
||||
|
||||
|
||||
PERMANENT_REDIRECTS = (301, 308)
|
||||
|
||||
|
||||
def _fetch_feed(url, etag='', last_modified=''):
|
||||
"""Download and parse a feed (with a timeout, unlike feedparser's own
|
||||
fetch). With etag/last_modified it is a conditional GET. Returns a
|
||||
FeedFetch; raises FeedFetchError when there is nothing usable."""
|
||||
headers = {'User-Agent': 'diora podcast refresh'}
|
||||
if etag:
|
||||
headers['If-None-Match'] = etag
|
||||
if last_modified:
|
||||
headers['If-Modified-Since'] = last_modified
|
||||
try:
|
||||
r = requests.get(url, timeout=FEED_TIMEOUT, headers=headers)
|
||||
except requests.Timeout:
|
||||
raise FeedFetchError('timeout')
|
||||
except requests.TooManyRedirects:
|
||||
raise FeedFetchError('too many redirects')
|
||||
except requests.ConnectionError:
|
||||
raise FeedFetchError('connection failed')
|
||||
except requests.RequestException as e:
|
||||
raise FeedFetchError(f'request failed ({type(e).__name__})')
|
||||
|
||||
# Each hop's target is the next hop's URL (the last one's is r.url).
|
||||
permanent_url = url
|
||||
targets = [h.url for h in r.history[1:]] + [r.url]
|
||||
for hop, target in zip(r.history, targets):
|
||||
if hop.status_code not in PERMANENT_REDIRECTS:
|
||||
break
|
||||
permanent_url = target
|
||||
|
||||
if r.status_code == 304:
|
||||
if not (etag or last_modified):
|
||||
raise FeedFetchError('HTTP 304') # nothing to be unchanged from
|
||||
return FeedFetch(304, None, etag, last_modified, r.url, permanent_url)
|
||||
if r.status_code >= 400:
|
||||
raise FeedFetchError(f'HTTP {r.status_code}')
|
||||
|
||||
parsed = feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
|
||||
if parsed.get('bozo') and not parsed.entries and not parsed.feed.get('title'):
|
||||
raise FeedFetchError('not a feed')
|
||||
new_etag = r.headers.get('ETag', '')
|
||||
new_modified = r.headers.get('Last-Modified', '')
|
||||
return FeedFetch(
|
||||
r.status_code, parsed,
|
||||
# A validator too long to store is dropped (a truncated one never matches).
|
||||
new_etag if len(new_etag) <= 500 else '',
|
||||
new_modified if len(new_modified) <= 100 else '',
|
||||
r.url, permanent_url,
|
||||
)
|
||||
|
||||
|
||||
def _validators(feeds):
|
||||
"""If-None-Match / If-Modified-Since for one download that serves all
|
||||
`feeds` (several users' subscriptions of the same URL): only when every
|
||||
one of them was fetched before and stored the same validators, because a
|
||||
304 then skips all of them. Otherwise {} (a full response)."""
|
||||
pairs = {(f.http_etag, f.http_last_modified) for f in feeds}
|
||||
if len(pairs) != 1 or any(f.last_refreshed_at is None for f in feeds):
|
||||
return {}
|
||||
etag, last_modified = pairs.pop()
|
||||
if not (etag or last_modified):
|
||||
return {}
|
||||
return {'etag': etag, 'last_modified': last_modified}
|
||||
|
||||
|
||||
def _error_text(e):
|
||||
if isinstance(e, FeedFetchError):
|
||||
return str(e)[:500]
|
||||
return (str(e) or type(e).__name__)[:500]
|
||||
|
||||
|
||||
def _record_error(feed_obj, e):
|
||||
feed_obj.last_error = _error_text(e)
|
||||
feed_obj.last_error_at = timezone.now()
|
||||
feed_obj.error_count += 1
|
||||
try:
|
||||
feed_obj.save(update_fields=['last_error', 'last_error_at', 'error_count'])
|
||||
except Exception:
|
||||
log.exception('could not store the refresh error of feed %s', feed_obj.pk)
|
||||
|
||||
|
||||
ERROR_FIELDS = ['last_error', 'last_error_at', 'error_count']
|
||||
|
||||
|
||||
def _clear_error(feed_obj):
|
||||
feed_obj.last_error = ''
|
||||
feed_obj.last_error_at = None
|
||||
feed_obj.error_count = 0
|
||||
|
||||
|
||||
BACKOFF_AFTER = 3 # failures in a row before the hourly job backs off
|
||||
BACKOFF_MAX_HOURS = 24
|
||||
|
||||
|
||||
def _backoff_hours(error_count):
|
||||
"""Hours between hourly-job attempts for a feed failing error_count
|
||||
times in a row: 2, 4, 8, 16, 24 … from the third failure on."""
|
||||
if error_count < BACKOFF_AFTER:
|
||||
return 0
|
||||
return min(2 ** (error_count - 2), BACKOFF_MAX_HOURS)
|
||||
|
||||
|
||||
def _in_backoff_q(now=None):
|
||||
"""Q for feeds that failed too often to be tried again yet."""
|
||||
now = now or timezone.now()
|
||||
n = BACKOFF_AFTER
|
||||
while _backoff_hours(n) < BACKOFF_MAX_HOURS:
|
||||
n += 1
|
||||
# From n failures on the wait is capped; below, one term per count.
|
||||
q = Q(error_count__gte=n, last_error_at__gt=now - timedelta(hours=BACKOFF_MAX_HOURS))
|
||||
for count in range(BACKOFF_AFTER, n):
|
||||
q |= Q(error_count=count, last_error_at__gt=now - timedelta(hours=_backoff_hours(count)))
|
||||
return q
|
||||
def _fetch_feed(url):
|
||||
"""Download and parse a feed (with a timeout, unlike feedparser's own fetch)."""
|
||||
r = requests.get(url, timeout=FEED_TIMEOUT, headers={'User-Agent': 'diora podcast refresh'})
|
||||
r.raise_for_status()
|
||||
return feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
|
||||
|
||||
|
||||
def _refresh_feeds(feeds, chunk=32):
|
||||
|
|
@ -416,31 +229,27 @@ def _refresh_feeds(feeds, chunk=32):
|
|||
refreshed = failed = new = 0
|
||||
for start in range(0, len(feeds), chunk):
|
||||
part = feeds[start:start + chunk]
|
||||
by_url = {}
|
||||
for f in part:
|
||||
by_url.setdefault(f.rss_url, []).append(f)
|
||||
urls = sorted({f.rss_url for f in part})
|
||||
results = {}
|
||||
|
||||
def fetch(url):
|
||||
try:
|
||||
return url, _fetch_feed(url, **_validators(by_url[url])), None
|
||||
return url, _fetch_feed(url), None
|
||||
except Exception as e: # network, HTTP or parser trouble: report and go on
|
||||
return url, None, e
|
||||
|
||||
urls = sorted(by_url)
|
||||
with ThreadPoolExecutor(max_workers=min(FEED_WORKERS, len(urls) or 1)) as pool:
|
||||
for url, fetched, err in pool.map(fetch, urls):
|
||||
results[url] = (fetched, err)
|
||||
for url, parsed, err in pool.map(fetch, urls):
|
||||
results[url] = (parsed, err)
|
||||
|
||||
for feed in part:
|
||||
fetched, err = results[feed.rss_url]
|
||||
parsed, err = results[feed.rss_url]
|
||||
if err is not None:
|
||||
failed += 1
|
||||
log.warning('refreshing %s failed: %s', feed.rss_url, err)
|
||||
_record_error(feed, err)
|
||||
continue
|
||||
try:
|
||||
new += _refresh_feed(feed, fetched)
|
||||
new += _refresh_feed(feed, parsed)
|
||||
refreshed += 1
|
||||
except Exception as e:
|
||||
failed += 1
|
||||
|
|
@ -448,47 +257,10 @@ def _refresh_feeds(feeds, chunk=32):
|
|||
return refreshed, failed, new
|
||||
|
||||
|
||||
def _refresh_feed(feed_obj, fetched=None):
|
||||
"""Fetch rss_url (unless `fetched`, a FeedFetch, is given) and upsert
|
||||
episodes. Returns count of new episodes. A failure is stored on the feed
|
||||
(last_error, error_count) and raised again."""
|
||||
if fetched is None:
|
||||
try:
|
||||
fetched = _fetch_feed(feed_obj.rss_url, **_validators([feed_obj]))
|
||||
except Exception as e:
|
||||
_record_error(feed_obj, e)
|
||||
raise
|
||||
try:
|
||||
return _apply_feed(feed_obj, fetched)
|
||||
except Exception as e:
|
||||
_record_error(feed_obj, e)
|
||||
raise
|
||||
|
||||
|
||||
def _move_to_permanent_url(feed_obj, new_url):
|
||||
"""The feed answered with a permanent redirect: subscribe to the new URL,
|
||||
unless the user already has that one too (then nothing changes)."""
|
||||
if not new_url or new_url == feed_obj.rss_url or len(new_url) > 1000:
|
||||
return
|
||||
try:
|
||||
with transaction.atomic():
|
||||
PodcastFeed.objects.filter(pk=feed_obj.pk).update(rss_url=new_url)
|
||||
except IntegrityError: # the user's other subscription has it
|
||||
return
|
||||
log.info('feed %s moved permanently: %s -> %s', feed_obj.pk, feed_obj.rss_url, new_url)
|
||||
feed_obj.rss_url = new_url
|
||||
|
||||
|
||||
def _apply_feed(feed_obj, fetched):
|
||||
now = timezone.now()
|
||||
if fetched.status == 304:
|
||||
# Unchanged since the last full download: nothing to parse or write.
|
||||
feed_obj.last_refreshed_at = now
|
||||
_clear_error(feed_obj)
|
||||
feed_obj.save(update_fields=['last_refreshed_at', *ERROR_FIELDS])
|
||||
return 0
|
||||
|
||||
parsed = fetched.parsed
|
||||
def _refresh_feed(feed_obj, parsed=None):
|
||||
"""Parse rss_url and upsert episodes. Returns count of new episodes."""
|
||||
if parsed is None:
|
||||
parsed = _fetch_feed(feed_obj.rss_url)
|
||||
channel = parsed.feed
|
||||
# The first fetch of a new subscription brings the whole back catalogue.
|
||||
first_fetch = feed_obj.last_refreshed_at is None
|
||||
|
|
@ -506,7 +278,7 @@ def _apply_feed(feed_obj, fetched):
|
|||
or image.get('url', '')
|
||||
)[:1000]
|
||||
|
||||
feed_obj.last_refreshed_at = now
|
||||
feed_obj.last_refreshed_at = timezone.now()
|
||||
feed_obj.save()
|
||||
|
||||
max_ep = getattr(settings, 'PODCAST_MAX_EPISODES_PER_FEED', 200)
|
||||
|
|
@ -609,20 +381,7 @@ def _apply_feed(feed_obj, fetched):
|
|||
)
|
||||
new_episodes_list = newest_first[:NEW_ON_SUBSCRIBE]
|
||||
|
||||
# The feed's own rules for its new episodes: those its filter turns down,
|
||||
# and all of them for 'ignore', go straight out of the inbox (not played);
|
||||
# 'queue' puts them in the queue (which also takes them out of the inbox);
|
||||
# 'inbox' leaves them there.
|
||||
if new_episodes_list:
|
||||
unwanted = [e for e in new_episodes_list if not feed_obj.matches_filter(e.title, e.duration_seconds)]
|
||||
if feed_obj.new_episodes == PodcastFeed.NEW_IGNORE:
|
||||
unwanted = new_episodes_list
|
||||
if unwanted:
|
||||
gone = {e.id for e in unwanted}
|
||||
_leave_inbox(feed_obj.user, list(gone))
|
||||
new_episodes_list = [e for e in new_episodes_list if e.id not in gone]
|
||||
|
||||
if feed_obj.new_episodes == PodcastFeed.NEW_QUEUE and new_episodes_list:
|
||||
if feed_obj.auto_queue and new_episodes_list:
|
||||
where = PodcastSettings.for_user(feed_obj.user).auto_queue_position
|
||||
# Newest first in front of what's there; oldest first when appended,
|
||||
# so the end of the queue plays in the order the episodes came out.
|
||||
|
|
@ -634,13 +393,6 @@ def _apply_feed(feed_obj, fetched):
|
|||
ordered = newest_first if where != PodcastSettings.BACK else newest_first[::-1]
|
||||
_enqueue(feed_obj.user, [e.id for e in ordered], where)
|
||||
|
||||
# Only now, with every episode written: the validators (a 304 next time
|
||||
# skips all of the above) and a clean error state.
|
||||
feed_obj.http_etag = fetched.etag
|
||||
feed_obj.http_last_modified = fetched.last_modified
|
||||
_clear_error(feed_obj)
|
||||
feed_obj.save(update_fields=['http_etag', 'http_last_modified', *ERROR_FIELDS])
|
||||
_move_to_permanent_url(feed_obj, fetched.permanent_url)
|
||||
return new_count
|
||||
|
||||
|
||||
|
|
@ -687,30 +439,16 @@ def feed_list(request):
|
|||
return JsonResponse({'feeds': _feeds_data(request.user)})
|
||||
|
||||
|
||||
FEED_SETTING_FIELDS = (
|
||||
'playback_speed', 'skip_intro', 'skip_outro', 'volume',
|
||||
'new_episodes', 'include_words', 'exclude_words', 'min_duration', 'auto_queue',
|
||||
)
|
||||
|
||||
|
||||
def _feed_settings(feed):
|
||||
return {'feed_id': feed.id, **{name: getattr(feed, name) for name in FEED_SETTING_FIELDS}}
|
||||
|
||||
|
||||
def _feeds_data(user):
|
||||
feeds = list(
|
||||
user.podcast_feeds
|
||||
.annotate(latest_episode_at=Max('episodes__pub_date'))
|
||||
.values('id', 'title', 'artwork_url', 'rss_url', 'last_refreshed_at', 'author', 'added_at',
|
||||
'latest_episode_at', 'last_error', 'last_error_at', 'error_count', *FEED_SETTING_FIELDS)
|
||||
.values('id', 'title', 'artwork_url', 'rss_url', 'last_refreshed_at', 'author', 'added_at', 'auto_queue', 'latest_episode_at')
|
||||
)
|
||||
|
||||
for f in feeds:
|
||||
f['feed_id'] = f['id'] # as in feeds/<id>/settings/
|
||||
if f['last_refreshed_at']:
|
||||
f['last_refreshed_at'] = f['last_refreshed_at'].isoformat()
|
||||
if f['last_error_at']:
|
||||
f['last_error_at'] = f['last_error_at'].isoformat()
|
||||
if f['added_at']:
|
||||
f['added_at'] = f['added_at'].isoformat()
|
||||
if f['latest_episode_at']:
|
||||
|
|
@ -796,99 +534,11 @@ def set_auto_queue(request, pk):
|
|||
feed = PodcastFeed.objects.get(pk=pk, user=request.user)
|
||||
except PodcastFeed.DoesNotExist:
|
||||
return JsonResponse({'error': 'not found'}, status=404)
|
||||
# On: new episodes go to the queue; off: back to the inbox (a feed set
|
||||
# to 'ignore' stays so). PodcastFeed.save() sets new_episodes to match.
|
||||
feed.auto_queue = bool(body.get('auto_queue', not feed.auto_queue))
|
||||
feed.save(update_fields=['auto_queue'])
|
||||
return JsonResponse({'ok': True, 'auto_queue': feed.auto_queue})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-feed settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _number(value):
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool)
|
||||
|
||||
|
||||
def _whole(value, low, high):
|
||||
if not _number(value) or value != int(value) or not low <= value <= high:
|
||||
raise ValueError
|
||||
return int(value)
|
||||
|
||||
|
||||
def _speed(value):
|
||||
if value is None:
|
||||
return None
|
||||
if not _number(value) or not 0.5 <= value <= 3.0:
|
||||
raise ValueError
|
||||
return float(value)
|
||||
|
||||
|
||||
def _volume(value):
|
||||
if not _number(value) or float(value) not in PodcastFeed.VOLUMES:
|
||||
raise ValueError
|
||||
return float(value)
|
||||
|
||||
|
||||
def _new_episodes(value):
|
||||
if value not in {key for key, _ in PodcastFeed.NEW_EPISODE_ACTIONS}:
|
||||
raise ValueError
|
||||
return value
|
||||
|
||||
|
||||
def _word_list(value):
|
||||
if value is None:
|
||||
return ''
|
||||
if not isinstance(value, str) or len(value.strip()) > 500:
|
||||
raise ValueError
|
||||
return value.strip()
|
||||
|
||||
|
||||
FEED_SETTING_PARSERS = {
|
||||
'playback_speed': _speed,
|
||||
'skip_intro': lambda v: _whole(v, 0, 600),
|
||||
'skip_outro': lambda v: _whole(v, 0, 600),
|
||||
'volume': _volume,
|
||||
'new_episodes': _new_episodes,
|
||||
'include_words': _word_list,
|
||||
'exclude_words': _word_list,
|
||||
'min_duration': lambda v: _whole(v, 0, 600),
|
||||
}
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
@require_http_methods(['POST'])
|
||||
def feed_settings(request, pk):
|
||||
"""Change some of one feed's settings (any of FEED_SETTING_PARSERS);
|
||||
returns all of them. Nothing is changed if one value is invalid."""
|
||||
if not request.user.is_authenticated:
|
||||
return JsonResponse({'error': 'authentication required'}, status=401)
|
||||
try:
|
||||
body = json.loads(request.body or b'{}')
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return JsonResponse({'error': 'invalid JSON'}, status=400)
|
||||
if not isinstance(body, dict):
|
||||
return JsonResponse({'error': 'invalid JSON'}, status=400)
|
||||
try:
|
||||
feed = PodcastFeed.objects.get(pk=pk, user=request.user)
|
||||
except PodcastFeed.DoesNotExist:
|
||||
return JsonResponse({'error': 'not found'}, status=404)
|
||||
|
||||
changes = {}
|
||||
for name, parse in FEED_SETTING_PARSERS.items():
|
||||
if name in body:
|
||||
try:
|
||||
changes[name] = parse(body[name])
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
return JsonResponse({'error': f'invalid {name}', 'field': name}, status=400)
|
||||
if changes:
|
||||
for name, value in changes.items():
|
||||
setattr(feed, name, value)
|
||||
feed.save(update_fields=list(changes))
|
||||
return JsonResponse(_feed_settings(feed))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feed episodes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -952,29 +602,20 @@ def feed_episodes(request, pk):
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_chapters(episode):
|
||||
"""Fetch the chapters the feed doesn't carry inline, once, and cache them
|
||||
on `episode`: the JSON at chapters_url, else (no URL, or nothing there)
|
||||
the CHAP frames in the ID3v2 tag at the start of the audio file.
|
||||
"""Fetch the JSON chapters of `episode` once and cache them on it.
|
||||
|
||||
Problems that may go away (timeouts, server errors) aren't cached, so the
|
||||
next request tries again; anything else is remembered as "no chapters".
|
||||
"""
|
||||
url = episode.chapters_url
|
||||
chapters = []
|
||||
try:
|
||||
if url:
|
||||
try:
|
||||
chapters = chapters_mod.fetch(url)
|
||||
except chapters_mod.ChaptersInvalid as e:
|
||||
log.info('chapters of episode %s unusable: %s', episode.id, e)
|
||||
if not chapters and episode.audio_url:
|
||||
try:
|
||||
chapters = chapters_mod.fetch_id3(episode.audio_url)
|
||||
except chapters_mod.ChaptersInvalid as e:
|
||||
log.info('ID3 chapters of episode %s unusable: %s', episode.id, e)
|
||||
chapters = chapters_mod.fetch(url)
|
||||
except chapters_mod.ChaptersUnavailable as e:
|
||||
log.info('chapters of episode %s not available: %s', episode.id, e)
|
||||
return []
|
||||
except chapters_mod.ChaptersInvalid as e:
|
||||
log.info('chapters of episode %s unusable: %s', episode.id, e)
|
||||
chapters = []
|
||||
now = timezone.now()
|
||||
# Only if a refresh hasn't changed the URL in the meantime.
|
||||
PodcastEpisode.objects.filter(pk=episode.pk, chapters_url=url).update(
|
||||
|
|
@ -995,7 +636,7 @@ def episode_detail(request, pk):
|
|||
return JsonResponse({'error': 'not found'}, status=404)
|
||||
|
||||
chapters = ep.chapters
|
||||
if not chapters and ep.chapters_fetched_at is None:
|
||||
if not chapters and ep.chapters_url and ep.chapters_fetched_at is None:
|
||||
chapters = _load_chapters(ep)
|
||||
|
||||
progress = EpisodeProgress.objects.filter(user=request.user, episode=ep).first()
|
||||
|
|
@ -1006,7 +647,6 @@ def episode_detail(request, pk):
|
|||
'timestamps': extract_timestamps(ep.description, ep.duration_seconds),
|
||||
'chapters': chapters,
|
||||
'transcripts': ep.transcripts,
|
||||
'has_transcript': _has_transcript(ep),
|
||||
'feed': {'id': ep.feed.id, 'title': ep.feed.title, 'artwork_url': ep.feed.artwork_url},
|
||||
'audio_url': ep.audio_url,
|
||||
'artwork_url': ep.artwork_url,
|
||||
|
|
@ -1022,65 +662,6 @@ def episode_detail(request, pk):
|
|||
})
|
||||
|
||||
|
||||
def _transcript_sources(ep):
|
||||
return [t['url'] for t in transcripts_mod.candidates(ep.transcripts)]
|
||||
|
||||
|
||||
def _has_transcript(ep):
|
||||
"""True if the episode offers a transcript we can read, and it hasn't
|
||||
turned out to be unusable."""
|
||||
sources = _transcript_sources(ep)
|
||||
if not sources:
|
||||
return False
|
||||
cached = EpisodeTranscript.objects.filter(episode=ep).values_list('sources', 'error').first()
|
||||
return not (cached and cached[0] == sources and cached[1])
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
@require_http_methods(['GET'])
|
||||
def episode_transcript(request, pk):
|
||||
"""The episode's transcript as {"segments": [{start, end, speaker, text}],
|
||||
"source": {url, type, language}}, fetched and parsed on first use, then
|
||||
kept. 404 {"error": "no transcript" | "unsupported" | "invalid"} when
|
||||
there's none to show; 502 {"error": "unavailable"} when it couldn't be
|
||||
loaded right now."""
|
||||
if not request.user.is_authenticated:
|
||||
return JsonResponse({'error': 'authentication required'}, status=401)
|
||||
try:
|
||||
ep = PodcastEpisode.objects.get(pk=pk, feed__user=request.user)
|
||||
except PodcastEpisode.DoesNotExist:
|
||||
return JsonResponse({'error': 'not found'}, status=404)
|
||||
if not ep.transcripts:
|
||||
return JsonResponse({'error': 'no transcript'}, status=404)
|
||||
sources = _transcript_sources(ep)
|
||||
if not sources:
|
||||
return JsonResponse({'error': 'unsupported'}, status=404)
|
||||
|
||||
cached = EpisodeTranscript.objects.filter(episode=ep).first()
|
||||
if cached is not None and cached.sources == sources:
|
||||
data, error = cached.data, cached.error
|
||||
else:
|
||||
try:
|
||||
data, final = transcripts_mod.load(ep.transcripts)
|
||||
error = ''
|
||||
except chapters_mod.FetchUnavailable as e:
|
||||
log.info('transcript of episode %s not available: %s', ep.id, e)
|
||||
return JsonResponse({'error': 'unavailable'}, status=502)
|
||||
except transcripts_mod.Unsupported as e:
|
||||
log.info('transcript of episode %s unsupported: %s', ep.id, e)
|
||||
data, error, final = None, 'unsupported', True
|
||||
except chapters_mod.FetchInvalid as e:
|
||||
log.info('transcript of episode %s unusable: %s', ep.id, e)
|
||||
data, error, final = None, 'invalid', True
|
||||
if final:
|
||||
EpisodeTranscript.objects.update_or_create(
|
||||
episode=ep, defaults={'sources': sources, 'data': data, 'error': error},
|
||||
)
|
||||
if error:
|
||||
return JsonResponse({'error': error}, status=404)
|
||||
return JsonResponse(data)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Favorites
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -1376,8 +957,7 @@ def refresh_feed_now(request):
|
|||
def refresh_all_feeds(request):
|
||||
"""Refresh all of the user's feeds on the server in one go. Feeds the
|
||||
hourly job (or an earlier call) refreshed within `max_age` seconds are
|
||||
skipped. Clients then fetch the result once (state/, inbox/ …).
|
||||
The user asked, so feeds the hourly job backs off from are tried too."""
|
||||
skipped. Clients then fetch the result once (state/, inbox/ …)."""
|
||||
if not request.user.is_authenticated:
|
||||
return JsonResponse({'error': 'authentication required'}, status=401)
|
||||
try:
|
||||
|
|
@ -1409,8 +989,7 @@ def podcast_state(request):
|
|||
'feeds': _feeds_data(request.user),
|
||||
'queue': _queue_data(request.user),
|
||||
'inbox': _inbox_data(request.user, 0, limit, with_description=False),
|
||||
'settings': {'enqueue_position': s.enqueue_position, 'auto_queue_position': s.auto_queue_position,
|
||||
'keep_sorted': s.keep_sorted},
|
||||
'settings': {'enqueue_position': s.enqueue_position, 'auto_queue_position': s.auto_queue_position},
|
||||
'server_time': timezone.now().isoformat(),
|
||||
})
|
||||
|
||||
|
|
@ -1522,8 +1101,6 @@ def queue_reorder(request):
|
|||
order = body.get('order', []) # list of episode ids
|
||||
for pos, ep_id in enumerate(order):
|
||||
PodcastQueue.objects.filter(user=request.user, episode_id=ep_id).update(position=pos)
|
||||
# A queue that is kept sorted goes straight back into its order.
|
||||
_keep_sorted(request.user)
|
||||
|
||||
return JsonResponse({'ok': True})
|
||||
|
||||
|
|
@ -1546,21 +1123,13 @@ def podcast_settings(request):
|
|||
return JsonResponse({'error': f'unknown {name}'}, status=400)
|
||||
setattr(s, name, body[name])
|
||||
fields.append(name)
|
||||
if 'keep_sorted' in body:
|
||||
if body['keep_sorted'] not in KEEP_SORTED:
|
||||
return JsonResponse({'error': 'unknown keep_sorted'}, status=400)
|
||||
s.keep_sorted = body['keep_sorted']
|
||||
fields.append('keep_sorted')
|
||||
if fields:
|
||||
s.save(update_fields=fields)
|
||||
if 'keep_sorted' in fields:
|
||||
_keep_sorted(request.user, s.keep_sorted) # sorted right away
|
||||
elif request.method != 'GET':
|
||||
return JsonResponse({'error': 'method not allowed'}, status=405)
|
||||
return JsonResponse({
|
||||
'enqueue_position': s.enqueue_position,
|
||||
'auto_queue_position': s.auto_queue_position,
|
||||
'keep_sorted': s.keep_sorted,
|
||||
'positions': [{'value': k, 'label': v} for k, v in PodcastSettings.POSITIONS],
|
||||
})
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue