11 changed files with 70 additions and 2570 deletions
--- a/podcasts/chapters.py
+++ b/podcasts/chapters.py
@ -1,39 +1,30 @@
-"""Podcast chapters: Podlove Simple Chapters from the feed, Podcasting 2.0
+"""Podcast chapters: Podlove Simple Chapters from the feed, and Podcasting 2.0
-JSON chapters fetched from the URL a feed points to, and the CHAP frames of
+JSON chapters fetched from the URL a feed points to.
 the ID3v2 tag at the start of the audio file.
 Both end up in one shape, a list sorted by start time:
    [{"start": 62.5, "title": "…", "url": "https://…", "img": "https://…"}]
 url and img only when present.
-The JSON file and the audio file live on servers the feed's author chose, so
+The JSON file lives on a server the feed's author chose, so fetching it is an
-fetching them is an SSRF surface like the WebDAV import, and it reuses that
+SSRF surface like the WebDAV import, and it reuses that module's guards:
-module's guards: public addresses only (checked before connecting and again
+public addresses only (checked before connecting and again on the connected
-on the connected socket), redirects followed by hand with each hop
+socket), redirects followed by hand with each hop re-checked, and a size cap.
 re-checked, and a size cap. Transcripts (transcripts.py) are fetched the same
 way, through `fetch_bytes`.
 """
 import json
 import re
 import time
 from urllib.parse import urljoin
 import requests
 from accounts.webdav import WebDAVError, WebDAVInputError, _assert_peer_is_safe, _read_capped, assert_safe_url
 from . import id3
 MAX_CHAPTERS = 500
 MAX_TITLE = 500
 MAX_URL = 1000
 CHAPTERS_TIMEOUT = 10          # seconds
 CHAPTERS_MAX_BYTES = 1024 * 1024
 CHAPTERS_MAX_REDIRECTS = 3
 ID3_MAX_BYTES = 4 * 1024 * 1024  # of the audio file's start; a larger tag is read this far
 ID3_DEADLINE = 30              # seconds for reading it, however slowly it trickles in
 ID3_CHUNK = 64 * 1024
 _SCHEMES = ('http://', 'https://')
@ -123,14 +114,8 @@ class ChaptersInvalid(Exception):
    """The chapters URL is unusable for good (blocked, missing, not chapters)."""
-# Generic names for the two kinds of failure, for other fetches (transcripts).
+def fetch_json(url):
-FetchUnavailable = ChaptersUnavailable
+    """Download and parse the JSON at `url`.
 FetchInvalid = ChaptersInvalid
 def _open(url, headers):
    """GET `url` with every redirect hop checked; the streamed response of
    the final hop, which the caller has to close.
    Raises ChaptersInvalid for permanent problems and ChaptersUnavailable for
    ones that may go away (network trouble, server errors).
@ -141,7 +126,10 @@ def _open(url, headers):
        except WebDAVInputError as e:
            raise ChaptersInvalid(str(e))
        try:
-            r = requests.get(url, timeout=CHAPTERS_TIMEOUT, allow_redirects=False, stream=True, headers=headers)
+            r = requests.get(
                url, timeout=CHAPTERS_TIMEOUT, allow_redirects=False, stream=True,
                headers={'User-Agent': 'diora podcast chapters', 'Accept': 'application/json'},
            )
        except requests.RequestException as e:
            raise ChaptersUnavailable(e.__class__.__name__)
        try:
@ -151,40 +139,23 @@ def _open(url, headers):
                if not location:
                    raise ChaptersInvalid('redirect without a location')
                url = urljoin(url, location)
                r.close()
                continue
            if r.status_code >= 500 or r.status_code == 429:
                raise ChaptersUnavailable(f'HTTP {r.status_code}')
            if r.status_code >= 400:
                raise ChaptersInvalid(f'HTTP {r.status_code}')
            body = _read_capped(r, CHAPTERS_MAX_BYTES, lambda: ChaptersInvalid('too large'))
        except WebDAVInputError as e:
            r.close()
            raise ChaptersInvalid(str(e))
        except Exception:
            r.close()
            raise
        return r
    raise ChaptersInvalid('too many redirects')
 def fetch_bytes(url, max_bytes, accept='*/*', agent='diora podcast chapters'):
    """The body at `url`, at most `max_bytes` of it (more is ChaptersInvalid)."""
    r = _open(url, {'User-Agent': agent, 'Accept': accept})
    try:
        return _read_capped(r, max_bytes, lambda: ChaptersInvalid('too large'))
        except WebDAVError as e:  # transfer broke off in _read_capped
            raise ChaptersUnavailable(str(e))
        finally:
            r.close()
 def fetch_json(url):
    """Download and parse the JSON at `url`."""
    body = fetch_bytes(url, CHAPTERS_MAX_BYTES, accept='application/json')
        try:
            return json.loads(body)
        except ValueError:
            raise ChaptersInvalid('not JSON')
    raise ChaptersInvalid('too many redirects')
 def fetch(url):
@ -193,52 +164,3 @@ def fetch(url):
        return from_json(fetch_json(url))
    except ValueError:
        raise ChaptersInvalid('not a chapters document')
 # ---------------------------------------------------------------------------
 # ID3v2 chapters (CHAP frames at the start of the audio file)
 # ---------------------------------------------------------------------------
 def read_id3_tag(url):
    """The ID3v2 tag at the start of the file at `url` (at most ID3_MAX_BYTES
    of it), or b'' if the file doesn't start with one.
    Asks for just that part with a Range request. Servers that ignore Range
    and send the whole file are read only as far as needed, then dropped.
    """
    r = _open(url, {
        'User-Agent': 'diora podcast chapters',
        'Range': f'bytes=0-{ID3_MAX_BYTES - 1}',
        'Accept-Encoding': 'identity',
    })
    data = bytearray()
    need = None  # bytes of the tag, known once its header is in
    deadline = time.monotonic() + ID3_DEADLINE
    try:
        for chunk in r.iter_content(chunk_size=ID3_CHUNK):
            data += chunk
            if need is None and len(data) >= id3.HEADER_SIZE:
                size = id3.tag_size(bytes(data[:id3.HEADER_SIZE]))
                if size is None:
                    return b''
                need = min(size, ID3_MAX_BYTES)
            if need is not None and len(data) >= need:
                break
            if time.monotonic() > deadline:
                raise ChaptersUnavailable('too slow')
    except requests.RequestException as e:
        raise ChaptersUnavailable(e.__class__.__name__)
    finally:
        r.close()
    if need is None:
        return b''  # shorter than a tag header
    return bytes(data[:need])  # may be less if the file ended early
 def fetch_id3(url):
    """Normalised chapters from the ID3v2 tag of the audio file at `url`;
    [] if it has none."""
    return _finish(
        _chapter(start_ms / 1000, title, link)
        for start_ms, _end_ms, title, link in id3.chapters(read_id3_tag(url))
    )
--- a/podcasts/id3.py
+++ b/podcasts/id3.py
@ -1,281 +0,0 @@
 """Chapters embedded in the ID3v2 tag at the start of an MP3 file.
 Written from the ID3v2.3.0 and ID3v2.4.0 specifications and the ID3v2
 Chapter Frame Addendum (CHAP and CTOC frames). Only what chapters need is
 read: the tag header, an optional extended header (skipped), and the frames.
 Of those, CHAP frames give the chapters, with the title from an embedded TIT2
 frame and a link from WXXX or another W*** frame; CTOC frames, when there
 are any, give the order (which matters only for chapters starting at the
 same time, as callers sort by start). Pictures (APIC) and everything else are
 skipped.
 Pure parsing, no I/O: the caller hands over the first bytes of the file
 (`tag_size` says how many are needed).
 """
 import re
 import struct
 import zlib
 HEADER_SIZE = 10
 MAX_FRAME_BYTES = 4 * 1024 * 1024   # a decompressed frame can't grow past this
 _FRAME_ID = re.compile(rb'[A-Z0-9]{4}\Z')
 # Header flags
 _UNSYNC = 0x80
 _EXTENDED = 0x40
 _FOOTER = 0x10
 def _synchsafe(raw):
    """Integer from bytes that carry 7 bits each (the high bit is always 0)."""
    value = 0
    for b in raw:
        value = (value << 7) | (b & 0x7F)
    return value
 def tag_size(header):
    """Bytes the ID3v2 tag at the start of a file takes up, header and footer
    included, from the file's first 10 bytes; None if there's no such tag."""
    if len(header) < HEADER_SIZE or header[:3] != b'ID3':
        return None
    major, revision, flags = header[3], header[4], header[5]
    if major == 0xFF or revision == 0xFF or any(b & 0x80 for b in header[6:10]):
        return None
    size = HEADER_SIZE + _synchsafe(header[6:10])
    if major >= 4 and flags & _FOOTER:
        size += HEADER_SIZE
    return size
 def _deunsync(data):
    """Undo unsynchronisation: every $FF $00 was an $FF before."""
    return data.replace(b'\xff\x00', b'\xff')
 # ---------------------------------------------------------------------------
 # Frames
 # ---------------------------------------------------------------------------
 def _plausible_frame_start(buf, at):
    """True if a frame (or padding, or the end of `buf`) can begin at `at`."""
    if at == len(buf):
        return True
    if at > len(buf):
        return False
    head = buf[at:at + 4]
    return bool(_FRAME_ID.match(head)) or not head.strip(b'\x00')
 def _v24_frame_size(buf, pos, raw):
    """Size of a v2.4 frame. The spec says synchsafe; some writers (older
    iTunes among them) store a plain integer, so where the two differ, take
    the one after which the next frame starts."""
    plain = int.from_bytes(raw, 'big')
    if any(b & 0x80 for b in raw):
        return plain  # can't be synchsafe
    safe = _synchsafe(raw)
    if safe == plain or _plausible_frame_start(buf, pos + HEADER_SIZE + safe):
        return safe
    if _plausible_frame_start(buf, pos + HEADER_SIZE + plain):
        return plain
    return safe
 def _frame_payload(data, fmt, major, unsync_all):
    """The frame's content after its format flags are dealt with, or None if
    it can't be read (encrypted, broken compression)."""
    if major == 3:
        compressed, encrypted = fmt & 0x80, fmt & 0x40
        # Added after the frame header, in flag order: decompressed size (4),
        # encryption method (1), group id (1).
        skip = (4 if compressed else 0) + (1 if encrypted else 0) + (1 if fmt & 0x20 else 0)
        unsync = False  # v2.3 unsynchronises the whole tag, undone already
    else:
        compressed, encrypted = fmt & 0x08, fmt & 0x04
        # Group id (1), encryption method (1), data length indicator (4).
        skip = (1 if fmt & 0x40 else 0) + (1 if encrypted else 0) + (4 if fmt & 0x01 else 0)
        unsync = unsync_all or fmt & 0x02
    if encrypted:
        return None
    data = data[skip:]
    if unsync:
        data = _deunsync(data)
    if compressed:
        try:
            data = zlib.decompressobj().decompress(data, MAX_FRAME_BYTES)
        except zlib.error:
            return None
    return data
 def _frames(buf, pos, major, unsync_all=False):
    """(frame id, content) for each complete frame in buf[pos:]."""
    while pos + HEADER_SIZE <= len(buf):
        frame_id = buf[pos:pos + 4]
        if not _FRAME_ID.match(frame_id):
            return  # padding, or something we can't make sense of
        raw_size = buf[pos + 4:pos + 8]
        size = _v24_frame_size(buf, pos, raw_size) if major >= 4 else int.from_bytes(raw_size, 'big')
        fmt = buf[pos + 9]
        start = pos + HEADER_SIZE
        if start + size > len(buf):
            return  # cut off (the caller read only part of a large tag)
        payload = _frame_payload(buf[start:start + size], fmt, major, unsync_all)
        if payload is not None:
            yield frame_id.decode('ascii'), payload
        pos = start + size
 # ---------------------------------------------------------------------------
 # Frame contents
 # ---------------------------------------------------------------------------
 _CODECS = {0: 'latin-1', 1: 'utf-16', 2: 'utf-16-be', 3: 'utf-8'}
 def _decode(raw, encoding):
    codec = _CODECS.get(encoding)
    if codec is None:
        return ''
    if encoding in (1, 2) and len(raw) % 2:
        raw = raw[:-1]
    return raw.decode(codec, errors='replace')
 def _text_frame(data):
    """The (first) string of a T*** frame."""
    if not data:
        return ''
    text = _decode(data[1:], data[0])
    # v2.4 allows several strings, separated by NUL; the first is the title.
    return text.split('\x00')[0].replace('', '').strip()
 def _wxxx_frame(data):
    """The URL of a WXXX frame: encoding, description, NUL, URL (latin-1)."""
    if not data:
        return ''
    encoding, rest = data[0], data[1:]
    if encoding in (1, 2):  # the description ends with a two-byte NUL
        i = 0
        while i + 1 < len(rest) and not (rest[i] == 0 and rest[i + 1] == 0):
            i += 2
        url = rest[i + 2:]
    else:
        i = rest.find(b'\x00')
        url = rest[i + 1:] if i >= 0 else b''
    return url.split(b'\x00')[0].decode('latin-1').strip()
 def _url_frame(data):
    """The URL of any other W*** frame, which is nothing but the URL."""
    return data.split(b'\x00')[0].decode('latin-1').strip()
 def _chap(data, major):
    """(element id, start ms, end ms, title, url) of a CHAP frame, or None."""
    nul = data.find(b'\x00')
    if nul < 0 or len(data) < nul + 17:
        return None
    element_id = data[:nul]
    start_ms, end_ms, _start_offset, _end_offset = struct.unpack('>IIII', data[nul + 1:nul + 17])
    title = url = ''
    # Embedded frames, in the same format as the tag's own. Unsynchronisation
    # of the whole tag has already been undone on `data`.
    for frame_id, payload in _frames(data, nul + 17, major):
        if frame_id == 'TIT2' and not title:
            title = _text_frame(payload)
        elif frame_id == 'WXXX' and not url:
            url = _wxxx_frame(payload)
        elif frame_id.startswith('W') and not url:
            url = _url_frame(payload)
    return element_id, start_ms, end_ms, title, url
 def _ctoc(data):
    """(element id, top level?, child element ids) of a CTOC frame, or None."""
    nul = data.find(b'\x00')
    if nul < 0 or len(data) < nul + 3:
        return None
    element_id, flags, count = data[:nul], data[nul + 1], data[nul + 2]
    children, pos = [], nul + 3
    for _ in range(count):
        end = data.find(b'\x00', pos)
        if end < 0:
            break
        children.append(data[pos:end])
        pos = end + 1
    return element_id, bool(flags & 0x02), children
 def _toc_order(chaps, tocs):
    """Element ids of the chapters the table of contents lists, in its order
    (top-level CTOCs first, nested ones walked depth first); [] if there's no
    usable table of contents."""
    if not tocs:
        return []
    nested = {child for _, _, children in tocs for child in children}
    roots = [t for t in tocs if t[1]] or [t for t in tocs if t[0] not in nested]
    by_id = {t[0]: t for t in tocs}
    order, seen = [], set()
    def walk(toc):
        if toc[0] in seen:
            return  # a loop
        seen.add(toc[0])
        for child in toc[2]:
            if child in chaps:
                if child not in order:
                    order.append(child)
            elif child in by_id:
                walk(by_id[child])
    for root in roots:
        walk(root)
    return order
 def chapters(data):
    """[(start ms, end ms, title, url)] of the chapters in the ID3v2 tag at
    the start of `data`, in table-of-contents order where there is one, else
    in tag order.
    [] if there is no tag or it has no chapters; a tag cut short yields the
    chapters in the part that's there."""
    size = tag_size(data)
    if size is None:
        return []
    major, flags = data[3], data[5]
    if major not in (3, 4):
        return []  # v2.2 has no chapter frames
    body = data[HEADER_SIZE:min(len(data), size)]
    unsync = bool(flags & _UNSYNC)
    if major == 3 and unsync:
        body = _deunsync(body)
    pos = 0
    if flags & _EXTENDED and len(body) >= 4:
        if major == 3:  # size of what follows the size field
            pos = 4 + int.from_bytes(body[:4], 'big')
        else:  # synchsafe size of the whole extended header
            pos = _synchsafe(body[:4])
    chaps, order, tocs = {}, [], []
    for frame_id, payload in _frames(body, pos, major, unsync_all=(major == 4 and unsync)):
        if frame_id == 'CHAP':
            chap = _chap(payload, major)
            if chap and chap[0] not in chaps:
                chaps[chap[0]] = chap
                order.append(chap[0])
        elif frame_id == 'CTOC':
            toc = _ctoc(payload)
            if toc:
                tocs.append(toc)
    # The table of contents' order first; chapters it doesn't list are kept
    # (after it) rather than dropped, since writers don't always list all.
    toc = _toc_order(chaps, tocs)
    listed = set(toc)
    order = toc + [e for e in order if e not in listed]
    return [chaps[e][1:] for e in order]
--- a/podcasts/management/commands/refresh_feeds.py
+++ b/podcasts/management/commands/refresh_feeds.py
@ -1,15 +1,16 @@
 import sys
 from django.core.management.base import BaseCommand
 from django.utils import timezone
 from podcasts.models import PodcastFeed
-from podcasts.views import _in_backoff_q, _refresh_feeds
+from podcasts.views import _refresh_feeds
 class Command(BaseCommand):
    help = 'Refresh podcast feeds (fetch new episodes from RSS)'
    def add_arguments(self, parser):
-        parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID (even when backing off)')
+        parser.add_argument('--feed-id', type=int, help='Refresh only this feed ID')
        parser.add_argument('--limit', type=int, default=0, help='Max number of feeds to refresh')
    def handle(self, *args, **options):
@ -17,10 +18,6 @@ class Command(BaseCommand):
        if options['feed_id']:
            qs = qs.filter(pk=options['feed_id'])
        else:
            # A feed failing 3+ times in a row is tried every 2, 4, 8, 16,
            # at most 24 hours instead of every hour.
            qs = qs.exclude(_in_backoff_q(timezone.now()))
        if options['limit']:
            qs = qs[: options['limit']]
--- a/podcasts/migrations/0007_feed_settings.py
+++ b/podcasts/migrations/0007_feed_settings.py
@ -1,74 +0,0 @@
 from django.db import migrations, models
 def auto_queue_to_new_episodes(apps, schema_editor):
    """Feeds that queued their new episodes keep doing so."""
    PodcastFeed = apps.get_model('podcasts', 'PodcastFeed')
    PodcastFeed.objects.filter(auto_queue=True).update(new_episodes='queue')
 def new_episodes_to_auto_queue(apps, schema_editor):
    PodcastFeed = apps.get_model('podcasts', 'PodcastFeed')
    PodcastFeed.objects.filter(new_episodes='queue').update(auto_queue=True)
 class Migration(migrations.Migration):
    dependencies = [
        ('podcasts', '0006_episodeprogress_favorite'),
    ]
    operations = [
        migrations.AddField(
            model_name='podcastfeed',
            name='exclude_words',
            field=models.CharField(blank=True, max_length=500),
        ),
        migrations.AddField(
            model_name='podcastfeed',
            name='include_words',
            field=models.CharField(blank=True, max_length=500),
        ),
        migrations.AddField(
            model_name='podcastfeed',
            name='min_duration',
            field=models.PositiveIntegerField(default=0),
        ),
        migrations.AddField(
            model_name='podcastfeed',
            name='new_episodes',
            field=models.CharField(
                choices=[('inbox', 'In „Neu“'), ('queue', 'In die Warteschlange'), ('ignore', 'Nicht anzeigen')],
                default='inbox', max_length=10),
        ),
        migrations.AddField(
            model_name='podcastfeed',
            name='playback_speed',
            field=models.FloatField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='podcastfeed',
            name='skip_intro',
            field=models.PositiveIntegerField(default=0),
        ),
        migrations.AddField(
            model_name='podcastfeed',
            name='skip_outro',
            field=models.PositiveIntegerField(default=0),
        ),
        migrations.AddField(
            model_name='podcastfeed',
            name='volume',
            field=models.FloatField(default=1.0),
        ),
        migrations.AddField(
            model_name='podcastsettings',
            name='keep_sorted',
            field=models.CharField(
                blank=True,
                choices=[('', 'Aus'), ('new', 'Neueste zuerst'), ('old', 'Älteste zuerst'),
                         ('short', 'Kürzeste zuerst'), ('long', 'Längste zuerst'), ('feed', 'Nach Podcast')],
                default='', max_length=10),
        ),
        migrations.RunPython(auto_queue_to_new_episodes, new_episodes_to_auto_queue),
    ]
--- a/podcasts/migrations/0008_feed_http_cache.py
+++ b/podcasts/migrations/0008_feed_http_cache.py
@ -1,38 +0,0 @@
 # Generated by Django 6.1 on 2026-10-02 12:00
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ('podcasts', '0007_feed_settings'),
    ]
    operations = [
        migrations.AddField(
            model_name='podcastfeed',
            name='http_etag',
            field=models.CharField(blank=True, max_length=500),
        ),
        migrations.AddField(
            model_name='podcastfeed',
            name='http_last_modified',
            field=models.CharField(blank=True, max_length=100),
        ),
        migrations.AddField(
            model_name='podcastfeed',
            name='last_error',
            field=models.CharField(blank=True, max_length=500),
        ),
        migrations.AddField(
            model_name='podcastfeed',
            name='last_error_at',
            field=models.DateTimeField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='podcastfeed',
            name='error_count',
            field=models.PositiveIntegerField(default=0),
        ),
    ]
--- a/podcasts/migrations/0009_episode_transcript_cache.py
+++ b/podcasts/migrations/0009_episode_transcript_cache.py
@ -1,25 +0,0 @@
 # Generated by Django 6.1 on 2026-10-02 07:44
 import django.db.models.deletion
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ('podcasts', '0008_feed_http_cache'),
    ]
    operations = [
        migrations.CreateModel(
            name='EpisodeTranscript',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('sources', models.JSONField(default=list)),
                ('data', models.JSONField(blank=True, null=True)),
                ('error', models.CharField(blank=True, max_length=20)),
                ('fetched_at', models.DateTimeField(auto_now=True)),
                ('episode', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='transcript_cache', to='podcasts.podcastepisode')),
            ],
        ),
    ]
--- a/podcasts/models.py
+++ b/podcasts/models.py
@ -3,18 +3,6 @@ from django.db import models
 class PodcastFeed(models.Model):
    # What happens to a new episode of this feed (one that passed the filter).
    NEW_INBOX = 'inbox'
    NEW_QUEUE = 'queue'
    NEW_IGNORE = 'ignore'
    NEW_EPISODE_ACTIONS = [
        (NEW_INBOX, 'In „Neu“'),
        (NEW_QUEUE, 'In die Warteschlange'),
        (NEW_IGNORE, 'Nicht anzeigen'),
    ]
    # Volume factors: below 1 quieter, above 1 louder (the apps boost those).
    VOLUMES = (0.2, 0.5, 1.0, 1.5, 2.0, 2.5)
    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='podcast_feeds')
    rss_url = models.URLField(max_length=1000)
    title = models.CharField(max_length=300)
@ -24,30 +12,7 @@ class PodcastFeed(models.Model):
    link = models.URLField(max_length=1000, blank=True)
    last_refreshed_at = models.DateTimeField(null=True, blank=True)
    added_at = models.DateTimeField(auto_now_add=True)
    # Always the same as new_episodes == 'queue' (see save()); kept for the
    # web UI's toggle and older clients.
    auto_queue = models.BooleanField(default=False)
    # HTTP validators of the last full (200) response, sent back as
    # If-None-Match / If-Modified-Since so an unchanged feed costs a 304.
    http_etag = models.CharField(max_length=500, blank=True)
    http_last_modified = models.CharField(max_length=100, blank=True)
    # The last failed refresh (network, HTTP status, not a feed); cleared on
    # success. error_count counts failures in a row and drives the backoff.
    last_error = models.CharField(max_length=500, blank=True)
    last_error_at = models.DateTimeField(null=True, blank=True)
    error_count = models.PositiveIntegerField(default=0)
    # Playback, for every device: null = the device's own speed.
    playback_speed = models.FloatField(null=True, blank=True)
    skip_intro = models.PositiveIntegerField(default=0)   # seconds
    skip_outro = models.PositiveIntegerField(default=0)   # seconds
    volume = models.FloatField(default=1.0)               # one of VOLUMES
    # New episodes: what happens to them, and which ones count at all.
    new_episodes = models.CharField(max_length=10, choices=NEW_EPISODE_ACTIONS, default=NEW_INBOX)
    # Comma-separated words, case-insensitive, matched against the title.
    include_words = models.CharField(max_length=500, blank=True)
    exclude_words = models.CharField(max_length=500, blank=True)
    min_duration = models.PositiveIntegerField(default=0)  # minutes, 0 = any
    class Meta:
        unique_together = ('user', 'rss_url')
@ -56,58 +21,6 @@ class PodcastFeed(models.Model):
    def __str__(self):
        return self.title
    @classmethod
    def from_db(cls, *args, **kwargs):
        obj = super().from_db(*args, **kwargs)
        obj._remember_sync()
        return obj
    def refresh_from_db(self, *args, **kwargs):
        super().refresh_from_db(*args, **kwargs)
        self._remember_sync()
    def _remember_sync(self):
        self._synced = (self.__dict__.get('auto_queue'), self.__dict__.get('new_episodes'))
    def _sync_auto_queue(self):
        """Keep auto_queue == (new_episodes == 'queue'). Whichever of the two
        was changed since loading (or, for a new feed, set away from its
        default) wins; new_episodes if both were."""
        auto_before, new_before = getattr(self, '_synced', (False, self.NEW_INBOX))
        if self.new_episodes != new_before or self.auto_queue == auto_before:
            self.auto_queue = self.new_episodes == self.NEW_QUEUE
        elif self.auto_queue:
            self.new_episodes = self.NEW_QUEUE
        elif self.new_episodes == self.NEW_QUEUE:
            self.new_episodes = self.NEW_INBOX
    def save(self, *args, **kwargs):
        update_fields = kwargs.get('update_fields')
        if update_fields is not None and not {'auto_queue', 'new_episodes'} & set(update_fields):
            return super().save(*args, **kwargs)
        self._sync_auto_queue()
        if update_fields is not None:
            kwargs['update_fields'] = set(update_fields) | {'auto_queue', 'new_episodes'}
        super().save(*args, **kwargs)
        self._remember_sync()
    def matches_filter(self, title, duration_seconds):
        """Whether a new episode passes this feed's filter: an include word
        (if any are given), no exclude word, and long enough (if known)."""
        title = (title or '').casefold()
        include = _words(self.include_words)
        if include and not any(w in title for w in include):
            return False
        if any(w in title for w in _words(self.exclude_words)):
            return False
        if self.min_duration and duration_seconds and duration_seconds < self.min_duration * 60:
            return False
        return True
 def _words(raw):
    return [w.strip().casefold() for w in (raw or '').split(',') if w.strip()]
 class PodcastEpisode(models.Model):
    feed = models.ForeignKey(PodcastFeed, on_delete=models.CASCADE, related_name='episodes')
@ -124,11 +37,9 @@ class PodcastEpisode(models.Model):
    # Podcasting 2.0 <podcast:chapters url=…>: a JSON file, loaded on demand.
    chapters_url = models.URLField(max_length=1000, blank=True)
    # [{start: seconds, title, url?, img?}], from Podlove Simple Chapters in
-    # the feed, or once fetched: the JSON at chapters_url, else (or if that
+    # the feed, or the JSON at chapters_url once it has been fetched.
    # has none) the CHAP frames of the audio file's ID3v2 tag.
    chapters = models.JSONField(default=list, blank=True)
-    # When chapters were fetched into `chapters` (from chapters_url or the
+    # When chapters_url was fetched into `chapters`; null = not yet.
    # audio file); null = not yet.
    chapters_fetched_at = models.DateTimeField(null=True, blank=True)
    # [{url, type, language}] from <podcast:transcript>.
    transcripts = models.JSONField(default=list, blank=True)
@ -141,27 +52,6 @@ class PodcastEpisode(models.Model):
        return self.title
 class EpisodeTranscript(models.Model):
    """An episode's transcript, fetched and parsed once (transcripts.py).
    A table of its own rather than fields on PodcastEpisode: a parsed
    transcript runs to hundreds of KB, and episode lists load whole rows.
    """
    episode = models.OneToOneField(PodcastEpisode, on_delete=models.CASCADE, related_name='transcript_cache')
    # URLs of the transcripts it was made from, best first. When the feed
    # offers different ones, the cache is stale.
    sources = models.JSONField(default=list)
    # {"segments": [...], "source": {...}}, or null with an error.
    data = models.JSONField(null=True, blank=True)
    # '' when data is there; else why there's none for good: 'unsupported'
    # (no format with timing), 'invalid' (missing, broken, too large).
    error = models.CharField(max_length=20, blank=True)
    fetched_at = models.DateTimeField(auto_now=True)
    def __str__(self):
        return f'Transcript of {self.episode}'
 # An episode counts as played this close to its end (AntennaPod's default),
 # the same rule as the apps' "als gehört zählen".
 PLAYED_BEFORE_END_SECONDS = 30
@ -204,8 +94,8 @@ class PodcastQueue(models.Model):
 class PodcastSettings(models.Model):
    """A user's podcast preferences. Where episodes go in the queue: when added
-    by hand (also when played from outside the queue), and when a feed that
+    by hand (also when played from outside the queue), and when a feed with
-    queues its new episodes brings some; and whether the queue stays sorted."""
+    auto_queue brings new ones."""
    BACK = 'back'
    FRONT = 'front'
    AFTER_CURRENT = 'after_current'
@ -215,22 +105,10 @@ class PodcastSettings(models.Model):
        (AFTER_CURRENT, 'Nach der laufenden Folge'),
    ]
    # Keep the queue sorted: after every change the whole queue is sorted
    # again by this rule ('' = off, the user's own order).
    SORT_ORDERS = [
        ('', 'Aus'),
        ('new', 'Neueste zuerst'),
        ('old', 'Älteste zuerst'),
        ('short', 'Kürzeste zuerst'),
        ('long', 'Längste zuerst'),
        ('feed', 'Nach Podcast'),
    ]
    user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='podcast_settings')
    enqueue_position = models.CharField(max_length=20, choices=POSITIONS, default=BACK)
    # front = how auto-queue always worked before this setting existed
    auto_queue_position = models.CharField(max_length=20, choices=POSITIONS, default=FRONT)
    keep_sorted = models.CharField(max_length=10, choices=SORT_ORDERS, default='', blank=True)
    @classmethod
    def for_user(cls, user):
--- a/podcasts/tests.py
+++ b/podcasts/tests.py
--- a/podcasts/transcripts.py
+++ b/podcasts/transcripts.py
@ -1,282 +0,0 @@
 """Podcast transcripts (<podcast:transcript>): pick the best one an episode
 offers, fetch it with the same guards as the chapters, and turn it into
 readable segments:
    {"segments": [{"start": 1.5, "end": 9.25, "speaker": "Anna" | None, "text": "…"}],
     "source": {"url": "…", "type": "text/vtt", "language": "de"}}
 Formats: Podcasting 2.0 JSON, WebVTT and SubRip (SRT). HTML transcripts have
 no timing and aren't supported. Cues are often a few words each; consecutive
 ones of the same speaker are merged up to a sentence end or about 30 s.
 """
 import html
 import json
 import re
 from . import chapters as fetching
 MAX_BYTES = 2 * 1024 * 1024
 MAX_SEGMENTS = 20_000
 MAX_TEXT = 5_000       # characters of one segment
 MAX_SPEAKER = 200
 # Merging
 MERGE_MAX_SECONDS = 30   # a segment never spans more than this
 MERGE_MIN_SECONDS = 5    # ... and ends at a sentence end once it's this long
 MERGE_MAX_GAP = 5        # silence that always starts a new segment
 JSON, VTT, SRT, HTML = 'json', 'vtt', 'srt', 'html'
 _KIND_RANK = {JSON: 0, VTT: 1, SRT: 2}
 _ACCEPT = {JSON: 'application/json', VTT: 'text/vtt', SRT: 'application/x-subrip'}
 class Unsupported(Exception):
    """The transcript is in a format without timing (HTML) or an unknown one."""
 def kind(transcript):
    """json, vtt, srt, html or '' (unknown) from the type, else the URL."""
    t = (transcript.get('type') or '').lower().split(';')[0].strip()
    if 'json' in t:
        return JSON
    if 'vtt' in t:
        return VTT
    if 'srt' in t or 'subrip' in t:
        return SRT
    if 'html' in t:
        return HTML
    path = (transcript.get('url') or '').lower().split('?')[0].split('#')[0]
    for ext, k in (('.json', JSON), ('.vtt', VTT), ('.srt', SRT), ('.html', HTML), ('.htm', HTML)):
        if path.endswith(ext):
            return k
    return ''
 def _language_rank(language):
    lang = (language or '').lower().split('-')[0].split('_')[0]
    if lang in ('de', ''):  # no language given: most likely the episode's own
        return 0
    if lang == 'en':
        return 1
    return 2
 def candidates(transcripts):
    """The episode's transcripts in a format we read, best first: German (or
    no language given) before English before others, then JSON before VTT
    before SRT."""
    usable = [(i, t) for i, t in enumerate(transcripts or []) if kind(t) in _KIND_RANK]
    usable.sort(key=lambda it: (_language_rank(it[1].get('language')), _KIND_RANK[kind(it[1])], it[0]))
    return [t for _, t in usable]
 # ---------------------------------------------------------------------------
 # Parsers: each returns fragments [(start, end, speaker, text)]
 # ---------------------------------------------------------------------------
 def _decode(body):
    if body.startswith((b'\xff\xfe', b'\xfe\xff')):
        return body.decode('utf-16', errors='replace')
    try:
        return body.decode('utf-8-sig')
    except UnicodeDecodeError:
        return body.decode('cp1252', errors='replace')
 def _clean(text):
    text = re.sub(r'<[^>]*>', ' ', text)
    return ' '.join(html.unescape(text).split())
 def _seconds(value):
    try:
        value = float(value)
    except (TypeError, ValueError):
        return None
    if value != value or value < 0 or value == float('inf'):
        return None
    return value
 def parse_json(text):
    """Podcasting 2.0 JSON: {"segments": [{"startTime", "endTime", "speaker", "body"}]}."""
    try:
        data = json.loads(text)
    except ValueError:
        raise fetching.FetchInvalid('not JSON')
    segments = data.get('segments') if isinstance(data, dict) else None
    if not isinstance(segments, list):
        raise fetching.FetchInvalid('no segments list')
    out = []
    for s in segments:
        if not isinstance(s, dict):
            continue
        start = _seconds(s.get('startTime'))
        if start is None:
            continue
        end = _seconds(s.get('endTime'))
        speaker = s.get('speaker')
        speaker = ' '.join(speaker.split()) if isinstance(speaker, str) else None
        body = s.get('body')
        out.append((start, end, speaker or None, _clean(body) if isinstance(body, str) else ''))
    return out
 # 00:01:02.500 / 01:02.500 (VTT), 00:01:02,500 (SRT)
 _TIME = r'(?:(\d+):)?(\d{1,2}):(\d{1,2})(?:[.,](\d{1,3}))?'
 _TIMING = re.compile(r'^\s*' + _TIME + r'\s*-->\s*' + _TIME + r'(?:\s|$)')
 _VOICE = re.compile(r'<v(?:\.[^\s>]*)?(?:\s+([^>]*))?>', re.IGNORECASE)
 def _time(hours, minutes, seconds, fraction):
    return int(hours or 0) * 3600 + int(minutes) * 60 + int(seconds) + (int(fraction.ljust(3, '0')) / 1000 if fraction else 0)
 def _cue_fragments(start, end, lines):
    """Split a cue's text at its voice tags (<v Anna>…) into fragments."""
    text = '\n'.join(lines)
    out = []
    speaker, pos = None, 0
    for m in _VOICE.finditer(text):
        out.append((start, end, speaker, _clean(text[pos:m.start()])))
        speaker = ' '.join((m.group(1) or '').split())[:MAX_SPEAKER] or None
        pos = m.end()
    out.append((start, end, speaker, _clean(text[pos:])))
    return [f for f in out if f[3]]
 def parse_cues(text):
    """WebVTT or SRT. Lines before a cue's timing line (VTT cue ids, SRT
    numbers) are dropped, as is everything outside cues (the WEBVTT header,
    NOTE, STYLE and REGION blocks)."""
    cues, current = [], None
    for line in text.replace('\r\n', '\n').replace('\r', '\n').split('\n'):
        m = _TIMING.match(line)
        if m:
            if current is not None and current[2] and current[2][-1].strip().isdigit():
                current[2].pop()  # an SRT number with no blank line before it
            g = m.groups()
            current = (_time(*g[:4]), _time(*g[4:]), [])
            cues.append(current)
        elif not line.strip():
            current = None
        elif current is not None:
            current[2].append(line)
    out = []
    for start, end, lines in cues:
        out.extend(_cue_fragments(start, end, lines))
    return out
 def parse(body, declared_kind):
    """Fragments from a downloaded transcript; the content decides the format
    when it's obvious, the declared type otherwise."""
    text = _decode(body)
    head = text.lstrip()[:200].lower()
    if head.startswith(('<!doctype', '<html', '<?xml', '<body', '<p', '<div')):
        raise Unsupported('HTML')
    if head.startswith(('{', '[')):
        return parse_json(text)
    if head.startswith('webvtt') or declared_kind in (VTT, SRT):
        return parse_cues(text)
    if declared_kind == JSON:
        return parse_json(text)
    raise Unsupported('unknown format')
 # ---------------------------------------------------------------------------
 # Merging
 # ---------------------------------------------------------------------------
 _SENTENCE_END = re.compile(r'[.!?…]["\'”’»)\]]*$')
 def merge(fragments):
    """Readable segments from cue fragments: consecutive ones of the same
    speaker are joined until a sentence ends (once the segment is at least
    MERGE_MIN_SECONDS long), MERGE_MAX_SECONDS are reached, or there's a
    pause of more than MERGE_MAX_GAP."""
    frags = sorted((f for f in fragments if f[3]), key=lambda f: f[0])[:MAX_SEGMENTS * 10]
    # Fragments without a usable end run until the next one starts.
    fixed = []
    for i, (start, end, speaker, text) in enumerate(frags):
        if end is None or end <= start:
            nxt = frags[i + 1][0] if i + 1 < len(frags) else start
            end = max(nxt, start)
        fixed.append((start, end, speaker, text))
    segments = []
    for start, end, speaker, text in fixed:
        last = segments[-1] if segments else None
        if (
            last is not None
            and last['speaker'] == speaker
            and start - last['end'] <= MERGE_MAX_GAP
            and end - last['start'] <= MERGE_MAX_SECONDS
            and len(last['text']) + len(text) < MAX_TEXT
            and not (_SENTENCE_END.search(last['text']) and last['end'] - last['start'] >= MERGE_MIN_SECONDS)
        ):
            last['text'] += ' ' + text
            last['end'] = max(last['end'], end)
            continue
        if len(segments) >= MAX_SEGMENTS:
            break
        segments.append({'start': start, 'end': end, 'speaker': speaker, 'text': text[:MAX_TEXT]})
    for s in segments:
        s['start'], s['end'] = round(s['start'], 3), round(s['end'], 3)
    return segments
 # ---------------------------------------------------------------------------
 # Fetching
 # ---------------------------------------------------------------------------
 def fetch(transcript):
    """The parsed transcript {"segments", "source"} from one of the episode's
    <podcast:transcript> entries. Raises Unsupported, FetchInvalid or
    FetchUnavailable."""
    k = kind(transcript)
    if k not in _KIND_RANK:
        raise Unsupported(transcript.get('type') or 'unknown type')
    body = fetching.fetch_bytes(
        transcript['url'], MAX_BYTES, accept=f'{_ACCEPT[k]}, */*;q=0.1', agent='diora podcast transcripts',
    )
    segments = merge(parse(body, k))
    if not segments:
        raise fetching.FetchInvalid('no segments')
    return {
        'segments': segments,
        'source': {
            'url': transcript['url'],
            'type': transcript.get('type') or '',
            'language': transcript.get('language') or '',
        },
    }
 def load(transcripts):
    """(parsed transcript, final) from the best candidate that works.
    final is False when a better candidate failed for now only, so the
    result shouldn't be kept for good. Raises FetchUnavailable when nothing
    worked and something may work later, Unsupported when no candidate is in
    a usable format, FetchInvalid otherwise.
    """
    retry_later = False
    unsupported_only = True
    for t in candidates(transcripts):
        try:
            return fetch(t), not retry_later
        except fetching.FetchUnavailable:
            retry_later = True
            unsupported_only = False
        except Unsupported:
            pass
        except fetching.FetchInvalid:
            unsupported_only = False
    if retry_later:
        raise fetching.FetchUnavailable('try again later')
    if unsupported_only:
        raise Unsupported('no usable transcript')
    raise fetching.FetchInvalid('no usable transcript')
--- a/podcasts/urls.py
+++ b/podcasts/urls.py
@ -12,11 +12,9 @@ urlpatterns = [
    path('state/', views.podcast_state, name='podcast_state'),
    path('feeds/<int:pk>/remove/', views.remove_feed, name='podcast_remove_feed'),
    path('feeds/<int:pk>/set-auto-queue/', views.set_auto_queue, name='podcast_set_auto_queue'),
    path('feeds/<int:pk>/settings/', views.feed_settings, name='podcast_feed_settings'),
    path('feeds/<int:pk>/episodes/', views.feed_episodes, name='podcast_feed_episodes'),
    path('episodes/', views.episode_list, name='podcast_episode_list'),
    path('episodes/<int:pk>/', views.episode_detail, name='podcast_episode_detail'),
    path('episodes/<int:pk>/transcript/', views.episode_transcript, name='podcast_episode_transcript'),
    path('episodes/<int:pk>/favorite/', views.set_favorite, name='podcast_set_favorite'),
    path('stats/', views.podcast_stats, name='podcast_stats'),
    path('queue/', views.queue_get, name='podcast_queue_get'),
--- a/podcasts/views.py
+++ b/podcasts/views.py
@ -2,14 +2,12 @@ import json
 import logging
 import urllib.parse
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from datetime import timedelta
 import xml.etree.ElementTree as ET
 import feedparser
 import requests
 from django.conf import settings
 from django.db import IntegrityError, transaction
 from django.db.models import (
    Case, Count, Exists, F, FilteredRelation, IntegerField, Max, OuterRef, Q, Sum, Value, When,
 )
@ -21,10 +19,7 @@ from django.views.decorators.http import require_http_methods
 from . import chapters as chapters_mod
 from . import feedext
-from . import transcripts as transcripts_mod
+from .models import EpisodeProgress, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings, counts_as_played
 from .models import (
    EpisodeProgress, EpisodeTranscript, PodcastEpisode, PodcastFeed, PodcastQueue, PodcastSettings, counts_as_played,
 )
 from .shownotes import extract_timestamps, sanitize_html
 feedext.install()
@ -43,7 +38,6 @@ MAX_TRANSCRIPTS = 10
 # ---------------------------------------------------------------------------
 QUEUE_POSITIONS = {key for key, _ in PodcastSettings.POSITIONS}
 KEEP_SORTED = {key for key, _ in PodcastSettings.SORT_ORDERS}
 def _queue_order(user):
@ -108,58 +102,9 @@ def _enqueue(user, episode_ids, where, current_hint=None):
    )
    _write_queue_order(user, queued[:at] + new + queued[at:])
    _leave_inbox(user, new)
    _keep_sorted(user)
    return new
 def _sort_key(rule):
    """Sort key for one queue row ({pub_date, duration, position, feed_title})
    by a PodcastSettings.SORT_ORDERS rule. Ties keep their current order
    (the sort is stable)."""
    def ts(row):
        return row['pub_date'].timestamp() if row['pub_date'] else None
    def remaining(row):
        return max((row['duration'] or 0) - (row['position'] or 0), 0)
    if rule == 'new':  # no date: last
        return lambda r: (ts(r) is None, -(ts(r) or 0))
    if rule == 'old':  # no date: last as well
        return lambda r: (ts(r) is None, ts(r) or 0)
    if rule == 'short':
        return remaining
    if rule == 'long':
        return lambda r: -remaining(r)
    if rule == 'feed':  # by podcast title, then newest first
        return lambda r: ((r['feed_title'] or '').casefold(), ts(r) is None, -(ts(r) or 0))
    return None
 def _keep_sorted(user, rule=None):
    """Sort the user's whole queue by their keep_sorted rule (or `rule`), if
    there is one. Returns True if it is kept sorted."""
    if rule is None:
        rule = PodcastSettings.objects.filter(user=user).values_list('keep_sorted', flat=True).first() or ''
    key = _sort_key(rule)
    if key is None:
        return False
    rows = list(
        PodcastQueue.objects.filter(user=user)
        .order_by('position', 'added_at', 'id')
        .values('episode_id', pub_date=F('episode__pub_date'), duration=F('episode__duration_seconds'),
                feed_title=F('episode__feed__title'))
    )
    positions = dict(
        EpisodeProgress.objects.filter(user=user, episode_id__in=[r['episode_id'] for r in rows])
        .values_list('episode_id', 'position_seconds')
    )
    for r in rows:
        r['position'] = positions.get(r['episode_id'], 0)
    rows.sort(key=key)
    _write_queue_order(user, [r['episode_id'] for r in rows])
    return True
 def _leave_inbox(user, episode_ids):
    """Queued or started episodes are no longer new (AntennaPod's rule): out
    of the inbox, without counting as played."""
@ -234,10 +179,8 @@ def _entry_transcripts(entry):
 def _apply_extras(episode, description, chapters_url, psc_chapters, transcripts):
    """Copy what the feed says now onto `episode`; True if anything changed.
-    Inline (Podlove) chapters win. Without them, fetched chapters (from
+    Inline (Podlove) chapters win. Without them, chapters fetched from
-    chapters_url, or from the audio file when there's no URL) stay cached for
+    chapters_url stay cached for as long as the URL stays the same.
    as long as the URL stays the same; inline chapters going away means a
    fresh fetch.
    """
    changed = []
    if episode.description != description:
@ -247,8 +190,8 @@ def _apply_extras(episode, description, chapters_url, psc_chapters, transcripts)
        episode.transcripts = transcripts
        changed.append('transcripts')
    if psc_chapters:
-        chapters, fetched_at = psc_chapters, None
+        chapters, fetched_at = psc_chapters, episode.chapters_fetched_at
-    elif chapters_url == episode.chapters_url and episode.chapters_fetched_at:
+    elif chapters_url and chapters_url == episode.chapters_url and episode.chapters_fetched_at:
        chapters, fetched_at = episode.chapters, episode.chapters_fetched_at
    else:
        chapters, fetched_at = [], None
@ -271,141 +214,11 @@ EXTRA_FIELDS = ['description', 'transcripts', 'chapters_url', 'chapters', 'chapt
 # Shared feed refresh helper
 # ---------------------------------------------------------------------------
-class FeedFetchError(Exception):
+def _fetch_feed(url):
-    """A feed that could not be fetched or is not a feed; str() is the short
+    """Download and parse a feed (with a timeout, unlike feedparser's own fetch)."""
-    message stored in PodcastFeed.last_error ("HTTP 404", "timeout", …)."""
+    r = requests.get(url, timeout=FEED_TIMEOUT, headers={'User-Agent': 'diora podcast refresh'})
-
+    r.raise_for_status()
-
+    return feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
@dataclass
 class FeedFetch:
    """What _fetch_feed got. status 200: `parsed` is the feedparser result and
    etag/last_modified are the new validators ('' when the server sent none).
    status 304: unchanged since the validators that were sent, parsed is None.
    final_url is where the content came from after all redirects;
    permanent_url is the requested URL after following only the 301/308 hops
    at the start of the redirect chain (== the requested URL without one)."""
    status: int
    parsed: object = None
    etag: str = ''
    last_modified: str = ''
    final_url: str = ''
    permanent_url: str = ''
 PERMANENT_REDIRECTS = (301, 308)
 def _fetch_feed(url, etag='', last_modified=''):
    """Download and parse a feed (with a timeout, unlike feedparser's own
    fetch). With etag/last_modified it is a conditional GET. Returns a
    FeedFetch; raises FeedFetchError when there is nothing usable."""
    headers = {'User-Agent': 'diora podcast refresh'}
    if etag:
        headers['If-None-Match'] = etag
    if last_modified:
        headers['If-Modified-Since'] = last_modified
    try:
        r = requests.get(url, timeout=FEED_TIMEOUT, headers=headers)
    except requests.Timeout:
        raise FeedFetchError('timeout')
    except requests.TooManyRedirects:
        raise FeedFetchError('too many redirects')
    except requests.ConnectionError:
        raise FeedFetchError('connection failed')
    except requests.RequestException as e:
        raise FeedFetchError(f'request failed ({type(e).__name__})')
    # Each hop's target is the next hop's URL (the last one's is r.url).
    permanent_url = url
    targets = [h.url for h in r.history[1:]] + [r.url]
    for hop, target in zip(r.history, targets):
        if hop.status_code not in PERMANENT_REDIRECTS:
            break
        permanent_url = target
    if r.status_code == 304:
        if not (etag or last_modified):
            raise FeedFetchError('HTTP 304')  # nothing to be unchanged from
        return FeedFetch(304, None, etag, last_modified, r.url, permanent_url)
    if r.status_code >= 400:
        raise FeedFetchError(f'HTTP {r.status_code}')
    parsed = feedparser.parse(r.content, response_headers={'content-location': r.url, **r.headers})
    if parsed.get('bozo') and not parsed.entries and not parsed.feed.get('title'):
        raise FeedFetchError('not a feed')
    new_etag = r.headers.get('ETag', '')
    new_modified = r.headers.get('Last-Modified', '')
    return FeedFetch(
        r.status_code, parsed,
        # A validator too long to store is dropped (a truncated one never matches).
        new_etag if len(new_etag) <= 500 else '',
        new_modified if len(new_modified) <= 100 else '',
        r.url, permanent_url,
    )
 def _validators(feeds):
    """If-None-Match / If-Modified-Since for one download that serves all
    `feeds` (several users' subscriptions of the same URL): only when every
    one of them was fetched before and stored the same validators, because a
    304 then skips all of them. Otherwise {} (a full response)."""
    pairs = {(f.http_etag, f.http_last_modified) for f in feeds}
    if len(pairs) != 1 or any(f.last_refreshed_at is None for f in feeds):
        return {}
    etag, last_modified = pairs.pop()
    if not (etag or last_modified):
        return {}
    return {'etag': etag, 'last_modified': last_modified}
 def _error_text(e):
    if isinstance(e, FeedFetchError):
        return str(e)[:500]
    return (str(e) or type(e).__name__)[:500]
 def _record_error(feed_obj, e):
    feed_obj.last_error = _error_text(e)
    feed_obj.last_error_at = timezone.now()
    feed_obj.error_count += 1
    try:
        feed_obj.save(update_fields=['last_error', 'last_error_at', 'error_count'])
    except Exception:
        log.exception('could not store the refresh error of feed %s', feed_obj.pk)
 ERROR_FIELDS = ['last_error', 'last_error_at', 'error_count']
 def _clear_error(feed_obj):
    feed_obj.last_error = ''
    feed_obj.last_error_at = None
    feed_obj.error_count = 0
 BACKOFF_AFTER = 3        # failures in a row before the hourly job backs off
 BACKOFF_MAX_HOURS = 24
 def _backoff_hours(error_count):
    """Hours between hourly-job attempts for a feed failing error_count
    times in a row: 2, 4, 8, 16, 24 … from the third failure on."""
    if error_count < BACKOFF_AFTER:
        return 0
    return min(2 ** (error_count - 2), BACKOFF_MAX_HOURS)
 def _in_backoff_q(now=None):
    """Q for feeds that failed too often to be tried again yet."""
    now = now or timezone.now()
    n = BACKOFF_AFTER
    while _backoff_hours(n) < BACKOFF_MAX_HOURS:
        n += 1
    # From n failures on the wait is capped; below, one term per count.
    q = Q(error_count__gte=n, last_error_at__gt=now - timedelta(hours=BACKOFF_MAX_HOURS))
    for count in range(BACKOFF_AFTER, n):
        q |= Q(error_count=count, last_error_at__gt=now - timedelta(hours=_backoff_hours(count)))
    return q
 def _refresh_feeds(feeds, chunk=32):
@ -416,31 +229,27 @@ def _refresh_feeds(feeds, chunk=32):
    refreshed = failed = new = 0
    for start in range(0, len(feeds), chunk):
        part = feeds[start:start + chunk]
-        by_url = {}
+        urls = sorted({f.rss_url for f in part})
        for f in part:
            by_url.setdefault(f.rss_url, []).append(f)
        results = {}
        def fetch(url):
            try:
-                return url, _fetch_feed(url, **_validators(by_url[url])), None
+                return url, _fetch_feed(url), None
            except Exception as e:  # network, HTTP or parser trouble: report and go on
                return url, None, e
        urls = sorted(by_url)
        with ThreadPoolExecutor(max_workers=min(FEED_WORKERS, len(urls) or 1)) as pool:
-            for url, fetched, err in pool.map(fetch, urls):
+            for url, parsed, err in pool.map(fetch, urls):
-                results[url] = (fetched, err)
+                results[url] = (parsed, err)
        for feed in part:
-            fetched, err = results[feed.rss_url]
+            parsed, err = results[feed.rss_url]
            if err is not None:
                failed += 1
                log.warning('refreshing %s failed: %s', feed.rss_url, err)
                _record_error(feed, err)
                continue
            try:
-                new += _refresh_feed(feed, fetched)
+                new += _refresh_feed(feed, parsed)
                refreshed += 1
            except Exception as e:
                failed += 1
@ -448,47 +257,10 @@ def _refresh_feeds(feeds, chunk=32):
    return refreshed, failed, new
-def _refresh_feed(feed_obj, fetched=None):
+def _refresh_feed(feed_obj, parsed=None):
-    """Fetch rss_url (unless `fetched`, a FeedFetch, is given) and upsert
+    """Parse rss_url and upsert episodes. Returns count of new episodes."""
-    episodes. Returns count of new episodes. A failure is stored on the feed
+    if parsed is None:
-    (last_error, error_count) and raised again."""
+        parsed = _fetch_feed(feed_obj.rss_url)
    if fetched is None:
        try:
            fetched = _fetch_feed(feed_obj.rss_url, **_validators([feed_obj]))
        except Exception as e:
            _record_error(feed_obj, e)
            raise
    try:
        return _apply_feed(feed_obj, fetched)
    except Exception as e:
        _record_error(feed_obj, e)
        raise
 def _move_to_permanent_url(feed_obj, new_url):
    """The feed answered with a permanent redirect: subscribe to the new URL,
    unless the user already has that one too (then nothing changes)."""
    if not new_url or new_url == feed_obj.rss_url or len(new_url) > 1000:
        return
    try:
        with transaction.atomic():
            PodcastFeed.objects.filter(pk=feed_obj.pk).update(rss_url=new_url)
    except IntegrityError:  # the user's other subscription has it
        return
    log.info('feed %s moved permanently: %s -> %s', feed_obj.pk, feed_obj.rss_url, new_url)
    feed_obj.rss_url = new_url
 def _apply_feed(feed_obj, fetched):
    now = timezone.now()
    if fetched.status == 304:
        # Unchanged since the last full download: nothing to parse or write.
        feed_obj.last_refreshed_at = now
        _clear_error(feed_obj)
        feed_obj.save(update_fields=['last_refreshed_at', *ERROR_FIELDS])
        return 0
    parsed = fetched.parsed
    channel = parsed.feed
    # The first fetch of a new subscription brings the whole back catalogue.
    first_fetch = feed_obj.last_refreshed_at is None
@ -506,7 +278,7 @@ def _apply_feed(feed_obj, fetched):
        or image.get('url', '')
    )[:1000]
-    feed_obj.last_refreshed_at = now
+    feed_obj.last_refreshed_at = timezone.now()
    feed_obj.save()
    max_ep = getattr(settings, 'PODCAST_MAX_EPISODES_PER_FEED', 200)
@ -609,20 +381,7 @@ def _apply_feed(feed_obj, fetched):
        )
        new_episodes_list = newest_first[:NEW_ON_SUBSCRIBE]
-    # The feed's own rules for its new episodes: those its filter turns down,
+    if feed_obj.auto_queue and new_episodes_list:
    # and all of them for 'ignore', go straight out of the inbox (not played);
    # 'queue' puts them in the queue (which also takes them out of the inbox);
    # 'inbox' leaves them there.
    if new_episodes_list:
        unwanted = [e for e in new_episodes_list if not feed_obj.matches_filter(e.title, e.duration_seconds)]
        if feed_obj.new_episodes == PodcastFeed.NEW_IGNORE:
            unwanted = new_episodes_list
        if unwanted:
            gone = {e.id for e in unwanted}
            _leave_inbox(feed_obj.user, list(gone))
            new_episodes_list = [e for e in new_episodes_list if e.id not in gone]
    if feed_obj.new_episodes == PodcastFeed.NEW_QUEUE and new_episodes_list:
        where = PodcastSettings.for_user(feed_obj.user).auto_queue_position
        # Newest first in front of what's there; oldest first when appended,
        # so the end of the queue plays in the order the episodes came out.
@ -634,13 +393,6 @@ def _apply_feed(feed_obj, fetched):
        ordered = newest_first if where != PodcastSettings.BACK else newest_first[::-1]
        _enqueue(feed_obj.user, [e.id for e in ordered], where)
    # Only now, with every episode written: the validators (a 304 next time
    # skips all of the above) and a clean error state.
    feed_obj.http_etag = fetched.etag
    feed_obj.http_last_modified = fetched.last_modified
    _clear_error(feed_obj)
    feed_obj.save(update_fields=['http_etag', 'http_last_modified', *ERROR_FIELDS])
    _move_to_permanent_url(feed_obj, fetched.permanent_url)
    return new_count
@ -687,30 +439,16 @@ def feed_list(request):
    return JsonResponse({'feeds': _feeds_data(request.user)})
 FEED_SETTING_FIELDS = (
    'playback_speed', 'skip_intro', 'skip_outro', 'volume',
    'new_episodes', 'include_words', 'exclude_words', 'min_duration', 'auto_queue',
 )
 def _feed_settings(feed):
    return {'feed_id': feed.id, **{name: getattr(feed, name) for name in FEED_SETTING_FIELDS}}
 def _feeds_data(user):
    feeds = list(
        user.podcast_feeds
        .annotate(latest_episode_at=Max('episodes__pub_date'))
-        .values('id', 'title', 'artwork_url', 'rss_url', 'last_refreshed_at', 'author', 'added_at',
+        .values('id', 'title', 'artwork_url', 'rss_url', 'last_refreshed_at', 'author', 'added_at', 'auto_queue', 'latest_episode_at')
                'latest_episode_at', 'last_error', 'last_error_at', 'error_count', *FEED_SETTING_FIELDS)
    )
    for f in feeds:
        f['feed_id'] = f['id']  # as in feeds/<id>/settings/
        if f['last_refreshed_at']:
            f['last_refreshed_at'] = f['last_refreshed_at'].isoformat()
        if f['last_error_at']:
            f['last_error_at'] = f['last_error_at'].isoformat()
        if f['added_at']:
            f['added_at'] = f['added_at'].isoformat()
        if f['latest_episode_at']:
@ -796,99 +534,11 @@ def set_auto_queue(request, pk):
        feed = PodcastFeed.objects.get(pk=pk, user=request.user)
    except PodcastFeed.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)
    # On: new episodes go to the queue; off: back to the inbox (a feed set
    # to 'ignore' stays so). PodcastFeed.save() sets new_episodes to match.
    feed.auto_queue = bool(body.get('auto_queue', not feed.auto_queue))
    feed.save(update_fields=['auto_queue'])
    return JsonResponse({'ok': True, 'auto_queue': feed.auto_queue})
 # ---------------------------------------------------------------------------
 # Per-feed settings
 # ---------------------------------------------------------------------------
 def _number(value):
    return isinstance(value, (int, float)) and not isinstance(value, bool)
 def _whole(value, low, high):
    if not _number(value) or value != int(value) or not low <= value <= high:
        raise ValueError
    return int(value)
 def _speed(value):
    if value is None:
        return None
    if not _number(value) or not 0.5 <= value <= 3.0:
        raise ValueError
    return float(value)
 def _volume(value):
    if not _number(value) or float(value) not in PodcastFeed.VOLUMES:
        raise ValueError
    return float(value)
 def _new_episodes(value):
    if value not in {key for key, _ in PodcastFeed.NEW_EPISODE_ACTIONS}:
        raise ValueError
    return value
 def _word_list(value):
    if value is None:
        return ''
    if not isinstance(value, str) or len(value.strip()) > 500:
        raise ValueError
    return value.strip()
 FEED_SETTING_PARSERS = {
    'playback_speed': _speed,
    'skip_intro': lambda v: _whole(v, 0, 600),
    'skip_outro': lambda v: _whole(v, 0, 600),
    'volume': _volume,
    'new_episodes': _new_episodes,
    'include_words': _word_list,
    'exclude_words': _word_list,
    'min_duration': lambda v: _whole(v, 0, 600),
 }
@csrf_exempt
@require_http_methods(['POST'])
 def feed_settings(request, pk):
    """Change some of one feed's settings (any of FEED_SETTING_PARSERS);
    returns all of them. Nothing is changed if one value is invalid."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        body = json.loads(request.body or b'{}')
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    if not isinstance(body, dict):
        return JsonResponse({'error': 'invalid JSON'}, status=400)
    try:
        feed = PodcastFeed.objects.get(pk=pk, user=request.user)
    except PodcastFeed.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)
    changes = {}
    for name, parse in FEED_SETTING_PARSERS.items():
        if name in body:
            try:
                changes[name] = parse(body[name])
            except (ValueError, TypeError, OverflowError):
                return JsonResponse({'error': f'invalid {name}', 'field': name}, status=400)
    if changes:
        for name, value in changes.items():
            setattr(feed, name, value)
        feed.save(update_fields=list(changes))
    return JsonResponse(_feed_settings(feed))
 # ---------------------------------------------------------------------------
 # Feed episodes
 # ---------------------------------------------------------------------------
@ -952,29 +602,20 @@ def feed_episodes(request, pk):
 # ---------------------------------------------------------------------------
 def _load_chapters(episode):
-    """Fetch the chapters the feed doesn't carry inline, once, and cache them
+    """Fetch the JSON chapters of `episode` once and cache them on it.
    on `episode`: the JSON at chapters_url, else (no URL, or nothing there)
    the CHAP frames in the ID3v2 tag at the start of the audio file.
    Problems that may go away (timeouts, server errors) aren't cached, so the
    next request tries again; anything else is remembered as "no chapters".
    """
    url = episode.chapters_url
    chapters = []
    try:
        if url:
    try:
        chapters = chapters_mod.fetch(url)
            except chapters_mod.ChaptersInvalid as e:
                log.info('chapters of episode %s unusable: %s', episode.id, e)
        if not chapters and episode.audio_url:
            try:
                chapters = chapters_mod.fetch_id3(episode.audio_url)
            except chapters_mod.ChaptersInvalid as e:
                log.info('ID3 chapters of episode %s unusable: %s', episode.id, e)
    except chapters_mod.ChaptersUnavailable as e:
        log.info('chapters of episode %s not available: %s', episode.id, e)
        return []
    except chapters_mod.ChaptersInvalid as e:
        log.info('chapters of episode %s unusable: %s', episode.id, e)
        chapters = []
    now = timezone.now()
    # Only if a refresh hasn't changed the URL in the meantime.
    PodcastEpisode.objects.filter(pk=episode.pk, chapters_url=url).update(
@ -995,7 +636,7 @@ def episode_detail(request, pk):
        return JsonResponse({'error': 'not found'}, status=404)
    chapters = ep.chapters
-    if not chapters and ep.chapters_fetched_at is None:
+    if not chapters and ep.chapters_url and ep.chapters_fetched_at is None:
        chapters = _load_chapters(ep)
    progress = EpisodeProgress.objects.filter(user=request.user, episode=ep).first()
@ -1006,7 +647,6 @@ def episode_detail(request, pk):
        'timestamps': extract_timestamps(ep.description, ep.duration_seconds),
        'chapters': chapters,
        'transcripts': ep.transcripts,
        'has_transcript': _has_transcript(ep),
        'feed': {'id': ep.feed.id, 'title': ep.feed.title, 'artwork_url': ep.feed.artwork_url},
        'audio_url': ep.audio_url,
        'artwork_url': ep.artwork_url,
@ -1022,65 +662,6 @@ def episode_detail(request, pk):
    })
 def _transcript_sources(ep):
    return [t['url'] for t in transcripts_mod.candidates(ep.transcripts)]
 def _has_transcript(ep):
    """True if the episode offers a transcript we can read, and it hasn't
    turned out to be unusable."""
    sources = _transcript_sources(ep)
    if not sources:
        return False
    cached = EpisodeTranscript.objects.filter(episode=ep).values_list('sources', 'error').first()
    return not (cached and cached[0] == sources and cached[1])
@csrf_exempt
@require_http_methods(['GET'])
 def episode_transcript(request, pk):
    """The episode's transcript as {"segments": [{start, end, speaker, text}],
    "source": {url, type, language}}, fetched and parsed on first use, then
    kept. 404 {"error": "no transcript" | "unsupported" | "invalid"} when
    there's none to show; 502 {"error": "unavailable"} when it couldn't be
    loaded right now."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
        ep = PodcastEpisode.objects.get(pk=pk, feed__user=request.user)
    except PodcastEpisode.DoesNotExist:
        return JsonResponse({'error': 'not found'}, status=404)
    if not ep.transcripts:
        return JsonResponse({'error': 'no transcript'}, status=404)
    sources = _transcript_sources(ep)
    if not sources:
        return JsonResponse({'error': 'unsupported'}, status=404)
    cached = EpisodeTranscript.objects.filter(episode=ep).first()
    if cached is not None and cached.sources == sources:
        data, error = cached.data, cached.error
    else:
        try:
            data, final = transcripts_mod.load(ep.transcripts)
            error = ''
        except chapters_mod.FetchUnavailable as e:
            log.info('transcript of episode %s not available: %s', ep.id, e)
            return JsonResponse({'error': 'unavailable'}, status=502)
        except transcripts_mod.Unsupported as e:
            log.info('transcript of episode %s unsupported: %s', ep.id, e)
            data, error, final = None, 'unsupported', True
        except chapters_mod.FetchInvalid as e:
            log.info('transcript of episode %s unusable: %s', ep.id, e)
            data, error, final = None, 'invalid', True
        if final:
            EpisodeTranscript.objects.update_or_create(
                episode=ep, defaults={'sources': sources, 'data': data, 'error': error},
            )
    if error:
        return JsonResponse({'error': error}, status=404)
    return JsonResponse(data)
 # ---------------------------------------------------------------------------
 # Favorites
 # ---------------------------------------------------------------------------
@ -1376,8 +957,7 @@ def refresh_feed_now(request):
 def refresh_all_feeds(request):
    """Refresh all of the user's feeds on the server in one go. Feeds the
    hourly job (or an earlier call) refreshed within `max_age` seconds are
-    skipped. Clients then fetch the result once (state/, inbox/ …).
+    skipped. Clients then fetch the result once (state/, inbox/ …)."""
    The user asked, so feeds the hourly job backs off from are tried too."""
    if not request.user.is_authenticated:
        return JsonResponse({'error': 'authentication required'}, status=401)
    try:
@ -1409,8 +989,7 @@ def podcast_state(request):
        'feeds': _feeds_data(request.user),
        'queue': _queue_data(request.user),
        'inbox': _inbox_data(request.user, 0, limit, with_description=False),
-        'settings': {'enqueue_position': s.enqueue_position, 'auto_queue_position': s.auto_queue_position,
+        'settings': {'enqueue_position': s.enqueue_position, 'auto_queue_position': s.auto_queue_position},
                     'keep_sorted': s.keep_sorted},
        'server_time': timezone.now().isoformat(),
    })
@ -1522,8 +1101,6 @@ def queue_reorder(request):
    order = body.get('order', [])  # list of episode ids
    for pos, ep_id in enumerate(order):
        PodcastQueue.objects.filter(user=request.user, episode_id=ep_id).update(position=pos)
    # A queue that is kept sorted goes straight back into its order.
    _keep_sorted(request.user)
    return JsonResponse({'ok': True})
@ -1546,21 +1123,13 @@ def podcast_settings(request):
                    return JsonResponse({'error': f'unknown {name}'}, status=400)
                setattr(s, name, body[name])
                fields.append(name)
        if 'keep_sorted' in body:
            if body['keep_sorted'] not in KEEP_SORTED:
                return JsonResponse({'error': 'unknown keep_sorted'}, status=400)
            s.keep_sorted = body['keep_sorted']
            fields.append('keep_sorted')
        if fields:
            s.save(update_fields=fields)
        if 'keep_sorted' in fields:
            _keep_sorted(request.user, s.keep_sorted)  # sorted right away
    elif request.method != 'GET':
        return JsonResponse({'error': 'method not allowed'}, status=405)
    return JsonResponse({
        'enqueue_position': s.enqueue_position,
        'auto_queue_position': s.auto_queue_position,
        'keep_sorted': s.keep_sorted,
        'positions': [{'value': k, 'label': v} for k, v in PodcastSettings.POSITIONS],
    })