diora-web/podcasts/sponsor.py

"""Skip the ad and the outro of a YouTube episode: what SponsorBlock knows about it.

A YouTube channel is a podcast here (docs/youtube.md): such an episode's audio_url is
``https://www.youtube.com/watch?v=<11-character id>``, and its sound belongs to the app —
the server never fetches it. What goes *out* of the sound is other people's work:
SponsorBlock (sponsor.ajay.app) lets listeners mark a video's parts by category, and the
two that hurt listening are the advertisement and the closing words. The server asks once
per episode and keeps the answer on it (PodcastEpisode.sponsor); the player then jumps over
those parts (podcasts-web.js for the browser and the desktop app, the app's own player for
Android).

Only those two categories are asked for, and only parts a listener skips: nothing about
intros, self-promotion, interaction reminders or non-music sections, and no part marked
``full`` — the whole video sponsored means there would be no sound left.
"""
import logging
from datetime import datetime

import requests

from django.utils import timezone

from . import youtube as youtube_mod

log = logging.getLogger(__name__)

VERSION = 1
API = 'https://sponsor.ajay.app/api/skipSegments'
CATEGORIES = ('sponsor', 'outro')
TIMEOUT = (3, 8)            # connect, read: a player waits for this, so keep it short
MIN_SPAN = 0.3              # s: a shorter part isn't worth a jump
STALE = 7 * 24 * 3600       # s: an answer is asked again this long after (segments get added)
STALE_ERROR = 3600          # s: a failed ask is tried again sooner (it may have been the network)


def video_id_of(audio_url):
    """The video an episode is, judged by its address (the same reading as youtube.py)."""
    return youtube_mod.video_id_in(audio_url or '')


def _spans(segments):
    """[[from, to], …] in seconds: the parts of `segments` that go, in order, merged.

    What SponsorBlock answered for the categories asked for, as far as it is usable: a
    segment has a start and an end, `skip` or `mute` means it goes (a jump is quieter than
    muting and the player jumps anyway), and two parts that touch are one.
    """
    found = []
    for seg in segments or []:
        if not isinstance(seg, dict):
            continue
        if seg.get('category') not in CATEGORIES:
            continue
        if seg.get('actionType', 'skip') not in ('skip', 'mute'):
            continue
        span = seg.get('segment')
        if not isinstance(span, (list, tuple)) or len(span) != 2:
            continue
        try:
            start, end = float(span[0]), float(span[1])
        except (TypeError, ValueError):
            continue
        if not (start >= 0 and end > start) or end - start < MIN_SPAN or end != end:  # NaN
            continue
        found.append([start, end])
    found.sort()
    out = []
    for start, end in found:
        if out and start <= out[-1][1] + 0.001:
            out[-1][1] = max(out[-1][1], end)
        else:
            out.append([start, end])
    return [[round(a, 3), round(b, 3)] for a, b in out]


def fetch(video_id):
    """[[from, to], …] for the video, from SponsorBlock. Raises what the asking raised."""
    r = requests.get(API, params={'videoID': video_id, 'categories': list(CATEGORIES)},
                     timeout=TIMEOUT, headers={'User-Agent': 'diora podcast sponsorblock'})
    if r.status_code == 404:      # nobody has marked this video: not an error
        return []
    if r.status_code >= 400:
        raise RuntimeError(f'HTTP {r.status_code}')
    try:
        data = r.json()
    except ValueError:
        raise RuntimeError('not JSON')
    if not isinstance(data, list):
        raise RuntimeError('unexpected answer')
    return _spans(data)


def ask(video_id):
    """The result to store on the episode: the parts that go, or why they weren't found."""
    try:
        return {'v': VERSION, 'spans': fetch(video_id), 'at': timezone.now().isoformat()}
    except Exception as e:      # unreachable API, timeout, nonsense: remembered, retried later
        log.warning('sponsor parts of video %s failed: %s', video_id, e)
        return {'v': VERSION, 'spans': [], 'error': str(e)[:200] or type(e).__name__,
                'at': timezone.now().isoformat()}


def current(stored):
    """Whether a stored result still counts (else the episode is asked again).

    A part that goes doesn't change; new ones appear, though, so an episode is asked again
    after a week — and a failure after an hour, since it was probably the network.
    """
    if not stored or stored.get('v') != VERSION or 'spans' not in stored:
        return False
    age = STALE_ERROR if stored.get('error') else STALE
    at = stored.get('at')
    if not at:
        return False
    try:
        when = datetime.fromisoformat(at)
    except (TypeError, ValueError):
        return False
    if timezone.is_naive(when):
        when = timezone.make_aware(when)
    return (timezone.now() - when).total_seconds() < age


def spans_of(episode):
    """[[from, to], …]: what to leave out of this episode, asking SponsorBlock if need be.

    Nothing at all for an episode that isn't a video (SponsorBlock knows only YouTube), and
    nothing when the asking fails. The episode is only handed back changed through the DB
    (`update`), so a caller that keeps the row doesn't have to save it.
    """
    from .models import PodcastEpisode
    empty = {'ready': True, 'v': VERSION, 'spans': []}
    video_id = video_id_of(episode.audio_url)
    if not video_id:
        return empty
    stored = episode.sponsor
    if not current(stored):
        stored = ask(video_id)
        PodcastEpisode.objects.filter(pk=episode.pk).update(sponsor=stored)
        episode.sponsor = stored
    out = dict(empty, spans=stored.get('spans', []))
    if stored.get('error'):
        out['error'] = stored['error']
    return out