diora-web/podcasts/shownotes.py

"""Shownotes HTML: an allowlist sanitizer and timestamp extraction.

Feed HTML is written by whoever runs the podcast, so it is untrusted. feedparser
already cleans it once while parsing, but the apps render what this server hands
out in web views, so we clean it again on the way out with a deliberately small
allowlist: unknown tags are dropped (their text kept), script-like tags are
dropped with their content, and only a handful of attributes survive, with
links limited to http(s)/mailto.
"""

import html
import re
from html.parser import HTMLParser

ALLOWED_TAGS = {
    'a', 'abbr', 'b', 'blockquote', 'br', 'cite', 'code', 'dd', 'del', 'div', 'dl', 'dt',
    'em', 'figcaption', 'figure', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img',
    'ins', 'li', 'mark', 'ol', 'p', 'pre', 'q', 's', 'small', 'span', 'strong', 'sub',
    'sup', 'table', 'tbody', 'td', 'tfoot', 'th', 'thead', 'time', 'tr', 'u', 'ul',
}
VOID_TAGS = {'br', 'hr', 'img'}
# Dropped together with everything inside them.
DROP_CONTENT_TAGS = {
    'script', 'style', 'iframe', 'object', 'embed', 'noscript', 'template', 'svg',
    'math', 'head', 'title', 'frameset', 'frame', 'applet', 'audio', 'video', 'canvas',
    'select', 'textarea', 'button',
}
ALLOWED_ATTRS = {
    'a': {'href', 'title'},
    'img': {'src', 'alt', 'title', 'width', 'height'},
    'td': {'colspan', 'rowspan'},
    'th': {'colspan', 'rowspan'},
    'abbr': {'title'},
    'time': {'datetime'},
    'ol': {'start'},
}
URL_ATTRS = {'href', 'src'}
LINK_SCHEMES = {'http', 'https', 'mailto'}
IMAGE_SCHEMES = {'http', 'https'}
# A link to a second in the episode ("#t=754"), as in the article of a transcript
# (docs/transcription.md): the only fragment link that is kept.
TIME_LINK = re.compile(r'^#t=\d{1,6}$')

_CONTROL_AND_SPACE = re.compile(r'[\x00-\x20\x7f]+')
_SCHEME = re.compile(r'^([a-z][a-z0-9+.\-]*):')


def _safe_url(value, schemes):
    """The URL if it is absolute with an allowed scheme, else None.

    Browsers ignore whitespace and control characters inside a scheme
    ("java\\tscript:"), so those are removed before looking at it.
    """
    compact = _CONTROL_AND_SPACE.sub('', value or '')
    m = _SCHEME.match(compact.lower())
    if not m or m.group(1) not in schemes:
        return None
    return value.strip()


class _Sanitizer(HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.out = []
        self.open = []     # allowed tags currently open, outermost first
        self.dropping = 0  # depth inside DROP_CONTENT_TAGS

    def handle_starttag(self, tag, attrs):
        self._start(tag, attrs, closed=False)

    def handle_startendtag(self, tag, attrs):
        self._start(tag, attrs, closed=True)

    def _start(self, tag, attrs, closed):
        if tag in DROP_CONTENT_TAGS:
            if not closed:
                self.dropping += 1
            return
        if self.dropping or tag not in ALLOWED_TAGS:
            return
        kept = []
        allowed = ALLOWED_ATTRS.get(tag, set())
        for name, value in attrs:
            name = name.lower()
            if name not in allowed or value is None:
                continue
            if tag == 'a' and name == 'href' and TIME_LINK.match(value.strip()):
                value = value.strip()
            elif name in URL_ATTRS:
                value = _safe_url(value, IMAGE_SCHEMES if tag == 'img' else LINK_SCHEMES)
                if value is None:
                    continue
            kept.append((name, value))
        if tag == 'img' and not any(n == 'src' for n, _ in kept):
            return
        if tag == 'a':
            kept.append(('rel', 'nofollow noopener noreferrer'))
        attr_text = ''.join(f' {n}="{html.escape(v, quote=True)}"' for n, v in kept)
        self.out.append(f'<{tag}{attr_text}>')
        if tag not in VOID_TAGS and not closed:
            self.open.append(tag)

    def handle_endtag(self, tag):
        if tag in DROP_CONTENT_TAGS:
            if self.dropping:
                self.dropping -= 1
            return
        if self.dropping or tag not in self.open:
            return
        # Close everything opened inside it too, so the output stays nested.
        while self.open:
            t = self.open.pop()
            self.out.append(f'</{t}>')
            if t == tag:
                break

    def handle_data(self, data):
        if not self.dropping:
            self.out.append(html.escape(data, quote=False))

    def result(self):
        self.close()
        while self.open:
            self.out.append(f'</{self.open.pop()}>')
        return ''.join(self.out)


def sanitize_html(text):
    """Return `text` reduced to the allowlist above."""
    if not text:
        return ''
    s = _Sanitizer()
    s.feed(text)
    return s.result()


# ---------------------------------------------------------------------------
# Plain text and timestamps
# ---------------------------------------------------------------------------

_BLOCK_TAGS = {
    'br', 'p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote',
    'pre', 'dd', 'dt', 'hr', 'table', 'ul', 'ol', 'figure', 'figcaption',
}


class _TextExtractor(HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.out = []
        self.dropping = 0

    def handle_starttag(self, tag, attrs):
        if tag in DROP_CONTENT_TAGS:
            self.dropping += 1
        elif tag in _BLOCK_TAGS:
            self.out.append('\n')

    def handle_startendtag(self, tag, attrs):
        if tag in _BLOCK_TAGS:
            self.out.append('\n')

    def handle_endtag(self, tag):
        if tag in DROP_CONTENT_TAGS:
            self.dropping = max(self.dropping - 1, 0)
        elif tag in _BLOCK_TAGS:
            self.out.append('\n')

    def handle_data(self, data):
        if not self.dropping:
            self.out.append(data)


def html_to_text(text):
    """Shownotes as plain text, one line per block/line break."""
    if not text:
        return ''
    p = _TextExtractor()
    p.feed(text)
    p.close()
    return ''.join(p.out)


# A time at the start of a line, optionally in brackets, then the label:
# "12:34 Intro", "(1:02:03) - Fragen", "[00:05] Begrüßung".
_TIMESTAMP_LINE = re.compile(
    r'^[\s\-*•·]*[(\[]?(?:(\d{1,2}):)?(\d{1,2}):(\d{2})(?:\.\d+)?[)\]]?(?=\s|$|[-–—:|])'
    r'\s*[-–—:|]?\s*(.*)$'
)
MAX_TIMESTAMPS = 300


def extract_timestamps(text, duration_seconds=0):
    """[{seconds, label}] for lines of the shownotes that start with a time.

    Only line starts are looked at: a time in the middle of a sentence is just
    as likely to be a time of day.
    """
    result = []
    for line in html_to_text(text).splitlines():
        m = _TIMESTAMP_LINE.match(line)
        if not m:
            continue
        hours, minutes, seconds, label = m.groups()
        minutes, seconds = int(minutes), int(seconds)
        if seconds >= 60 or (hours is not None and minutes >= 60):
            continue
        total = int(hours or 0) * 3600 + minutes * 60 + seconds
        if duration_seconds and total > duration_seconds:
            continue
        result.append({'seconds': total, 'label': ' '.join(label.split())[:300]})
        if len(result) >= MAX_TIMESTAMPS:
            break
    return result