"""Shownotes HTML: an allowlist sanitizer and timestamp extraction.
Feed HTML is written by whoever runs the podcast, so it is untrusted. feedparser
already cleans it once while parsing, but the apps render what this server hands
out in web views, so we clean it again on the way out with a deliberately small
allowlist: unknown tags are dropped (their text kept), script-like tags are
dropped with their content, and only a handful of attributes survive, with
links limited to http(s)/mailto.
"""
import html
import re
from html.parser import HTMLParser
ALLOWED_TAGS = {
'a', 'abbr', 'b', 'blockquote', 'br', 'cite', 'code', 'dd', 'del', 'div', 'dl', 'dt',
'em', 'figcaption', 'figure', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img',
'ins', 'li', 'mark', 'ol', 'p', 'pre', 'q', 's', 'small', 'span', 'strong', 'sub',
'sup', 'table', 'tbody', 'td', 'tfoot', 'th', 'thead', 'time', 'tr', 'u', 'ul',
}
VOID_TAGS = {'br', 'hr', 'img'}
# Dropped together with everything inside them.
DROP_CONTENT_TAGS = {
'script', 'style', 'iframe', 'object', 'embed', 'noscript', 'template', 'svg',
'math', 'head', 'title', 'frameset', 'frame', 'applet', 'audio', 'video', 'canvas',
'select', 'textarea', 'button',
}
ALLOWED_ATTRS = {
'a': {'href', 'title'},
'img': {'src', 'alt', 'title', 'width', 'height'},
'td': {'colspan', 'rowspan'},
'th': {'colspan', 'rowspan'},
'abbr': {'title'},
'time': {'datetime'},
'ol': {'start'},
}
URL_ATTRS = {'href', 'src'}
LINK_SCHEMES = {'http', 'https', 'mailto'}
IMAGE_SCHEMES = {'http', 'https'}
_CONTROL_AND_SPACE = re.compile(r'[\x00-\x20\x7f]+')
_SCHEME = re.compile(r'^([a-z][a-z0-9+.\-]*):')
def _safe_url(value, schemes):
"""The URL if it is absolute with an allowed scheme, else None.
Browsers ignore whitespace and control characters inside a scheme
("java\\tscript:"), so those are removed before looking at it.
"""
compact = _CONTROL_AND_SPACE.sub('', value or '')
m = _SCHEME.match(compact.lower())
if not m or m.group(1) not in schemes:
return None
return value.strip()
class _Sanitizer(HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True)
self.out = []
self.open = [] # allowed tags currently open, outermost first
self.dropping = 0 # depth inside DROP_CONTENT_TAGS
def handle_starttag(self, tag, attrs):
self._start(tag, attrs, closed=False)
def handle_startendtag(self, tag, attrs):
self._start(tag, attrs, closed=True)
def _start(self, tag, attrs, closed):
if tag in DROP_CONTENT_TAGS:
if not closed:
self.dropping += 1
return
if self.dropping or tag not in ALLOWED_TAGS:
return
kept = []
allowed = ALLOWED_ATTRS.get(tag, set())
for name, value in attrs:
name = name.lower()
if name not in allowed or value is None:
continue
if name in URL_ATTRS:
value = _safe_url(value, IMAGE_SCHEMES if tag == 'img' else LINK_SCHEMES)
if value is None:
continue
kept.append((name, value))
if tag == 'img' and not any(n == 'src' for n, _ in kept):
return
if tag == 'a':
kept.append(('rel', 'nofollow noopener noreferrer'))
attr_text = ''.join(f' {n}="{html.escape(v, quote=True)}"' for n, v in kept)
self.out.append(f'<{tag}{attr_text}>')
if tag not in VOID_TAGS and not closed:
self.open.append(tag)
def handle_endtag(self, tag):
if tag in DROP_CONTENT_TAGS:
if self.dropping:
self.dropping -= 1
return
if self.dropping or tag not in self.open:
return
# Close everything opened inside it too, so the output stays nested.
while self.open:
t = self.open.pop()
self.out.append(f'')
if t == tag:
break
def handle_data(self, data):
if not self.dropping:
self.out.append(html.escape(data, quote=False))
def result(self):
self.close()
while self.open:
self.out.append(f'')
return ''.join(self.out)
def sanitize_html(text):
"""Return `text` reduced to the allowlist above."""
if not text:
return ''
s = _Sanitizer()
s.feed(text)
return s.result()
# ---------------------------------------------------------------------------
# Plain text and timestamps
# ---------------------------------------------------------------------------
_BLOCK_TAGS = {
'br', 'p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote',
'pre', 'dd', 'dt', 'hr', 'table', 'ul', 'ol', 'figure', 'figcaption',
}
class _TextExtractor(HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True)
self.out = []
self.dropping = 0
def handle_starttag(self, tag, attrs):
if tag in DROP_CONTENT_TAGS:
self.dropping += 1
elif tag in _BLOCK_TAGS:
self.out.append('\n')
def handle_startendtag(self, tag, attrs):
if tag in _BLOCK_TAGS:
self.out.append('\n')
def handle_endtag(self, tag):
if tag in DROP_CONTENT_TAGS:
self.dropping = max(self.dropping - 1, 0)
elif tag in _BLOCK_TAGS:
self.out.append('\n')
def handle_data(self, data):
if not self.dropping:
self.out.append(data)
def html_to_text(text):
"""Shownotes as plain text, one line per block/line break."""
if not text:
return ''
p = _TextExtractor()
p.feed(text)
p.close()
return ''.join(p.out)
# A time at the start of a line, optionally in brackets, then the label:
# "12:34 Intro", "(1:02:03) - Fragen", "[00:05] Begrüßung".
_TIMESTAMP_LINE = re.compile(
r'^[\s\-*•·]*[(\[]?(?:(\d{1,2}):)?(\d{1,2}):(\d{2})(?:\.\d+)?[)\]]?(?=\s|$|[-–—:|])'
r'\s*[-–—:|]?\s*(.*)$'
)
MAX_TIMESTAMPS = 300
def extract_timestamps(text, duration_seconds=0):
"""[{seconds, label}] for lines of the shownotes that start with a time.
Only line starts are looked at: a time in the middle of a sentence is just
as likely to be a time of day.
"""
result = []
for line in html_to_text(text).splitlines():
m = _TIMESTAMP_LINE.match(line)
if not m:
continue
hours, minutes, seconds, label = m.groups()
minutes, seconds = int(minutes), int(seconds)
if seconds >= 60 or (hours is not None and minutes >= 60):
continue
total = int(hours or 0) * 3600 + minutes * 60 + seconds
if duration_seconds and total > duration_seconds:
continue
result.append({'seconds': total, 'label': ' '.join(label.split())[:300]})
if len(result) >= MAX_TIMESTAMPS:
break
return result