diff --git a/podcasts/silence.py b/podcasts/silence.py index 2c5ab04..f0e5f84 100644 --- a/podcasts/silence.py +++ b/podcasts/silence.py @@ -1,60 +1,178 @@ -"""Skip silence in the browser: the pauses of an episode, found once on the server. +"""Skip silence in the browser: what to cut out of an episode's pauses, found once on the server. -Android's player skips silence itself. A browser can't listen to the audio of -most podcast servers (no CORS: Web Audio hears only zeros), so the server -reads the file once with ffmpeg's silencedetect and keeps the pauses on the -episode (PodcastEpisode.silences); the browser then jumps over them while it -plays (podcasts-web.js). +Android shortens pauses while it plays (diora-android PauseCutter.kt). A browser +can't listen to the audio of most podcast servers (no CORS: Web Audio hears only +zeros), so the server reads the file once and keeps the result on the episode +(PodcastEpisode.silences); the browser then runs fast through those parts while +it plays (podcasts-web.js). + +The rules are PauseCutter's, so a pause sounds the same everywhere: the audio +is judged in 10 ms windows against the recording itself — silence is well below +the recent speech level *and* close to the recording's noise floor, never above +-30 dB — and a pause of 90 ms or more keeps a fifth of its length (at least +60 ms, at most 250 ms), half before and half after the part that goes. """ import logging +import math import os -import re import subprocess import tempfile import threading +import warnings import requests from django.db import close_old_connections +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + try: + import audioop # the RMS of a window in C; gone from Python 3.13 (then: audioop-lts) + except ImportError: # pragma: no cover + audioop = None + log = logging.getLogger(__name__) -NOISE = '-35dB' # quieter than this counts as silence -MIN_SILENCE = 0.4 # seconds; shorter pauses stay (they belong to speech) +VERSION = 2 # what's stored on the episode; an older result is found anew +RATE = 8000 # Hz, mono: enough to judge loudness +WINDOW = 0.01 # s TIMEOUT = 900 # seconds for decoding one episode MAX_BYTES = 800 * 1024 * 1024 # a larger file isn't an episode to analyse MAX_RUNNING = 2 # episodes analysed at the same time -_START = re.compile(r'silence_start: (-?[\d.]+)') -_END = re.compile(r'silence_end: ([\d.]+)') +# PauseCutter.Params +MIN_PAUSE_MS = 90 +KEEP_MIN_MS = 60 +KEEP_RATIO = 0.2 +KEEP_MAX_MS = 250 +FADE_MS = 12 +BELOW_SPEECH_DB = 20.0 +ABOVE_NOISE_DB = 12.0 +CEILING_DB = -30.0 +FLOOR_DB = -70.0 +DECAY_DB_PER_WINDOW = 6.0 * WINDOW # 6 dB per second +NOISE_SHARE = 0.1 +NOISE_WINDOWS = 1000 # 10 s +NOISE_MIN_DB = -90.0 +NOISE_MAX_DB = -35.0 _running = set() _lock = threading.Lock() _slots = threading.BoundedSemaphore(MAX_RUNNING) -def parse(stderr): - """ffmpeg's silencedetect output -> [[start, end], …] in seconds (2 decimals).""" - spans = [] +class _NoiseHistory: + """The last windows' levels in 1 dB steps, for a running percentile.""" + BINS = 121 + + def __init__(self): + self.ring = [0] * NOISE_WINDOWS + self.bins = [0] * self.BINS + self.count = 0 + self.at = 0 + + def add(self, db): + b = min(max(int(db + 120.0), 0), self.BINS - 1) + if self.count == NOISE_WINDOWS: + self.bins[self.ring[self.at]] -= 1 + else: + self.count += 1 + self.ring[self.at] = b + self.bins[b] += 1 + self.at = (self.at + 1) % NOISE_WINDOWS + + def level(self): + if self.count < NOISE_WINDOWS // 10: + return -60.0 # not enough seen yet + target = max(int(self.count * NOISE_SHARE), 1) + seen = 0 + for b, n in enumerate(self.bins): + seen += n + if seen >= target: + return b - 120.0 + return 0.0 + + +def _cut_of(start, end): + """The part of the pause [start, end) (window indexes) that goes, in seconds, or None.""" + p = round((end - start) * WINDOW * 1000) # ms + if p < MIN_PAUSE_MS: + return None + keep = min(max(round(p * KEEP_RATIO), KEEP_MIN_MS), KEEP_MAX_MS, p) + if p - keep < FADE_MS: + return None + head = keep // 2 + tail = keep - head + return [round(start * WINDOW + head / 1000, 3), round(end * WINDOW - tail / 1000, 3)] + + +def cuts(levels): + """[[from, to], …] in seconds: what to leave out, for 10 ms window levels in dB.""" + level = -20.0 # recent speech level + noise = -60.0 # noise floor + history = _NoiseHistory() + out = [] start = None - for line in stderr.splitlines(): - m = _START.search(line) - if m: - start = max(0.0, float(m.group(1))) - continue - m = _END.search(line) - if m and start is not None: - end = float(m.group(1)) - if end - start >= MIN_SILENCE: - spans.append([round(start, 2), round(end, 2)]) + for i, db in enumerate(levels): + threshold = max(min(level - BELOW_SPEECH_DB, noise + ABOVE_NOISE_DB, CEILING_DB), FLOOR_DB) + silent = db < threshold + level = db if db > level else max(FLOOR_DB, level - DECAY_DB_PER_WINDOW) + history.add(db) + noise = min(max(history.level(), NOISE_MIN_DB), NOISE_MAX_DB) + if silent: + if start is None: + start = i + elif start is not None: + cut = _cut_of(start, i) + if cut: + out.append(cut) start = None - return spans + return out # a pause running to the end stays as it is + + +def _level_db(window): + if audioop is not None: + rms = audioop.rms(window, 2) + else: # pragma: no cover + samples = memoryview(window).cast('h') + rms = math.sqrt(sum(s * s for s in samples) / len(samples)) + return -120.0 if rms < 1 else 20 * math.log10(rms / 32768) + + +def levels_of(path): + """10 ms window levels (dB full scale) of the audio file at `path`.""" + import imageio_ffmpeg + proc = subprocess.Popen( + [imageio_ffmpeg.get_ffmpeg_exe(), '-nostdin', '-hide_banner', '-loglevel', 'error', + '-vn', '-sn', '-dn', '-i', path, '-ac', '1', '-ar', str(RATE), '-f', 's16le', '-'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + size = int(RATE * WINDOW) * 2 + levels = [] + rest = b'' + timer = threading.Timer(TIMEOUT, proc.kill) + timer.start() + try: + while True: + chunk = proc.stdout.read(1 << 20) + if not chunk: + break + data = rest + chunk + whole = len(data) // size * size + for i in range(0, whole, size): + levels.append(_level_db(data[i:i + size])) + rest = data[whole:] + err = proc.stderr.read().decode('utf-8', 'replace') + if proc.wait() != 0: + raise RuntimeError((err.strip().splitlines()[-1:] or [f'ffmpeg failed ({proc.returncode})'])[0][:200]) + finally: + timer.cancel() + return levels def analyze(url): - """The pauses of the audio at `url`. Downloaded here first: the static ffmpeg + """What to cut out of the audio at `url`. Downloaded here first: the static ffmpeg build can't look up host names (it crashes doing so), it reads a local file.""" - import imageio_ffmpeg fd, path = tempfile.mkstemp(suffix='.audio') try: with os.fdopen(fd, 'wb') as out, requests.get( @@ -67,16 +185,7 @@ def analyze(url): if size > MAX_BYTES: raise RuntimeError('file too large') out.write(chunk) - proc = subprocess.run( - [imageio_ffmpeg.get_ffmpeg_exe(), '-nostdin', '-hide_banner', '-nostats', - '-vn', '-sn', '-dn', '-i', path, - '-af', f'silencedetect=noise={NOISE}:d={MIN_SILENCE}', '-f', 'null', '-'], - capture_output=True, text=True, timeout=TIMEOUT, - ) - if proc.returncode != 0: - tail = (proc.stderr or '').strip().splitlines()[-1:] or [f'ffmpeg failed ({proc.returncode})'] - raise RuntimeError(tail[0][:200]) - return parse(proc.stderr) + return cuts(levels_of(path)) finally: try: os.remove(path) @@ -84,8 +193,13 @@ def analyze(url): pass +def current(silences): + """Whether a stored result is from these rules (else it's found anew).""" + return bool(silences) and silences.get('v') == VERSION + + def ensure(episode_id): - """Find the episode's pauses in the background, unless that runs already.""" + """Find the episode's cuts in the background, unless that runs already.""" with _lock: if episode_id in _running: return @@ -102,10 +216,10 @@ def _work(episode_id): if ep is None: return try: - result = {'spans': analyze(ep.audio_url)} + result = {'v': VERSION, 'spans': analyze(ep.audio_url)} except Exception as e: # unreachable file, timeout, not audio: remembered, not retried log.warning('silences of episode %s failed: %s', episode_id, e) - result = {'spans': [], 'error': str(e)[:200] or type(e).__name__} + result = {'v': VERSION, 'spans': [], 'error': str(e)[:200] or type(e).__name__} PodcastEpisode.objects.filter(pk=episode_id).update(silences=result) finally: with _lock: diff --git a/podcasts/tests.py b/podcasts/tests.py index e33a44a..7a608dd 100644 --- a/podcasts/tests.py +++ b/podcasts/tests.py @@ -2675,15 +2675,48 @@ class DiscardTests(TestCase): # --------------------------------------------------------------------------- class SilenceTests(TestCase): - STDERR = """Input #0, mp3, from 'x': -[silencedetect @ 0x1] silence_start: 1.5 -[silencedetect @ 0x1] silence_end: 2.4 | silence_duration: 0.9 -[silencedetect @ 0x1] silence_start: 10 -[silencedetect @ 0x1] silence_end: 10.2 | silence_duration: 0.2 -[silencedetect @ 0x1] silence_start: -0.01 -[silencedetect @ 0x1] silence_end: 0.75 | silence_duration: 0.76 -[silencedetect @ 0x1] silence_start: 99.123456 -""" + """The rules are diora-android's PauseCutter: what Android cuts, the browser runs through.""" + + @staticmethod + def speech(seconds, db=-22.0): + return [db + (0.5 if i % 2 else 0.0) for i in range(int(seconds * 100))] + + @staticmethod + def quiet(seconds, db=-75.0): + return [db] * int(seconds * 100) + + def test_a_pause_keeps_a_fifth_half_before_half_after(self): + from .silence import cuts + # 2 s of speech (the noise floor needs some history), a 0.5 s pause, speech. + levels = self.speech(12) + self.quiet(0.5) + self.speech(2) + self.assertEqual(cuts(levels), [[12.05, 12.45]]) # keeps 100 ms: 50 + 50 + + def test_short_gaps_between_words_stay(self): + from .silence import cuts + self.assertEqual(cuts(self.speech(12) + self.quiet(0.08) + self.speech(2)), []) + + def test_a_long_pause_keeps_at_most_a_quarter_second(self): + from .silence import cuts + self.assertEqual(cuts(self.speech(12) + self.quiet(3) + self.speech(2)), [[12.125, 14.875]]) + + def test_a_short_pause_keeps_at_least_60_ms(self): + from .silence import cuts + self.assertEqual(cuts(self.speech(12) + self.quiet(0.1) + self.speech(2)), [[12.03, 12.07]]) + + def test_a_quiet_second_voice_is_not_a_pause(self): + from .silence import cuts + # A talk with real pauses now and then (that's where the noise floor shows), + # then a voice well below the speaker but far above the floor: someone + # asking from the back. Only the real pauses are cut. + talk = (self.speech(1.8) + self.quiet(0.2)) * 6 + levels = talk + self.quiet(2, db=-48.0) + self.speech(2) + found = cuts(levels) + self.assertEqual(len(found), 6) + self.assertTrue(all(end <= 12.0 for _, end in found)) + + def test_a_pause_running_to_the_end_stays(self): + from .silence import cuts + self.assertEqual(cuts(self.speech(12) + self.quiet(2)), []) def setUp(self): self.user = User.objects.create_user(username='alice', password='pw12345678') @@ -2691,11 +2724,6 @@ class SilenceTests(TestCase): feed = PodcastFeed.objects.create(user=self.user, rss_url='https://example.org/f', title='F') self.ep = PodcastEpisode.objects.create(feed=feed, guid='e', title='E', audio_url='https://example.org/e.mp3') - def test_parse_keeps_pauses_long_enough(self): - from .silence import parse - # 0.2 s is too short; a start below 0 is 0; an open start at the end is dropped. - self.assertEqual(parse(self.STDERR), [[1.5, 2.4], [0.0, 0.75]]) - def test_first_request_starts_the_search_then_answers_with_the_pauses(self): with patch('podcasts.silence.ensure') as ensure: r = self.client.get(f'/podcasts/episodes/{self.ep.id}/silences/') @@ -2705,7 +2733,14 @@ class SilenceTests(TestCase): from .silence import _work _work(self.ep.id) r = self.client.get(f'/podcasts/episodes/{self.ep.id}/silences/') - self.assertEqual(r.json(), {'ready': True, 'spans': [[1.5, 2.4]]}) + self.assertEqual(r.json(), {'ready': True, 'v': 2, 'spans': [[1.5, 2.4]]}) + + def test_a_result_of_the_old_rules_is_found_anew(self): + PodcastEpisode.objects.filter(pk=self.ep.id).update(silences={'spans': [[1.0, 2.0]]}) + with patch('podcasts.silence.ensure') as ensure: + r = self.client.get(f'/podcasts/episodes/{self.ep.id}/silences/') + self.assertEqual(r.json(), {'ready': False}) + ensure.assert_called_once_with(self.ep.id) def test_a_failure_is_remembered_not_retried(self): from .silence import _work @@ -2714,7 +2749,7 @@ class SilenceTests(TestCase): with patch('podcasts.silence.ensure') as ensure: r = self.client.get(f'/podcasts/episodes/{self.ep.id}/silences/') ensure.assert_not_called() - self.assertEqual(r.json(), {'ready': True, 'spans': [], 'error': 'HTTP error 404 Not Found'}) + self.assertEqual(r.json(), {'ready': True, 'v': 2, 'spans': [], 'error': 'HTTP error 404 Not Found'}) def test_only_own_episodes(self): bob = User.objects.create_user(username='bob', password='pw12345678') diff --git a/podcasts/views.py b/podcasts/views.py index 50f3f8e..2124148 100644 --- a/podcasts/views.py +++ b/podcasts/views.py @@ -1173,18 +1173,19 @@ def _has_transcript(ep): @csrf_exempt @require_http_methods(['GET']) def episode_silences(request, pk): - """The episode's pauses for skipping silence in the browser: - {"ready": true, "spans": [[start, end], …]} once found; the first request - starts the search on the server ({"ready": false}, ask again later).""" + """What to cut out of the episode's pauses for skipping silence in the + browser: {"ready": true, "v": 2, "spans": [[from, to], …]} once found (the + parts to leave out, podcasts/silence.py); the first request starts the search + on the server ({"ready": false}, ask again later).""" if not request.user.is_authenticated: return JsonResponse({'error': 'authentication required'}, status=401) ep = PodcastEpisode.objects.filter(pk=pk, feed__user=request.user).only('id', 'silences').first() if ep is None: return JsonResponse({'error': 'not found'}, status=404) - if ep.silences is None: + if not silence_mod.current(ep.silences): # none yet, or found by older rules silence_mod.ensure(ep.pk) return JsonResponse({'ready': False}) - return JsonResponse({'ready': True, 'spans': ep.silences.get('spans', []), + return JsonResponse({'ready': True, 'v': silence_mod.VERSION, 'spans': ep.silences.get('spans', []), **({'error': ep.silences['error']} if ep.silences.get('error') else {})}) diff --git a/static/js/podcasts-web.js b/static/js/podcasts-web.js index a129f86..108da64 100644 --- a/static/js/podcasts-web.js +++ b/static/js/podcasts-web.js @@ -691,7 +691,7 @@ function applyKind() { const rate = speedFor(curFeed()); audio.defaultPlaybackRate = rate; - if (audio.playbackRate !== rate) audio.playbackRate = rate; + if (!silenceFast && audio.playbackRate !== rate) audio.playbackRate = rate; // else: after the pause audio.preservesPitch = true; applyVolume(); } @@ -949,10 +949,10 @@ }, 1000); // ---- skip silence ---------------------------------------------------------------------- - // A browser can't listen to most podcast servers' audio (no CORS), so the server finds an - // episode's pauses once (podcasts/silence.py, it takes a little while the first time) and - // the player jumps over them, a little of each kept. The podcast's own choice, else this - // browser's (as in the app, PlaybackRules.skipSilence). + // A browser can't listen to most podcast servers' audio (no CORS), so the server works out + // once per episode which parts of its pauses go (podcasts/silence.py, the same rules as the + // app's PauseCutter: a fifth of each pause stays). It takes a little while the first time. + // The podcast's own choice, else this browser's (as in the app, PlaybackRules.skipSilence). function skipSilenceFor(feedId) { const f = feed(feedId); return f && f.skip_silence != null ? !!f.skip_silence : !!prefs.skip_silence; @@ -960,6 +960,7 @@ const silence = {id: null, spans: null, asked: 0}; + // [[from, to], …] in seconds: what to leave out of this episode, once known. function silenceSpans() { const id = curId(); if (id == null || !skipSilenceFor(curFeed())) return null; @@ -968,71 +969,104 @@ silence.asked = Date.now(); fetch(`/podcasts/episodes/${id}/silences/`, {credentials: 'same-origin', cache: 'no-store'}) .then(r => (r.ok ? r.json() : null)) - .then(d => { if (d && d.ready && silence.id === id) silence.spans = d.spans || []; }) + .then(d => { if (d && d.ready && d.v === 2 && silence.id === id) silence.spans = d.spans || []; }) .catch(() => {}); } return silence.spans; } - // A pause is run through fast rather than jumped over: a jump in a streamed - // file waits for the network again (a hitch you hear), a faster rate doesn't. - // A little of each pause stays at both ends; only a long one is still jumped. - // Checked every 40 ms while playing (timeupdate comes only every 250 ms, too - // late for a short pause). A browser tab in the background gets its timers - // slowed down to a second or more — speech at four times the speed for that - // long would be worse than a hitch, so there it jumps as before. - const SILENCE_RATE = 4; // × the podcast's own speed, inside a pause - const SILENCE_KEEP = 0.12; // s of each pause kept at either end - const SILENCE_JUMP = 3; // s: a pause longer than this is jumped over + // Such a part is run through faster rather than jumped over: a jump in a streamed file waits + // for the network again (a hitch you hear), a faster rate doesn't. Only a long part is still + // jumped. Start and end are timed (timeupdate comes only every 250 ms), but not to the + // millisecond: a timer is a few ms late and currentTime moves in steps of 10–20 ms, which at + // 8× is up to 200 ms of speech. So no faster than 4× (Chromium mutes above that: a word run + // into would be gone, the room tone cut off), and back to normal early enough that the + // switch lands in the pause — per part, the rate that leaves out the most of it that way. + // A browser tab in the background gets its timers slowed to a second or more — speech run + // fast for that long would be worse than a hitch, so there it jumps. + const SILENCE_RATES = [2, 2.5, 3, 4]; + const SILENCE_SLACK = 0.025; // s of real time the switch back may come late + const SILENCE_GAIN = 0.01; // s: a part that would save less is played as it is + const SILENCE_JUMP = 2.5; // s: a longer part is jumped over let silenceFast = false; + let silenceEnd = 0; // where the part run through now is left + let silenceDone = -1; // the end of the part last run through (not entered again) let silenceTimer = null; - // The end of the pause the player is in (less what stays of it), or null. - function silenceEndAt(t) { - const spans = silenceSpans(); - if (!spans) return null; - for (const [start, end] of spans) { // in order - if (start + SILENCE_KEEP > t) return null; - if (t < end - SILENCE_KEEP) return end - SILENCE_KEEP; - } - return null; - } - function silenceNormal() { if (!silenceFast) return; silenceFast = false; audio.playbackRate = speedFor(curFeed()); } - function silenceTick() { - if (!loaded || audio.paused || audio.seeking) { silenceNormal(); return; } - const t = audio.currentTime; - const endAt = silenceEndAt(t); - if (endAt === null) { silenceNormal(); return; } - const timersFine = !document.hidden || !!window.dioraDesktop; // the desktop app never slows them - if (endAt - t > SILENCE_JUMP || !timersFine) { - silenceNormal(); - try { audio.currentTime = endAt; } catch (e) {} - } else { - const fast = Math.min(speedFor(curFeed()) * SILENCE_RATE, 8); - silenceFast = true; - if (audio.playbackRate !== fast) audio.playbackRate = fast; // also after applyKind reset it + function silenceSchedule(ms) { + clearTimeout(silenceTimer); + silenceTimer = setTimeout(silenceTick, Math.max(5, Math.min(ms, 1000))); + } + + // The rate for running through the rest of a part, [t, to), and where to leave it, or null. + function silencePlan(t, to, speed) { + let best = null; + for (const rate of SILENCE_RATES) { + if (rate < speed * 1.25) continue; + const end = to - rate * SILENCE_SLACK; + const gain = (end - t) * (1 / speed - 1 / rate); + if (gain >= SILENCE_GAIN && (!best || gain > best.gain)) best = {rate, end, gain}; } + return best; + } + + function silenceTick() { + silenceTimer = null; + if (!loaded || audio.paused || audio.seeking) { silenceNormal(); return; } + const spans = silenceSpans(); + if (!spans) { silenceNormal(); silenceSchedule(1000); return; } + const t = audio.currentTime; + const speed = speedFor(curFeed()) || 1; + const timersFine = !document.hidden || !!window.dioraDesktop; // the desktop app never slows them + let next = null; + for (const [from, to] of spans) { // in order + if (to <= t || to === silenceDone) continue; + if (from > t) { next = from; break; } + // Inside a part that goes. + if (to - t > SILENCE_JUMP || !timersFine) { + silenceNormal(); + silenceDone = to; + try { audio.currentTime = to; } catch (e) {} + silenceSchedule(50); + return; + } + if (silenceFast && t < silenceEnd) { // under way: keep the rate chosen on the way in + silenceSchedule((silenceEnd - t) / audio.playbackRate * 1000); + return; + } + const plan = silenceFast ? null : silencePlan(t, to, speed); + if (!plan) { silenceDone = to; continue; } // too little left of it: played as it is + silenceFast = true; + silenceEnd = plan.end; + audio.playbackRate = plan.rate; + silenceSchedule((plan.end - t) / plan.rate * 1000); + return; + } + silenceNormal(); + if (next !== null) silenceSchedule((next - t) / (audio.playbackRate || 1) * 1000); } function silenceWatch() { - clearInterval(silenceTimer); + clearTimeout(silenceTimer); silenceTimer = null; - silenceNormal(); - if (loaded && !audio.paused && skipSilenceFor(curFeed())) silenceTimer = setInterval(silenceTick, 40); + if (loaded && !audio.paused && skipSilenceFor(curFeed())) silenceTick(); // stays fast inside a part + else silenceNormal(); } audio.addEventListener('playing', silenceWatch); audio.addEventListener('pause', silenceWatch); audio.addEventListener('ended', silenceWatch); audio.addEventListener('emptied', silenceWatch); - // Back from the background, or the setting changed: the timer may have to start. + audio.addEventListener('seeked', () => { silenceDone = -1; silenceWatch(); }); // a part again + audio.addEventListener('ratechange', () => { if (!silenceFast && silenceTimer) silenceWatch(); }); + // Back from the background, or the setting changed: start over. document.addEventListener('visibilitychange', silenceWatch); - audio.addEventListener('timeupdate', () => { if (!silenceTimer && !audio.paused) silenceWatch(); else silenceTick(); }); + audio.addEventListener('timeupdate', () => { if (!silenceTimer && !audio.paused) silenceWatch(); }); // ---- radio and podcasts take turns --------------------------------------------------- diff --git a/static/js/sw.js b/static/js/sw.js index efabfca..5fe14c6 100644 --- a/static/js/sw.js +++ b/static/js/sw.js @@ -2,7 +2,7 @@ * diora service worker — caches the app shell for offline use. */ -const CACHE = 'diora-v109'; +const CACHE = 'diora-v110'; const SHELL = [ '/static/css/app.css', '/static/js/app/01-core.js',