122 lines
6.4 KiB
Python
122 lines
6.4 KiB
Python
"""People in the user's podcasts (docs/podcasts.md, Stöbern): who is talked with, who gives a
|
||
talk — found in episode titles and descriptions, no service asked.
|
||
|
||
A name is two or three capitalised words ("Peter Decker", "Ulrike Eifler", "Jean-Luc Mélenchon"),
|
||
possibly with a particle ("Wolfgang von Goethe"). Two kinds of evidence:
|
||
- strong: right after a cue that announces a person — "mit", "w/", "with", "feat.", "Gast:",
|
||
"im Gespräch mit", "interview with", "von" in a talk's title, " - Name - " between dashes;
|
||
- weak: anywhere else in a title or description.
|
||
A name counts when it has strong evidence in at least STRONG_MIN episodes, or turns up in at least
|
||
FEEDS_MIN podcasts (a person known across the user's world). Words that are no names (places,
|
||
institutions, weekdays, sentence starts) are left out by a list and by the rule that a name's
|
||
first word must not be one of them.
|
||
"""
|
||
import html
|
||
import re
|
||
from collections import Counter, defaultdict
|
||
|
||
_W = r"[A-ZÄÖÜÀ-Ý][a-zäöüßà-ÿ]+(?:-[A-ZÄÖÜÀ-Ý][a-zäöüßà-ÿ]+)?"
|
||
_PART = r"(?:von|van|der|de|da|di|du|la|le|zu|ten|del|bin|al)"
|
||
_NAME = rf"{_W}(?:\s+{_PART})?(?:\s+{_W}){{1,2}}"
|
||
NAME_RE = re.compile(rf"(?<![\w\-])({_NAME})(?![\w])")
|
||
CUE_RE = re.compile(
|
||
r"(?i:\bmit\b|\bw/|\bwith\b|\bfeat\.?|\bft\.|\bGast:|\bGäste:|\bguest:|\binterview(?:t)? with\b|"
|
||
r"\bim Gespräch mit\b|\bin conversation with\b|\bvon\b|\bby\b)\s*(" + _NAME + r")|\s[-–|]\s(" + _NAME + r")")
|
||
# (only the cue ignores case: a name's words must start with a capital)
|
||
|
||
# First or last words that make a capitalised pair no person.
|
||
NOT_NAME_WORDS = set('''
|
||
Der Die Das Den Dem Des Ein Eine Einen Einem Einer Und Oder Aber Mit Von Nach Bei Für Auf Aus Im In Am Zum Zur
|
||
Vor Seit Als Wie Wenn Teil Folge Episode Staffel Kapitel Band Live Special Bonus Trailer Podcast Interview
|
||
Gespräch Vortrag Diskussion Lesung Hörspiel Feature Interviews Folgen Talk Show News Update Week Day Days
|
||
The A An And Or But With From For After Before On At By To Of As It This That These Those Our Your Their
|
||
Mr Mrs Ms Dr Prof Herr Frau Doktor Professor Sir Saint Sankt St
|
||
Monday Tuesday Wednesday Thursday Friday Saturday Sunday Montag Dienstag Mittwoch Donnerstag Freitag
|
||
Samstag Sonntag January February March April May June July August September October November December
|
||
Januar Februar März Juni Juli Oktober Dezember Frühling Sommer Herbst Winter
|
||
Deutschland Deutschlands Deutsche Deutschen Deutscher Europa Europas Europäische Europäischen Amerika
|
||
Amerikas Usa China Chinas Russland Russlands Ukraine Israel Gaza Iran Frankreich Großbritannien England
|
||
Berlin Hamburg München Wien Zürich Köln Leipzig Dresden London Paris New York Washington Moskau
|
||
United States Kingdom Union Republic Republik Bank Partei Linke Linken Grüne Grünen Spd Cdu Afd Fdp Bsw
|
||
Universität University College Institute Institut Stiftung Verlag Radio Deutschlandfunk Kultur Nova
|
||
Zeitung Magazin Review Times Post News Press Media Netflix Youtube Spotify Apple Google Amazon Twitter
|
||
Google Meta Tesla Marxismus Kapitalismus Sozialismus Kommunismus Faschismus Imperialismus Krieg Frieden
|
||
Gott Jesus Christus Weltkrieg Revolution Kalter Kalten Krise Wahl Wahlen Bundestag Bundesregierung
|
||
Remaster Remastered Audio Enhanced Edition Version Full Part Chapter Book Season Classic Klassiker
|
||
Hegel Marx Engels Kant Nietzsche Freud Lacan
|
||
Doku Hd Reupload Reaction Re View Neue Neuer Zu Gast Gäste My Last Free Speech Open Source Big Tech Cup World
|
||
War Files Kanzler Bundeskanzler Präsident Präsidentin President Minister Ministerin Trumps Künstliche
|
||
White House Middle East Silicon Valley Wall Street Mental Health Red Thread Critical Role Trio Infernal
|
||
Asian Japan Nhk Arte Spiegel Zeit Faz Taz Bild Planet Money Lage Nation Inside Austria Today Daily Weekly
|
||
Political Science Letzte Foreign Private Golden Keine Tag El Los Las San Santa Rio Fort Port Lake Mount
|
||
'''.split())
|
||
STRONG_MIN = 2
|
||
FEEDS_MIN = 3
|
||
|
||
|
||
def plain(text):
|
||
text = re.sub(r'<[^>]+>', ' ', text or '')
|
||
return re.sub(r'\s+', ' ', html.unescape(text)).strip()
|
||
|
||
|
||
def _clean(name):
|
||
words = name.split()
|
||
# "Mit Peter Decker" etc.: drop leading words that are no name.
|
||
while words and words[0] in NOT_NAME_WORDS:
|
||
words = words[1:]
|
||
while words and words[-1] in NOT_NAME_WORDS:
|
||
words = words[:-1]
|
||
caps = [w for w in words if w[0].isupper()]
|
||
if len(caps) < 2 or any(w in NOT_NAME_WORDS for w in caps):
|
||
return None
|
||
if any(len(w) > 1 and w.isupper() for w in caps): # "WAS TUN", "ARTE Reportage": no person
|
||
return None
|
||
return ' '.join(words)
|
||
|
||
|
||
def mentions(title):
|
||
"""(strong, weak): the names in one episode's title."""
|
||
strong, weak = set(), set()
|
||
for m in CUE_RE.finditer(title or ''):
|
||
n = _clean(m.group(1) or m.group(2))
|
||
if n:
|
||
strong.add(n)
|
||
for m in NAME_RE.finditer(title or ''):
|
||
n = _clean(m.group(1))
|
||
if n:
|
||
weak.add(n)
|
||
return strong, weak - strong
|
||
|
||
|
||
def people(episodes, weights=None):
|
||
"""episodes: [(episode_id, feed_id, title)]; weights: {feed_id: how much the user listens}
|
||
(heard_365). The people found, the most relevant first: [{'name', 'episodes', 'feeds'}].
|
||
|
||
Only titles: descriptions carry credits and boilerplate. First names are learnt from the
|
||
strong mentions ("mit Peter Decker": Peter); a name found elsewhere in a title counts only
|
||
with such a first name."""
|
||
found = [(ep_id, feed_id, *mentions(title)) for ep_id, feed_id, title in episodes]
|
||
surnames = defaultdict(set)
|
||
for _, _, strong, _ in found:
|
||
for n in strong:
|
||
w = n.split()
|
||
surnames[w[0]].add(w[-1])
|
||
# A first name: one that comes with at least two different surnames in strong mentions.
|
||
first_names = {f for f, last in surnames.items() if len(last) >= 2 and f not in NOT_NAME_WORDS}
|
||
eps = defaultdict(set)
|
||
feeds = defaultdict(set)
|
||
strong_n = Counter()
|
||
for ep_id, feed_id, strong, weak in found:
|
||
for n in strong:
|
||
strong_n[n] += 1
|
||
for n in {n for n in strong | weak if n.split()[0] in first_names}:
|
||
eps[n].add(ep_id)
|
||
feeds[n].add(feed_id)
|
||
weights = weights or {}
|
||
out = []
|
||
for n, ids in eps.items():
|
||
if strong_n[n] >= STRONG_MIN or len(feeds[n]) >= FEEDS_MIN:
|
||
score = len(ids) + 5 * sum(min(weights.get(f, 0), 50) for f in feeds[n]) ** 0.5
|
||
out.append({'name': n, 'episodes': len(ids), 'feeds': len(feeds[n]), 'score': round(score, 1)})
|
||
out.sort(key=lambda p: (-p['score'], p['name']))
|
||
return out
|