123 lines
6.4 KiB
Python
123 lines
6.4 KiB
Python
|
"""People in the user's podcasts (docs/podcasts.md, Stöbern): who is talked with, who gives a
|
|||
|
|
talk — found in episode titles and descriptions, no service asked.
|
|||
|
|
|
|||
|
|
A name is two or three capitalised words ("Peter Decker", "Ulrike Eifler", "Jean-Luc Mélenchon"),
|
|||
|
|
possibly with a particle ("Wolfgang von Goethe"). Two kinds of evidence:
|
|||
|
|
- strong: right after a cue that announces a person — "mit", "w/", "with", "feat.", "Gast:",
|
|||
|
|
"im Gespräch mit", "interview with", "von" in a talk's title, " - Name - " between dashes;
|
|||
|
|
- weak: anywhere else in a title or description.
|
|||
|
|
A name counts when it has strong evidence in at least STRONG_MIN episodes, or turns up in at least
|
|||
|
|
FEEDS_MIN podcasts (a person known across the user's world). Words that are no names (places,
|
|||
|
|
institutions, weekdays, sentence starts) are left out by a list and by the rule that a name's
|
|||
|
|
first word must not be one of them.
|
|||
|
|
"""
|
|||
|
|
import html
|
|||
|
|
import re
|
|||
|
|
from collections import Counter, defaultdict
|
|||
|
|
|
|||
|
|
_W = r"[A-ZÄÖÜÀ-Ý][a-zäöüßà-ÿ]+(?:-[A-ZÄÖÜÀ-Ý][a-zäöüßà-ÿ]+)?"
|
|||
|
|
_PART = r"(?:von|van|der|de|da|di|du|la|le|zu|ten|del|bin|al)"
|
|||
|
|
_NAME = rf"{_W}(?:\s+{_PART})?(?:\s+{_W}){{1,2}}"
|
|||
|
|
NAME_RE = re.compile(rf"(?<![\w\-])({_NAME})(?![\w])")
|
|||
|
|
CUE_RE = re.compile(
|
|||
|
|
r"(?i:\bmit\b|\bw/|\bwith\b|\bfeat\.?|\bft\.|\bGast:|\bGäste:|\bguest:|\binterview(?:t)? with\b|"
|
|||
|
|
r"\bim Gespräch mit\b|\bin conversation with\b|\bvon\b|\bby\b)\s*(" + _NAME + r")|\s[-–|]\s(" + _NAME + r")")
|
|||
|
|
# (only the cue ignores case: a name's words must start with a capital)
|
|||
|
|
|
|||
|
|
# First or last words that make a capitalised pair no person.
|
|||
|
|
NOT_NAME_WORDS = set('''
|
|||
|
|
Der Die Das Den Dem Des Ein Eine Einen Einem Einer Und Oder Aber Mit Von Nach Bei Für Auf Aus Im In Am Zum Zur
|
|||
|
|
Vor Seit Als Wie Wenn Teil Folge Episode Staffel Kapitel Band Live Special Bonus Trailer Podcast Interview
|
|||
|
|
Gespräch Vortrag Diskussion Lesung Hörspiel Feature Interviews Folgen Talk Show News Update Week Day Days
|
|||
|
|
The A An And Or But With From For After Before On At By To Of As It This That These Those Our Your Their
|
|||
|
|
Mr Mrs Ms Dr Prof Herr Frau Doktor Professor Sir Saint Sankt St
|
|||
|
|
Monday Tuesday Wednesday Thursday Friday Saturday Sunday Montag Dienstag Mittwoch Donnerstag Freitag
|
|||
|
|
Samstag Sonntag January February March April May June July August September October November December
|
|||
|
|
Januar Februar März Juni Juli Oktober Dezember Frühling Sommer Herbst Winter
|
|||
|
|
Deutschland Deutschlands Deutsche Deutschen Deutscher Europa Europas Europäische Europäischen Amerika
|
|||
|
|
Amerikas Usa China Chinas Russland Russlands Ukraine Israel Gaza Iran Frankreich Großbritannien England
|
|||
|
|
Berlin Hamburg München Wien Zürich Köln Leipzig Dresden London Paris New York Washington Moskau
|
|||
|
|
United States Kingdom Union Republic Republik Bank Partei Linke Linken Grüne Grünen Spd Cdu Afd Fdp Bsw
|
|||
|
|
Universität University College Institute Institut Stiftung Verlag Radio Deutschlandfunk Kultur Nova
|
|||
|
|
Zeitung Magazin Review Times Post News Press Media Netflix Youtube Spotify Apple Google Amazon Twitter
|
|||
|
|
Google Meta Tesla Marxismus Kapitalismus Sozialismus Kommunismus Faschismus Imperialismus Krieg Frieden
|
|||
|
|
Gott Jesus Christus Weltkrieg Revolution Kalter Kalten Krise Wahl Wahlen Bundestag Bundesregierung
|
|||
|
|
Remaster Remastered Audio Enhanced Edition Version Full Part Chapter Book Season Classic Klassiker
|
|||
|
|
Hegel Marx Engels Kant Nietzsche Freud Lacan
|
|||
|
|
Doku Hd Reupload Reaction Re View Neue Neuer Zu Gast Gäste My Last Free Speech Open Source Big Tech Cup World
|
|||
|
|
War Files Kanzler Bundeskanzler Präsident Präsidentin President Minister Ministerin Trumps Künstliche
|
|||
|
|
White House Middle East Silicon Valley Wall Street Mental Health Red Thread Critical Role Trio Infernal
|
|||
|
|
Asian Japan Nhk Arte Spiegel Zeit Faz Taz Bild Planet Money Lage Nation Inside Austria Today Daily Weekly
|
|||
|
|
Political Science Letzte Foreign Private Golden Keine Tag El Los Las San Santa Rio Fort Port Lake Mount
|
|||
|
|
'''.split())
|
|||
|
|
STRONG_MIN = 2
|
|||
|
|
FEEDS_MIN = 3
|
|||
|
|
|
|||
|
|
|
|||
|
|
def plain(text):
|
|||
|
|
text = re.sub(r'<[^>]+>', ' ', text or '')
|
|||
|
|
return re.sub(r'\s+', ' ', html.unescape(text)).strip()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _clean(name):
|
|||
|
|
words = name.split()
|
|||
|
|
# "Mit Peter Decker" etc.: drop leading words that are no name.
|
|||
|
|
while words and words[0] in NOT_NAME_WORDS:
|
|||
|
|
words = words[1:]
|
|||
|
|
while words and words[-1] in NOT_NAME_WORDS:
|
|||
|
|
words = words[:-1]
|
|||
|
|
caps = [w for w in words if w[0].isupper()]
|
|||
|
|
if len(caps) < 2 or any(w in NOT_NAME_WORDS for w in caps):
|
|||
|
|
return None
|
|||
|
|
if any(len(w) > 1 and w.isupper() for w in caps): # "WAS TUN", "ARTE Reportage": no person
|
|||
|
|
return None
|
|||
|
|
return ' '.join(words)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def mentions(title):
|
|||
|
|
"""(strong, weak): the names in one episode's title."""
|
|||
|
|
strong, weak = set(), set()
|
|||
|
|
for m in CUE_RE.finditer(title or ''):
|
|||
|
|
n = _clean(m.group(1) or m.group(2))
|
|||
|
|
if n:
|
|||
|
|
strong.add(n)
|
|||
|
|
for m in NAME_RE.finditer(title or ''):
|
|||
|
|
n = _clean(m.group(1))
|
|||
|
|
if n:
|
|||
|
|
weak.add(n)
|
|||
|
|
return strong, weak - strong
|
|||
|
|
|
|||
|
|
|
|||
|
|
def people(episodes, weights=None):
|
|||
|
|
"""episodes: [(episode_id, feed_id, title)]; weights: {feed_id: how much the user listens}
|
|||
|
|
(heard_365). The people found, the most relevant first: [{'name', 'episodes', 'feeds'}].
|
|||
|
|
|
|||
|
|
Only titles: descriptions carry credits and boilerplate. First names are learnt from the
|
|||
|
|
strong mentions ("mit Peter Decker": Peter); a name found elsewhere in a title counts only
|
|||
|
|
with such a first name."""
|
|||
|
|
found = [(ep_id, feed_id, *mentions(title)) for ep_id, feed_id, title in episodes]
|
|||
|
|
surnames = defaultdict(set)
|
|||
|
|
for _, _, strong, _ in found:
|
|||
|
|
for n in strong:
|
|||
|
|
w = n.split()
|
|||
|
|
surnames[w[0]].add(w[-1])
|
|||
|
|
# A first name: one that comes with at least two different surnames in strong mentions.
|
|||
|
|
first_names = {f for f, last in surnames.items() if len(last) >= 2 and f not in NOT_NAME_WORDS}
|
|||
|
|
eps = defaultdict(set)
|
|||
|
|
feeds = defaultdict(set)
|
|||
|
|
strong_n = Counter()
|
|||
|
|
for ep_id, feed_id, strong, weak in found:
|
|||
|
|
for n in strong:
|
|||
|
|
strong_n[n] += 1
|
|||
|
|
for n in {n for n in strong | weak if n.split()[0] in first_names}:
|
|||
|
|
eps[n].add(ep_id)
|
|||
|
|
feeds[n].add(feed_id)
|
|||
|
|
weights = weights or {}
|
|||
|
|
out = []
|
|||
|
|
for n, ids in eps.items():
|
|||
|
|
if strong_n[n] >= STRONG_MIN or len(feeds[n]) >= FEEDS_MIN:
|
|||
|
|
score = len(ids) + 5 * sum(min(weights.get(f, 0), 50) for f in feeds[n]) ** 0.5
|
|||
|
|
out.append({'name': n, 'episodes': len(ids), 'feeds': len(feeds[n]), 'score': round(score, 1)})
|
|||
|
|
out.sort(key=lambda p: (-p['score'], p['name']))
|
|||
|
|
return out
|