143 lines
6.8 KiB
Python
143 lines
6.8 KiB
Python
"""New papers for a project's Radar (docs/projects.md): PubMed, arXiv, bioRxiv, medRxiv.
|
|
|
|
What is searched comes from the radar's config (projects/radar_presets.py): PubMed queries per
|
|
topic, one arXiv query, the preprint servers filtered by a keyword pattern. collect() returns
|
|
(records, errors) and keeps nothing — the caller remembers what was reported, so a failed run
|
|
loses nothing. Ported from F:\\neuropolitik\\tools\\fetch_papers.py (Neuropolitik session)."""
|
|
import json
|
|
import re
|
|
import time
|
|
import xml.etree.ElementTree as ET
|
|
from datetime import date, timedelta
|
|
|
|
import requests
|
|
|
|
UA = 'diora-radar/1.0 (personal research digest; https://diora.creamfresh.xyz)'
|
|
ABSTRACT_CHARS = 1200
|
|
RXIV_SERVERS = ('biorxiv', 'medrxiv')
|
|
|
|
|
|
def get(url, params=None, retries=3, timeout=60):
|
|
for attempt in range(retries):
|
|
try:
|
|
r = requests.get(url, params=params, headers={'User-Agent': UA}, timeout=timeout)
|
|
r.raise_for_status()
|
|
return r.text
|
|
except requests.RequestException:
|
|
if attempt == retries - 1:
|
|
raise
|
|
time.sleep(3 * (attempt + 1))
|
|
|
|
|
|
def clip(text):
|
|
text = re.sub(r'\s+', ' ', text or '').strip()
|
|
return text if len(text) <= ABSTRACT_CHARS else text[:ABSTRACT_CHARS].rsplit(' ', 1)[0] + ' …'
|
|
|
|
|
|
def fetch_pubmed(queries, days):
|
|
base = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
|
|
records = []
|
|
for topic, query in queries.items():
|
|
ids = json.loads(get(base + 'esearch.fcgi', {'db': 'pubmed', 'term': query, 'reldate': days,
|
|
'datetype': 'edat', 'retmax': 300, 'retmode': 'json'}))
|
|
ids = ids['esearchresult']['idlist']
|
|
for i in range(0, len(ids), 100):
|
|
root = ET.fromstring(get(base + 'efetch.fcgi', {'db': 'pubmed', 'id': ','.join(ids[i:i + 100]),
|
|
'retmode': 'xml'}))
|
|
for art in root.iter('PubmedArticle'):
|
|
pmid = art.findtext('.//PMID')
|
|
t = art.find('.//ArticleTitle')
|
|
abstract = ' '.join(''.join(a.itertext()) for a in art.findall('.//Abstract/AbstractText'))
|
|
doi = next((e.text for e in art.findall('.//ArticleId') if e.get('IdType') == 'doi'), None)
|
|
authors = art.findall('.//Author')
|
|
records.append({
|
|
'id': f'pmid:{pmid}', 'doi': doi, 'source': 'PubMed', 'topic': topic,
|
|
'title': clip(''.join(t.itertext()) if t is not None else ''),
|
|
'journal': art.findtext('.//Journal/Title'),
|
|
'first_author': authors[0].findtext('LastName') if authors else None,
|
|
'n_authors': len(authors), 'pub_types': [p.text for p in art.findall('.//PublicationType')],
|
|
'date': art.findtext('.//PubDate/Year'), 'url': f'https://pubmed.ncbi.nlm.nih.gov/{pmid}/',
|
|
'abstract': clip(abstract),
|
|
})
|
|
time.sleep(0.4) # NCBI: 3 requests a second without a key
|
|
return records
|
|
|
|
|
|
def fetch_arxiv(query, days):
|
|
cutoff = (date.today() - timedelta(days=days)).isoformat()
|
|
xml = get('https://export.arxiv.org/api/query', {'search_query': query, 'sortBy': 'submittedDate',
|
|
'sortOrder': 'descending', 'max_results': 150})
|
|
ns = {'a': 'http://www.w3.org/2005/Atom', 'arxiv': 'http://arxiv.org/schemas/atom'}
|
|
records = []
|
|
for e in ET.fromstring(xml).findall('a:entry', ns):
|
|
published = e.findtext('a:published', '', ns)[:10]
|
|
if published < cutoff:
|
|
continue
|
|
arxiv_id = e.findtext('a:id', '', ns).rsplit('/abs/', 1)[-1]
|
|
cat = e.find('arxiv:primary_category', ns)
|
|
authors = e.findall('a:author', ns)
|
|
records.append({
|
|
'id': 'arxiv:' + re.sub(r'v\d+$', '', arxiv_id), 'doi': None, 'source': 'arXiv', 'topic': None,
|
|
'title': clip(e.findtext('a:title', '', ns)),
|
|
'journal': f"arXiv {cat.get('term') if cat is not None else ''}".strip(),
|
|
'first_author': authors[0].findtext('a:name', '', ns) if authors else None,
|
|
'n_authors': len(authors), 'pub_types': ['Preprint'], 'date': published,
|
|
'url': f'https://arxiv.org/abs/{arxiv_id}', 'abstract': clip(e.findtext('a:summary', '', ns)),
|
|
})
|
|
return records
|
|
|
|
|
|
def fetch_rxiv(server, days, keywords):
|
|
start = (date.today() - timedelta(days=days)).isoformat()
|
|
end = date.today().isoformat()
|
|
records, cursor, total = [], 0, None
|
|
while total is None or cursor < total:
|
|
data = json.loads(get(f'https://api.biorxiv.org/details/{server}/{start}/{end}/{cursor}'))
|
|
msg = data['messages'][0]
|
|
if msg.get('status') != 'ok':
|
|
break
|
|
total = int(msg.get('total', 0))
|
|
batch = data.get('collection', [])
|
|
if not batch:
|
|
break
|
|
for p in batch:
|
|
if not keywords.search(f"{p.get('title', '')} {p.get('abstract', '')}"):
|
|
continue
|
|
authors = (p.get('authors') or '').split(';')
|
|
records.append({
|
|
'id': 'doi:' + p['doi'], 'doi': p['doi'], 'source': server, 'topic': None,
|
|
'title': clip(p.get('title')), 'journal': f"{server} ({p.get('category')})",
|
|
'first_author': authors[0], 'n_authors': len(authors), 'pub_types': ['Preprint'],
|
|
'date': p.get('date'), 'url': f"https://www.{server}.org/content/{p['doi']}v{p.get('version', 1)}",
|
|
'abstract': clip(p.get('abstract')), 'institution': p.get('author_corresponding_institution'),
|
|
})
|
|
cursor += len(batch)
|
|
time.sleep(0.3)
|
|
return records
|
|
|
|
|
|
def collect(config, days, seen=frozenset()):
|
|
"""Every source of the config; doubles (same id, DOI or title) and what was reported
|
|
before (seen) left out. (records, errors)."""
|
|
keywords = re.compile(config.get('keywords') or r'(?!)', re.I)
|
|
jobs = []
|
|
if config.get('pubmed'):
|
|
jobs.append(('PubMed', lambda: fetch_pubmed(config['pubmed'], days)))
|
|
if config.get('arxiv'):
|
|
jobs.append(('arXiv', lambda: fetch_arxiv(config['arxiv'], days)))
|
|
for server in config.get('rxiv', RXIV_SERVERS) if config.get('keywords') else ():
|
|
jobs.append((server, lambda s=server: fetch_rxiv(s, days, keywords)))
|
|
records, errors = [], []
|
|
for name, fn in jobs:
|
|
try:
|
|
records += fn()
|
|
except Exception as e: # noqa: BLE001 - a source down mustn't stop the others
|
|
errors.append(f'{name}: {e}')
|
|
unique, keys = [], set()
|
|
for r in records:
|
|
k = {r['id'], (r['doi'] or '').lower(), re.sub(r'\W', '', r['title'].lower())} - {''}
|
|
if keys & k or r['id'] in seen or (r['doi'] and f"doi:{r['doi']}" in seen):
|
|
continue
|
|
keys |= k
|
|
unique.append(r)
|
|
return unique, errors
|