"""New papers for a project's Radar (docs/projects.md): PubMed, arXiv, bioRxiv, medRxiv. What is searched comes from the radar's config (projects/radar_presets.py): PubMed queries per topic, one arXiv query, the preprint servers filtered by a keyword pattern. collect() returns (records, errors) and keeps nothing — the caller remembers what was reported, so a failed run loses nothing. Ported from F:\\neuropolitik\\tools\\fetch_papers.py (Neuropolitik session).""" import json import re import time import xml.etree.ElementTree as ET from datetime import date, timedelta import requests UA = 'diora-radar/1.0 (personal research digest; https://diora.creamfresh.xyz)' ABSTRACT_CHARS = 1200 RXIV_SERVERS = ('biorxiv', 'medrxiv') def get(url, params=None, retries=3, timeout=60): for attempt in range(retries): try: r = requests.get(url, params=params, headers={'User-Agent': UA}, timeout=timeout) r.raise_for_status() return r.text except requests.RequestException: if attempt == retries - 1: raise time.sleep(3 * (attempt + 1)) def clip(text): text = re.sub(r'\s+', ' ', text or '').strip() return text if len(text) <= ABSTRACT_CHARS else text[:ABSTRACT_CHARS].rsplit(' ', 1)[0] + ' …' def fetch_pubmed(queries, days): base = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/' records = [] for topic, query in queries.items(): ids = json.loads(get(base + 'esearch.fcgi', {'db': 'pubmed', 'term': query, 'reldate': days, 'datetype': 'edat', 'retmax': 300, 'retmode': 'json'})) ids = ids['esearchresult']['idlist'] for i in range(0, len(ids), 100): root = ET.fromstring(get(base + 'efetch.fcgi', {'db': 'pubmed', 'id': ','.join(ids[i:i + 100]), 'retmode': 'xml'})) for art in root.iter('PubmedArticle'): pmid = art.findtext('.//PMID') t = art.find('.//ArticleTitle') abstract = ' '.join(''.join(a.itertext()) for a in art.findall('.//Abstract/AbstractText')) doi = next((e.text for e in art.findall('.//ArticleId') if e.get('IdType') == 'doi'), None) authors = art.findall('.//Author') records.append({ 'id': f'pmid:{pmid}', 'doi': doi, 'source': 'PubMed', 'topic': topic, 'title': clip(''.join(t.itertext()) if t is not None else ''), 'journal': art.findtext('.//Journal/Title'), 'first_author': authors[0].findtext('LastName') if authors else None, 'n_authors': len(authors), 'pub_types': [p.text for p in art.findall('.//PublicationType')], 'date': art.findtext('.//PubDate/Year'), 'url': f'https://pubmed.ncbi.nlm.nih.gov/{pmid}/', 'abstract': clip(abstract), }) time.sleep(0.4) # NCBI: 3 requests a second without a key return records def fetch_arxiv(query, days): cutoff = (date.today() - timedelta(days=days)).isoformat() xml = get('https://export.arxiv.org/api/query', {'search_query': query, 'sortBy': 'submittedDate', 'sortOrder': 'descending', 'max_results': 150}) ns = {'a': 'http://www.w3.org/2005/Atom', 'arxiv': 'http://arxiv.org/schemas/atom'} records = [] for e in ET.fromstring(xml).findall('a:entry', ns): published = e.findtext('a:published', '', ns)[:10] if published < cutoff: continue arxiv_id = e.findtext('a:id', '', ns).rsplit('/abs/', 1)[-1] cat = e.find('arxiv:primary_category', ns) authors = e.findall('a:author', ns) records.append({ 'id': 'arxiv:' + re.sub(r'v\d+$', '', arxiv_id), 'doi': None, 'source': 'arXiv', 'topic': None, 'title': clip(e.findtext('a:title', '', ns)), 'journal': f"arXiv {cat.get('term') if cat is not None else ''}".strip(), 'first_author': authors[0].findtext('a:name', '', ns) if authors else None, 'n_authors': len(authors), 'pub_types': ['Preprint'], 'date': published, 'url': f'https://arxiv.org/abs/{arxiv_id}', 'abstract': clip(e.findtext('a:summary', '', ns)), }) return records def fetch_rxiv(server, days, keywords): start = (date.today() - timedelta(days=days)).isoformat() end = date.today().isoformat() records, cursor, total = [], 0, None while total is None or cursor < total: data = json.loads(get(f'https://api.biorxiv.org/details/{server}/{start}/{end}/{cursor}')) msg = data['messages'][0] if msg.get('status') != 'ok': break total = int(msg.get('total', 0)) batch = data.get('collection', []) if not batch: break for p in batch: if not keywords.search(f"{p.get('title', '')} {p.get('abstract', '')}"): continue authors = (p.get('authors') or '').split(';') records.append({ 'id': 'doi:' + p['doi'], 'doi': p['doi'], 'source': server, 'topic': None, 'title': clip(p.get('title')), 'journal': f"{server} ({p.get('category')})", 'first_author': authors[0], 'n_authors': len(authors), 'pub_types': ['Preprint'], 'date': p.get('date'), 'url': f"https://www.{server}.org/content/{p['doi']}v{p.get('version', 1)}", 'abstract': clip(p.get('abstract')), 'institution': p.get('author_corresponding_institution'), }) cursor += len(batch) time.sleep(0.3) return records def collect(config, days, seen=frozenset()): """Every source of the config; doubles (same id, DOI or title) and what was reported before (seen) left out. (records, errors).""" keywords = re.compile(config.get('keywords') or r'(?!)', re.I) jobs = [] if config.get('pubmed'): jobs.append(('PubMed', lambda: fetch_pubmed(config['pubmed'], days))) if config.get('arxiv'): jobs.append(('arXiv', lambda: fetch_arxiv(config['arxiv'], days))) for server in config.get('rxiv', RXIV_SERVERS) if config.get('keywords') else (): jobs.append((server, lambda s=server: fetch_rxiv(s, days, keywords))) records, errors = [], [] for name, fn in jobs: try: records += fn() except Exception as e: # noqa: BLE001 - a source down mustn't stop the others errors.append(f'{name}: {e}') unique, keys = [], set() for r in records: k = {r['id'], (r['doi'] or '').lower(), re.sub(r'\W', '', r['title'].lower())} - {''} if keys & k or r['id'] in seen or (r['doi'] and f"doi:{r['doi']}" in seen): continue keys |= k unique.append(r) return unique, errors