diora-web/feeds/models.py

from django.contrib.auth.models import User
from django.db import models


class Source(models.Model):
    """A subscribed feed. Every source is either news or a blog, and that shapes
    everything after it: news gets bundled into stories, shows up in the
    "Seit letztem Mal" briefing and expires after a few days; blog articles stay
    until read and are listed per blog."""
    NEWS = 'news'
    BLOG = 'blog'
    KINDS = [(NEWS, 'Nachrichten'), (BLOG, 'Blog')]

    # Whether the whole article is fetched from its page: "auto" when the feed
    # only carries a teaser.
    FULLTEXT_AUTO = 'auto'
    FULLTEXT_ALWAYS = 'always'
    FULLTEXT_NEVER = 'never'
    FULLTEXTS = [(FULLTEXT_AUTO, 'Automatisch'), (FULLTEXT_ALWAYS, 'Immer'), (FULLTEXT_NEVER, 'Nie')]

    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_sources')
    url = models.URLField(max_length=1000)
    title = models.CharField(max_length=300)
    site_url = models.URLField(max_length=1000, blank=True)
    icon_url = models.URLField(max_length=1000, blank=True)
    kind = models.CharField(max_length=10, choices=KINDS, default=NEWS)
    # News: the section it belongs to (Politik, Tech …), groups the briefing.
    category = models.CharField(max_length=100, blank=True)
    fulltext = models.CharField(max_length=10, choices=FULLTEXTS, default=FULLTEXT_AUTO)
    added_at = models.DateTimeField(auto_now_add=True)

    # Refresh bookkeeping, as for podcast feeds (see podcasts.views._fetch_feed).
    last_refreshed_at = models.DateTimeField(null=True, blank=True)
    http_etag = models.CharField(max_length=500, blank=True)
    http_last_modified = models.CharField(max_length=100, blank=True)
    http_body_hash = models.CharField(max_length=64, blank=True)
    last_error = models.CharField(max_length=500, blank=True)
    last_error_at = models.DateTimeField(null=True, blank=True)
    error_count = models.PositiveIntegerField(default=0)

    class Meta:
        unique_together = ('user', 'url')
        ordering = ['title']

    def __str__(self):
        return f'{self.title} ({self.kind}, user={self.user_id})'


class Story(models.Model):
    """News items about the same event from one or more sources (see cluster.py)."""
    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_stories')
    # The item the story is shown with (the first one, or the longest text).
    lead = models.ForeignKey('Item', null=True, blank=True, on_delete=models.SET_NULL, related_name='+')
    first_at = models.DateTimeField()
    last_at = models.DateTimeField(db_index=True)
    size = models.PositiveIntegerField(default=1)      # items in it
    sources = models.PositiveIntegerField(default=1)   # distinct sources among them
    # Words that describe it (space-separated, for matching new items).
    terms = models.TextField(blank=True)

    class Meta:
        ordering = ['-last_at']


class Item(models.Model):
    source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name='items')
    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_items')
    kind = models.CharField(max_length=10, choices=Source.KINDS)  # the source's, at the time
    guid_hash = models.CharField(max_length=64)  # sha256 of the entry's id (or link)
    url = models.URLField(max_length=2000, blank=True)
    title = models.CharField(max_length=500)
    author = models.CharField(max_length=300, blank=True)
    published = models.DateTimeField(db_index=True)
    fetched_at = models.DateTimeField(auto_now_add=True)
    summary = models.TextField(blank=True)   # plain text, short
    content = models.TextField(blank=True)   # sanitized HTML
    # The content came from the article's page, not the feed.
    content_full = models.BooleanField(default=False)
    # Fetching the page failed or gave nothing: don't try on every open.
    fulltext_failed = models.BooleanField(default=False)
    # The page showed only the free part (paywall): fetched again once there's
    # a subscriber login for the site (logins.py).
    paywalled = models.BooleanField(default=False)
    image_url = models.URLField(max_length=2000, blank=True)
    read_at = models.DateTimeField(null=True, blank=True)
    starred = models.BooleanField(default=False)
    story = models.ForeignKey(Story, null=True, blank=True, on_delete=models.SET_NULL, related_name='items')
    # Expired news: the row stays a while without its text, so the entry isn't
    # taken in again as new while the feed still carries it (refresh.expire).
    gone = models.BooleanField(default=False)

    class Meta:
        unique_together = ('source', 'guid_hash')
        indexes = [
            models.Index(fields=['user', 'kind', 'published']),
            models.Index(fields=['user', 'kind', 'read_at']),
        ]
        ordering = ['-published']

    def __str__(self):
        return self.title


class SiteLogin(models.Model):
    """A subscriber login for a paywalled site (logins.py); the password encrypted."""
    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='feed_site_logins')
    domain = models.CharField(max_length=200)
    username = models.CharField(max_length=300)
    secret = models.TextField()   # the password, Fernet-encrypted
    ok = models.BooleanField(default=True)
    last_error = models.CharField(max_length=300, blank=True)
    checked_at = models.DateTimeField(null=True, blank=True)

    class Meta:
        unique_together = ('user', 'domain')


class FeedSettings(models.Model):
    user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='feed_settings')
    # The last time the news briefing was seen; what came after is "Seit letztem Mal".
    news_seen_at = models.DateTimeField(null=True, blank=True)
    # News older than this goes (unless starred).
    news_keep_days = models.PositiveIntegerField(default=2)