code

Author: subdavis

rss_glue/feeds/feed.py

@@ -403,3 +403,36 @@ def posts(
             self.post(post.id) for post in self.source.posts(limit=limit, start=start, end=end)
         ]
         return [post for post in posts if post is not None]
+
+
+class AugmentFeed(BaseFeed, ABC):
+    """
+    A feed that augments another feed with additional data.
+    """
+
+    source: BaseFeed
+    post_cls: type[ReferenceFeedItem] = ReferenceFeedItem
+    name: str = "augment"
+    limit: int
+
+    def __init__(self, source: BaseFeed, limit: int = 10):
+        self.source = source
+        self.limit = limit
+        self.author = source.author
+        self.origin_url = source.origin_url
+        self.title = source.title
+        super().__init__()
+
+    @property
+    def namespace(self):
+        return f"{self.name}_{self.source.namespace}"
+    
+    def sources(self) -> Iterable[BaseFeed]:
+        yield self.source
+
+    def next_update(self, force):
+        source_next_update, source_needs_update = self.source.next_update(force)
+        if self.source.last_updated >= self.last_updated:
+            return self.source.last_updated, True
+        return source_next_update, source_needs_update
+

rss_glue/feeds/smart_filter.py

@@ -6,7 +6,7 @@
 from rss_glue.feeds import ai_client, feed
 from rss_glue.utils import from_subpost
 
-base_prompt = """
+_base_prompt = """
 Please take a look at the following bit of a content from an RSS feed:
 
 Title: {title}
@@ -43,18 +43,17 @@ class AiFilterPost(feed.ReferenceFeedItem):
     include_post: bool
 
 
-class AiFilterFeed(feed.BaseFeed):
+class AiFilterFeed(feed.AugmentFeed):
     """
     AiFilterFeed is a feed that filters out posts based
     on a given prompt.
     """
 
-    source: feed.BaseFeed
-    limit: int
     client: ai_client.AiClient
     name: str = "smart_filter"
     post_cls: type[AiFilterPost] = AiFilterPost
 
+
     def __init__(
         self,
         source: feed.BaseFeed,
@@ -64,29 +63,29 @@ def __init__(
         limit: int = -1,
         title: Optional[str] = None,
     ):
-        self.source = source
-        self.limit = limit
         self.prompt = prompt
         self.client = client
-
-        self.author = source.author
-        self.origin_url = source.origin_url
         self.content_limit = content_limit
+
+        super().__init__(source=source, limit=limit)
+
         if title:
             self.title = title
         else:
             self.title = f"Filter {source.title}"
-        super().__init__()
 
-    @property
-    def namespace(self):
-        return f"{self.name}_{self.source.namespace}"
+    def posts(
+        self, limit: int = 50, start: Optional[datetime] = None, end: Optional[datetime] = None
+    ) -> list[feed.FeedItem]:
+        source_posts = self.source.posts(limit=limit, start=start, end=end)
+        posts = [cast(AiFilterPost, self.post(post.id)) for post in source_posts]
+        return [post for post in posts if post and post.include_post]
 
-    def format_prompt(self, post: feed.FeedItem) -> str:
+    def _format_prompt(self, post: feed.FeedItem) -> str:
         f = HTMLFilter()
         f.feed(post.render())
 
-        return base_prompt.format(
+        return _base_prompt.format(
             title=post.title,
             author=post.author,
             content=f.text[: self.content_limit],
@@ -95,38 +94,14 @@ def format_prompt(self, post: feed.FeedItem) -> str:
             prompt=self.prompt,
         )
 
-    def posts(
-        self, limit: int = 50, start: Optional[datetime] = None, end: Optional[datetime] = None
-    ) -> list[feed.FeedItem]:
-        source_posts = self.source.posts(limit=limit, start=start, end=end)
-        posts = [cast(AiFilterPost, self.post(post.id)) for post in source_posts]
-        return [post for post in posts if post and post.include_post]
-
-    def sources(self) -> Iterable[feed.BaseFeed]:
-        yield self.source
-
-    def next_update(self, force):
-        return self.source.next_update(force)
-
     def update(self) -> None:
-        """
-        This feed only updates when the source feed updates
-        """
-        source_posts = self.source.posts()
-        # Sort by posted_time
-        source_posts.sort(key=lambda post: post.posted_time, reverse=True)
-        # Limit to the user specified limit
-        if self.limit != -1:
-            source_posts = source_posts[: self.limit]
-        # Figure out which ones we haven't tested yet
-
-        for source_post in source_posts:
+        for source_post in self.source.posts(limit=self.limit):
             post = self.post(source_post.id)
             if post:
                 self.logger.debug(f" skipping filter check for {source_post.id}")
                 continue
 
-            msg = self.client.get_response(self.format_prompt(source_post))
+            msg = self.client.get_response(self._format_prompt(source_post))
 
             include_post = False
             if "yes" in msg.response.lower():
@@ -144,4 +119,4 @@ def update(self) -> None:
                 include_post=include_post,
             )
             self.logger.info(f"post={source_post.id} include_post={include_post}")
-            self.cache_set(value.id, value.to_dict())
+            self.cache_set(value.id, value.to_dict())

tests/test_smart_filter.py

@@ -70,3 +70,43 @@ def test_smart_filter_basic_filtering(self, fs_config, mock_http_requests):
         assert filtered_post.include_post is True
         assert filtered_post.token_cost == 100
         assert filtered_post.namespace == f"smart_filter_{source_feed.namespace}"
+
+    def test_smart_filter_updates_when_source_updated_but_not_due(
+        self, fs_config, mock_http_requests
+    ):
+        """Test that smart filter needs update when source was updated more recently, even if source doesn't need update."""
+        mock_rss_feed()
+
+        # Create a source feed
+        source_feed = RssFeed(
+            id="test_feed",
+            url="https://example.com/feed.xml",
+            limit=10,
+            interval=timedelta(hours=1),
+        )
+        fs_config([source_feed])
+        source_feed.update()
+
+        # Create a mock AI client
+        mock_client = Mock(spec=AiClient)
+        mock_client.get_response.return_value = AiClientResponse(response="yes", tokens_used=100)
+
+        # Create the smart filter but don't update it yet
+        filter_feed = AiFilterFeed(
+            source=source_feed,
+            client=mock_client,
+            prompt="Only include posts about technology",
+            limit=10,
+        )
+        fs_config([source_feed, filter_feed])
+
+        # At this point, source has been updated but filter hasn't
+        # So source.last_updated > filter.last_updated (or filter.last_updated is None)
+        assert source_feed.last_updated is not None
+
+        # Check next_update - should return True even though source doesn't need update
+        next_update_time, needs_update = filter_feed.next_update(force=False)
+
+        # Should indicate it needs an update because source was updated more recently
+        assert needs_update is True
+        assert next_update_time == source_feed.last_updated