Speed up adding feed

Updating the feed and marking every entry as read takes to long

Fixes #137

discord_twitter_webhooks/main.py

@@ -13,7 +13,7 @@
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from loguru import logger
-from reader import Feed, FeedNotFoundError, InvalidFeedURLError, Reader, StorageError
+from reader import Feed, FeedNotFoundError, Reader
 from starlette import status
 
 from discord_twitter_webhooks._dataclasses import (
@@ -31,8 +31,6 @@
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
-    from reader.types import Entry, EntryLike
-
 app = FastAPI()
 app.mount("/static", StaticFiles(directory=Path(__file__).parent / "static"), name="static")
 templates = Jinja2Templates(directory=Path(__file__).parent / "templates")
@@ -115,6 +113,39 @@ async def modify(request: Request, uuid: str) -> Response:
     raise HTTPException(status_code=404, detail=f"Group with UUID '{uuid}' not found")
 
 
+def add_feed_to_feed_groups(name_url: str, group_uuid: str) -> None:
+    """Add or modify the groups tag of a feed.
+
+    Args:
+        name_url: The feed URL.
+        group_uuid: The UUID of the group where we add the group tag.
+
+    Raises:
+        HTTPException: If the feed doesn't exist.
+    """
+    our_feed: Feed = reader.get_feed(name_url)
+    if not our_feed:
+        raise HTTPException(status_code=404, detail=f"Feed with name or URL '{name_url}' not found")
+
+    groups = list(reader.get_tag(our_feed, "groups", []))
+    groups.append(group_uuid)
+    reader.set_tag(our_feed, "groups", list(set(groups)))
+    logger.info(f"Added group {group_uuid} to feed {name_url}")
+
+
+def add_feed_to_global_groups(uuid: str) -> None:
+    """Add or modify the groups tag of a feed.
+
+    Args:
+        uuid: The UUID of the feed where we add the group tag.
+    """
+    groups = list(reader.get_tag((), "groups", []))
+    groups.append(uuid)
+    reader.set_tag((), "groups", list(set(groups)))
+    logger.info(f"Added group {uuid} to groups list")
+    logger.info(f"Group list is now {set(groups)}")
+
+
 @app.post("/feed")
 async def feed(  # noqa: PLR0913, ANN201
     name: Annotated[str, Form(title="Group Name")],
@@ -148,7 +179,7 @@ async def feed(  # noqa: PLR0913, ANN201
         logger.info(f"Creating new group {name}")
         uuid = str(uuid4())
 
-    # Set is to remove duplicates
+    # set() is to remove duplicates
     usernames_split = list(set(usernames.splitlines()))
 
     # Get the RSS feeds for each username
@@ -186,48 +217,19 @@ async def feed(  # noqa: PLR0913, ANN201
         replace_youtube=replace_youtube,
     )
 
-    # This will be used when adding group.rss_feeds
-    rss_feeds = []
-
     # Add the group to the reader
     reader.set_tag((), uuid, group.__dict__)
-    for _name in usernames_split:
-        name_url: str = f"{get_app_settings(reader).nitter_instance}/{_name}/with_replies/rss"
-        # TODO: Check if feed URL is valid
-        # Add the rss feed to the reader
-        try:
-            reader.add_feed(name_url, exist_ok=True)
-        except InvalidFeedURLError:
-            logger.error(f"Invalid URL {name_url}")
-            continue
-        except StorageError:
-            logger.error(f"Got StorageError when adding {name_url}")
-            continue
 
-        # Update the feed
-        reader.update_feeds(feed=name_url, workers=4)
-
-        # Mark every entry as read
-        _entry: Entry | EntryLike
-        for _entry in reader.get_entries(feed=name_url, read=False):
-            logger.debug(f"Marking entry {_entry.link} as read")
-            reader.mark_entry_as_read(_entry)
-
-        # Add what groups the feed is connected to
-        our_feed: Feed = reader.get_feed(name_url)
-        groups = list(reader.get_tag(our_feed, "groups", []))
-        groups.append(uuid)
-        reader.set_tag(our_feed, "groups", list(set(groups)))
-        logger.info(f"Added group {group.uuid} to feed {name_url}")
+    # Add the RSS feeds to the reader
+    for rss_feed_url in rss_feeds:
+        # Add the rss feed to the reader
+        reader.add_feed(rss_feed_url, exist_ok=True, allow_invalid_url=True)
 
-        rss_feeds.append(name_url)
+        # Each RSS feed will have a tag with the groups it belongs to
+        add_feed_to_feed_groups(name_url=rss_feed_url, group_uuid=uuid)
 
-    # Add the group to the groups list
-    groups = list(reader.get_tag((), "groups", []))
-    groups.append(uuid)
-    reader.set_tag((), "groups", list(set(groups)))
-    logger.info(f"Added group {group.uuid} to groups list")
-    logger.info(f"Group list is now {set(groups)}")
+    # There is a global tag with all the group UUIDs
+    add_feed_to_global_groups(uuid=uuid)
 
     # Redirect to the index page.
     return RedirectResponse(url="/", status_code=status.HTTP_303_SEE_OTHER)

discord_twitter_webhooks/send_to_discord.py

@@ -398,6 +398,21 @@ def too_old(entry: Entry | EntryLike, reader: Reader) -> bool:
     return False
 
 
+def mark_new_feed_as_read(reader: Reader) -> None:
+    """Mark all entries from new feeds as read.
+
+    Args:
+        reader: The reader which contains the entries.
+    """
+    # Loop through the new feeds and mark all entries as read
+    for feed in reader.get_feeds(new=True):
+        reader.update_feed(feed)
+        logger.info(f"Found a new feed: {feed.title}")
+        for entry in reader.get_entries(feed=feed):
+            reader.mark_entry_as_read(entry)
+            logger.info(f"Marked {entry.link} as unread because it's from a new feed")
+
+
 def send_to_discord(reader: Reader) -> None:  # noqa: C901, PLR0912
     """Send all new entries to Discord.
 
@@ -406,11 +421,11 @@ def send_to_discord(reader: Reader) -> None:  # noqa: C901, PLR0912
     Args:
         reader: The reader which contains the entries.
     """
+    mark_new_feed_as_read(reader)
     reader.update_feeds(workers=4)
 
     # Loop through the unread (unsent) entries.
     unread_entries = list(reader.get_entries(read=False))
-
     if not unread_entries:
         logger.info("No new entries found")
         return
@@ -430,10 +445,6 @@ def send_to_discord(reader: Reader) -> None:  # noqa: C901, PLR0912
 
         for _group in list(reader.get_tag((), "groups", [])):
             group: Group = get_group(reader, str(_group))
-            if not group:
-                logger.error("Group {} not found", _group)
-                continue
-
             for feed in group.rss_feeds:
                 # Loop through the RSS feeds that the group has and check if the entry is from one of them.
                 if entry.feed_url == feed: