Add telegram bot (#40)

* Fix some missing data from Steam Signed-off-by: Eiko Wagenknecht <[email protected]> * Add telegram bot functionality Signed-off-by: Eiko Wagenknecht <[email protected]> * Telegram menu Signed-off-by: Eiko Wagenknecht <[email protected]> * Manage menu Signed-off-by: Eiko Wagenknecht <[email protected]> * Send new offers method Signed-off-by: Eiko Wagenknecht <[email protected]> * Bot only when configured Signed-off-by: Eiko Wagenknecht <[email protected]> * Enable details button and subscriptions Signed-off-by: Eiko Wagenknecht <[email protected]> * Count offers sent Signed-off-by: Eiko Wagenknecht <[email protected]> * No Steam refresh every time Signed-off-by: Eiko Wagenknecht <[email protected]> * More send new offers text Signed-off-by: Eiko Wagenknecht <[email protected]> * Readme Signed-off-by: Eiko Wagenknecht <[email protected]> * Better error handling Signed-off-by: Eiko Wagenknecht <[email protected]>
eikowagenknecht · Apr 22, 2022 · b4277dc · b4277dc
1 parent 0e29ae6
commit b4277dc
Show file tree

Hide file tree

Showing 15 changed files with 1,095 additions and 314 deletions.
diff --git a/README.md b/README.md
@@ -6,13 +6,16 @@
 
 You enjoy getting games for free but you *don’t* enjoy having to keep track of the various sources (Amazon Prime, Epic Games, Steam, ...) for free offers? Also your F5 key starts to look a bit worn out? Then this is for you!
 
-This Python (3.10+) application uses Selenium to automatically visit sites with free gaming related offers (currently Amazon Prime, Epic Games and Steam are supported, more will follow) and then neatly puts the gathered information into RSS feeds. So now you can track the offers using your favorite news reader like Feedly instead of manually visiting the sites.
+This Python (3.10+) application uses Selenium to automatically visit sites with free gaming related offers (currently Amazon Prime, Epic Games and Steam are supported, more will follow) and then neatly puts the gathered information into RSS feeds and a Telegram bot. So now you can track the offers using your favorite news reader like Feedly instead of manually visiting the sites or get a message every time a new offer is available.
 
 ## Usage
 
 You can either run this script locally on your computer or in any environment capable of running a Docker container.
 
 Just want the feeds? Sure. You can use the links below. They are updated every 20 minutes.
+
+If you prefer Telegram, you can instead subscribe to the [Telegram LootScraperBot](https://t.me/LootScraperBot) to get push notifications for new offers.
+
 - Amazon Prime ([games](https://feed.phenx.de/lootscraper_amazon_game.xml) and [ingame loot](https://feed.phenx.de/lootscraper_amazon_loot.xml))
 - [Epic Games (games only)](https://feed.phenx.de/lootscraper_epic_game.xml)
 - [Gog (games only)](https://feed.phenx.de/lootscraper_gog_game.xml)

diff --git a/alembic/versions/20220419_124703_split_game_information.py b/alembic/versions/20220419_124703_split_game_information.py
@@ -6,13 +6,13 @@
 
 """
 from typing import Any
-from alembic import op
-import sqlalchemy as sa
 
-from app.sqlalchemy import AwareDateTime
+import sqlalchemy as sa
 from sqlalchemy import orm
 from sqlalchemy.ext.declarative import declarative_base
 
+from alembic import op
+from app.sqlalchemy import AwareDateTime
 
 # revision identifiers, used by Alembic.
 revision = "038c26b62555"

diff --git a/alembic/versions/20220420_144219_user_table.py b/alembic/versions/20220420_144219_user_table.py
@@ -0,0 +1,51 @@
+"""User table
+
+Revision ID: 8cfaaf08b306
+Revises: 038c26b62555
+Create Date: 2022-04-20 14:42:19.402926+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+from app.sqlalchemy import AwareDateTime
+
+# revision identifiers, used by Alembic.
+revision = "8cfaaf08b306"
+down_revision = "038c26b62555"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "users",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("registration_date", AwareDateTime(), nullable=True),
+        sa.Column("offers_received_count", sa.Integer(), nullable=True),
+        sa.Column("telegram_id", sa.Integer(), nullable=True),
+        sa.Column("telegram_chat_id", sa.Integer(), nullable=True),
+        sa.Column("telegram_user_details", sa.JSON(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "telegram_subscriptions",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "source",
+            sa.Enum("AMAZON", "EPIC", "STEAM", "GOG", name="source"),
+            nullable=False,
+        ),
+        sa.Column("type", sa.Enum("LOOT", "GAME", name="offertype"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["users.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("telegram_subscriptions")
+    op.drop_table("users")
diff --git a/alembic/versions/20220422_090529_last_sent_offer.py b/alembic/versions/20220422_090529_last_sent_offer.py
@@ -0,0 +1,45 @@
+"""Last sent offer
+
+Revision ID: 52ea632ee417
+Revises: 8cfaaf08b306
+Create Date: 2022-04-22 09:05:29.092417+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "52ea632ee417"
+down_revision = "8cfaaf08b306"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("offers", schema=None) as batch_op:  # type: ignore
+        batch_op.alter_column("seen_first", existing_type=sa.DATETIME(), nullable=False)
+        batch_op.alter_column("seen_last", existing_type=sa.DATETIME(), nullable=False)
+
+    with op.batch_alter_table(  # type: ignore
+        "telegram_subscriptions", schema=None
+    ) as batch_op:
+        batch_op.add_column(sa.Column("last_offer_id", sa.Integer(), nullable=True))
+
+    op.execute("UPDATE telegram_subscriptions SET last_offer_id = 0")
+
+    with op.batch_alter_table(  # type: ignore
+        "telegram_subscriptions", schema=None
+    ) as batch_op:
+        batch_op.alter_column(
+            "last_offer_id", existing_type=sa.Integer(), nullable=False
+        )
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("telegram_subscriptions", schema=None) as batch_op:  # type: ignore
+        batch_op.drop_column("last_offer_id")
+
+    with op.batch_alter_table("offers", schema=None) as batch_op:  # type: ignore
+        batch_op.alter_column("seen_last", existing_type=sa.DATETIME(), nullable=True)
+        batch_op.alter_column("seen_first", existing_type=sa.DATETIME(), nullable=True)
diff --git a/app/common.py b/app/common.py
@@ -2,7 +2,7 @@
 
 TIMESTAMP_SHORT = "%Y-%m-%d"
 TIMESTAMP_LONG = "%Y-%m-%d %H:%M:%S"
-TIMESTAMP_READABLE_WITH_HOUR = "%Y-%m-%d - %H:%M (UTC)"
+TIMESTAMP_READABLE_WITH_HOUR = "%Y-%m-%d - %H:%M UTC"
 
 
 class OfferType(Enum):
@@ -15,3 +15,7 @@ class Source(Enum):
     EPIC = "Epic Games"
     STEAM = "Steam"
     GOG = "GOG"
+
+
+def chunkstring(string: str, length: int) -> list[str]:
+    return list((string[0 + i : length + i] for i in range(0, len(string), length)))
diff --git a/app/configparser.py b/app/configparser.py
@@ -37,6 +37,7 @@ class ParsedConfig:
     scrape_info: bool = True  # Not used anywhere yet
     generate_feed: bool = True
     upload_feed: bool = False
+    telegram_bot: bool = False
 
     # Telegram
     telegram_access_token: str = ""
@@ -118,6 +119,7 @@ def get() -> ParsedConfig:
             parsed_config.scrape_info = config.getboolean("actions", "ScrapeInfo")
             parsed_config.generate_feed = config.getboolean("actions", "GenerateFeed")
             parsed_config.upload_feed = config.getboolean("actions", "UploadFtp")
+            parsed_config.telegram_bot = config.getboolean("actions", "TelegramBot")
 
             parsed_config.telegram_access_token = config["telegram"]["AccessToken"]
             parsed_config.telegram_developer_chat_id = int(

diff --git a/app/feed.py b/app/feed.py
@@ -116,7 +116,15 @@ def generate_feed(
                 if game.steam_info.metacritic_url:
                     text = f'<a href="{html.escape(game.steam_info.metacritic_url)}">{text}</a>'
                 ratings.append(text)
-
+            if (
+                game.steam_info
+                and game.steam_info.percent
+                and game.steam_info.score
+                and game.steam_info.recommendations
+            ):
+                text = f"Steam {game.steam_info.percent} % ({game.steam_info.score}/10, {game.steam_info.recommendations} recommendations)"
+                text = f'<a href="{html.escape(game.steam_info.url)}">{text}</a>'
+                ratings.append(text)
             if (
                 game.igdb_info
                 and game.igdb_info.meta_ratings
@@ -133,15 +141,6 @@ def generate_feed(
                 text = f"IGDB User {game.igdb_info.user_score} % ({game.igdb_info.user_ratings} sources)"
                 text = f'<a href="{html.escape(game.igdb_info.url)}">{text}</a>'
                 ratings.append(text)
-            if (
-                game.steam_info
-                and game.steam_info.percent
-                and game.steam_info.score
-                and game.steam_info.recommendations
-            ):
-                text = f"Steam {game.steam_info.percent} % ({game.steam_info.score}/10, {game.steam_info.recommendations} recommendations)"
-                text = f'<a href="{html.escape(game.steam_info.url)}">{text}</a>'
-                ratings.append(text)
             if len(ratings) > 0:
                 content += f"<li><b>Ratings:</b> {' / '.join(ratings)}</li>"
             if game.igdb_info and game.igdb_info.release_date:
@@ -159,8 +158,8 @@ def generate_feed(
                     f"<li><b>Genres:</b> {html.escape(game.steam_info.genres)}</li>"
                 )
             content += "</ul>"
+            content += "<p>* Any information about the offer is automatically grabbed and may in rare cases not match the correct game.</p>"
 
-        content += "<p>* Any information about the offer is automatically grabbed and may in rare cases not match the correct game.</p>"
         content += f'<p><small>Source: {html.escape(offer.source.value)}, Seen first: {offer.seen_first.strftime(TIMESTAMP_LONG)}, Generated by <a href="https://github.com/eikowagenknecht/lootscraper">LootScraper</a></small></p>'
         feed_entry.content(content, type="xhtml")
         # - Link

diff --git a/app/scraper/info/igdb.py b/app/scraper/info/igdb.py
@@ -1,7 +1,8 @@
 import json
 import logging
-from datetime import datetime, timezone
 import re
+from datetime import datetime, timezone
+
 import requests
 from igdb.wrapper import IGDBWrapper
 

diff --git a/app/scraper/info/steam.py b/app/scraper/info/steam.py
@@ -26,9 +26,12 @@
 STEAM_DETAILS_STORE = "https://store.steampowered.com/app/"
 STEAM_DETAILS_REVIEW_SCORE = '//div[@id="userReviews"]/div[@itemprop="aggregateRating"]'  # data-tooltip-html attribute
 STEAM_DETAILS_REVIEW_SCORE_VALUE = '//div[@id="userReviews"]/div[@itemprop="aggregateRating"]//meta[@itemprop="ratingValue"]'  # content attribute
+STEAM_DETAILS_REVIEW_COUNT = '//div[@id="userReviews"]/div[@itemprop="aggregateRating"]//meta[@itemprop="reviewCount"]'  # content attribute
 STEAM_DETAILS_LOADED = '//div[contains(concat(" ", normalize-space(@class), " "), " game_page_background ")]'
 STEAM_PRICE_FULL = '(//div[contains(concat(" ", normalize-space(@class), " "), " game_area_purchase_game ")])[1]//div[contains(concat(" ", normalize-space(@class), " "), " game_purchase_action")]//div[contains(concat(" ", normalize-space(@class), " "), " game_purchase_price")]'  # text=" 27,99€ "
 STEAM_PRICE_DISCOUNTED_ORIGINAL = '(//div[contains(concat(" ", normalize-space(@class), " "), " game_area_purchase_game ")])[1]//div[contains(concat(" ", normalize-space(@class), " "), " game_purchase_action")]//div[contains(concat(" ", normalize-space(@class), " "), " discount_original_price")]'  # text=" 27,99€ "
+STEAM_RELEASE_DATE = '//div[@id="genresAndManufacturer"]'
+
 MAX_WAIT_SECONDS = 30  # Needs to be quite high in Docker for first run
 
 
@@ -112,7 +115,7 @@ def get_steam_details(
         # No entry found, not adding any data
         return None
 
-    logging.info(f"Steam: Reading details for app id {id}")
+    logging.info(f"Steam: Reading details for app id {steam_app_id}")
 
     steam_info = SteamInfo()
     steam_info.id = steam_app_id
@@ -242,10 +245,8 @@ def get_steam_details(
         element: WebElement = driver.find_element(By.XPATH, STEAM_DETAILS_REVIEW_SCORE)
         rating_str: str = element.get_attribute("data-tooltip-html")  # type: ignore
         steam_info.percent = int(rating_str.split("%")[0].strip())
-
     except WebDriverException:
         logging.error(f"No Steam percentage found for {steam_app_id}!")
-
     except ValueError:
         logging.error(f"Invalid Steam percentage {rating_str} for {steam_app_id}!")
 
@@ -254,51 +255,74 @@ def get_steam_details(
             By.XPATH, STEAM_DETAILS_REVIEW_SCORE_VALUE
         )
         rating2_str: str = element2.get_attribute("content")  # type: ignore
-        try:
-            steam_info.score = int(rating2_str)
-        except ValueError:
-            pass
-
+        steam_info.score = int(rating2_str)
     except WebDriverException:
         logging.error(f"No Steam rating found for {steam_app_id}!")
-
     except ValueError:
         logging.error(f"Invalid Steam rating {rating2_str} for {steam_app_id}!")
 
+    if steam_info.recommendations is None:
+        try:
+            element6: WebElement = driver.find_element(
+                By.XPATH, STEAM_DETAILS_REVIEW_COUNT
+            )
+            recommendations_str: str = element6.get_attribute("content")  # type: ignore
+            steam_info.recommendations = int(recommendations_str)
+        except WebDriverException:
+            logging.error(f"No Steam rating found for {steam_app_id}!")
+        except ValueError:
+            logging.error(f"Invalid Steam rating {recommendations_str} for {steam_app_id}!")
+
     if steam_info.recommended_price_eur is None:
         try:
             element3: WebElement = driver.find_element(
                 By.XPATH, STEAM_PRICE_DISCOUNTED_ORIGINAL
             )
             price_str: str = element3.text
-            try:
-                steam_info.recommended_price_eur = float(
-                    price_str.replace("€", "").replace(",", ".").strip()
-                )
-            except ValueError:
-                pass
-
+            steam_info.recommended_price_eur = float(
+                price_str.replace("€", "").replace(",", ".").strip()
+            )
         except WebDriverException:
             logging.debug(
                 f"No Steam discounted original price found on shop page for {steam_app_id}"
             )
+        except ValueError:
+            logging.debug(
+                f"Steam discounted original price has wrong format for {steam_app_id}"
+            )
 
     if steam_info.recommended_price_eur is None:
         try:
             element4: WebElement = driver.find_element(By.XPATH, STEAM_PRICE_FULL)
-            price2_str: str = element4.text.replace("€", "").strip()
+            price2_str: str = element4.text
             if "free" in price2_str.lower():
                 steam_info.recommended_price_eur = 0
             else:
-                try:
-                    steam_info.recommended_price_eur = float(price2_str)
-                except ValueError:
-                    pass
-
+                steam_info.recommended_price_eur = float(
+                    price2_str.replace("€", "").replace(",", ".").strip()
+                )
         except WebDriverException:
             logging.debug(f"No Steam full price found on shop page for {steam_app_id}")
+        except ValueError:
+            logging.debug(f"Steam full price has wrong format for {steam_app_id}")
 
     if steam_info.recommended_price_eur is None:
         logging.error(f"No Steam price found for {steam_app_id}")
 
+    if steam_info.release_date is None:
+        try:
+            element5: WebElement = driver.find_element(By.XPATH, STEAM_RELEASE_DATE)
+            release_date_str: str = element5.text
+            release_date_str = release_date_str.split("RELEASE DATE:")[1].strip()
+            release_date: datetime = datetime.strptime(
+                release_date_str, "%d %b, %Y"
+            ).replace(tzinfo=timezone.utc)
+            steam_info.release_date = release_date
+
+        except WebDriverException:
+            logging.debug(f"No release date found on shop page for {steam_app_id}")
+        except (IndexError, ValueError):
+            logging.debug(
+                f"Release date in wrong format on shop page for {steam_app_id}"
+            )
     return steam_info