Add beta handling

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
 
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.9"
+          python-version: "3.10"
 
       - uses: actions/cache@v2
         with:

Dockerfile

@@ -1,24 +1,26 @@
-FROM python:3.9 as builder
+FROM python:3.10 as builder
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-        build-essential libcairo2-dev libgirepository1.0-dev \
-        gir1.2-ostree-1.0 flatpak
+    build-essential libcairo2-dev libgirepository1.0-dev \
+    gir1.2-ostree-1.0 flatpak
 
 ADD requirements.txt /requirements.txt
 RUN python -m venv /venv && \
     /venv/bin/pip install -r requirements.txt \
     && rm -f /requirements.txt
 
-FROM python:3.9-slim
+FROM python:3.10-slim
 
 EXPOSE 8000
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-        libcairo2 gir1.2-ostree-1.0 flatpak && \
+    libcairo2 gir1.2-ostree-1.0 flatpak && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
-    flatpak --user remote-add flathub https://flathub.org/repo/flathub.flatpakrepo
+    flatpak --user remote-add flathub https://flathub.org/repo/flathub.flatpakrepo && \
+    flatpak --user remote-add flathub-beta https://flathub.org/beta-repo/flathub-beta.flatpakrepo
+
 
 COPY ./app /app
 COPY ./data /data

app/apps.py

@@ -6,43 +6,55 @@
 
 
 def load_appstream():
-    apps = utils.appstream2dict("repo")
+    apps_stable = utils.appstream2dict("repo")
+    apps_beta = utils.appstream2dict("beta-repo")
 
-    current_apps = {app[5:] for app in db.redis_conn.smembers("apps:index")}
+    apps = merge_apps(apps_beta, apps_stable)
+
+    current_apps = db.get_apps("stable")
+    current_apps_beta = db.get_apps("beta")
     current_categories = db.redis_conn.smembers("categories:index")
+    current_developers = db.redis_conn.smembers("developers:index")
 
     db.initialize()
     with db.redis_conn.pipeline() as p:
         p.delete("categories:index", *current_categories)
+        p.delete("developers:index", *current_developers)
 
         for appid in apps:
             redis_key = f"apps:{appid}"
 
             clean_html_re = re.compile("<.*?>")
-            search_description = re.sub(clean_html_re, "", apps[appid]["description"])
-
-            if search_keywords := apps[appid].get("keywords"):
-                search_keywords = " ".join(search_keywords)
-            else:
-                search_keywords = ""
-
-            fts = {
-                "id": appid,
-                "name": apps[appid]["name"],
-                "summary": apps[appid]["summary"],
-                "description": search_description,
-                "keywords": search_keywords,
-            }
+            if "stable" in apps[appid]:
+                search_description = re.sub(
+                    clean_html_re, "", apps[appid]["stable"]["description"]
+                )
+
+                if search_keywords := apps[appid]["stable"].get("keywords"):
+                    search_keywords = " ".join(search_keywords)
+                else:
+                    search_keywords = ""
+
+                fts = {
+                    "id": appid,
+                    "name": apps[appid]["stable"]["name"],
+                    "summary": apps[appid]["stable"]["summary"],
+                    "description": search_description,
+                    "keywords": search_keywords,
+                }
+                p.hset(f"fts:{appid}", mapping=fts)
+                if categories := apps[appid]["stable"].get("categories"):
+                    for category in categories:
+                        p.sadd("categories:index", category)
+                        p.sadd(f"categories:{category}", redis_key)
+
+                if developer_name := apps[appid]["stable"].get("developer_name"):
+                    p.sadd("developers:index", developer_name)
+                    p.sadd(f"developers:{developer_name}", redis_key)
 
             p.set(f"apps:{appid}", json.dumps(apps[appid]))
-            p.hset(f"fts:{appid}", mapping=fts)
-
-            if categories := apps[appid].get("categories"):
-                for category in categories:
-                    p.sadd("categories:index", category)
-                    p.sadd(f"categories:{category}", redis_key)
 
-        for appid in current_apps - set(apps):
+        for appid in current_apps - set(apps_stable):
             p.delete(
                 f"apps:{appid}",
                 f"fts:{appid}",
@@ -51,51 +63,85 @@ def load_appstream():
             )
             db.redis_search.delete_document(f"fts:{appid}")
 
-        new_apps = set(apps) - current_apps
+        new_apps = set(apps_stable) - current_apps
         if not len(new_apps):
             new_apps = None
 
+        new_apps_beta = set(apps_beta) - current_apps_beta
+        if not len(new_apps_beta):
+            new_apps_beta = None
+
         p.delete("apps:index")
-        p.sadd("apps:index", *[f"apps:{appid}" for appid in apps])
+        p.sadd("apps:index", *[f"apps:{appid}" for appid in apps_stable])
+        p.delete("apps:index_beta")
+        p.sadd("apps:index_beta", *[f"apps:{appid}" for appid in apps_beta])
         p.execute()
 
-    return new_apps
+    return new_apps, new_apps_beta
+
+
+def merge_apps(apps_beta, apps_stable):
+    apps = {}
+    for appid in apps_stable:
+        apps[appid] = {"stable": apps_stable[appid]}
+
+    for appid in apps_beta:
+        if appid not in apps:
+            apps[appid] = {"beta": apps_beta[appid]}
+        else:
+            apps[appid] = {"stable": apps_stable[appid], "beta": apps_beta[appid]}
+
+    return apps
 
 
 def list_appstream(repo: str = "stable"):
-    apps = {app[5:] for app in db.redis_conn.smembers("apps:index")}
-    return sorted(apps)
+    return sorted(db.get_apps(repo))
+
 
+def get_recently_updated(limit: int = 100, repo: str = "stable"):
+    if repo == "stable":
+        zset = db.redis_conn.zrevrange("recently_updated_zset", 0, limit - 1)
+    if repo == "beta":
+        zset = db.redis_conn.zrevrange("recently_updated_beta_zset", 0, limit - 1)
 
-def get_recently_updated(limit: int = 100):
-    zset = db.redis_conn.zrevrange("recently_updated_zset", 0, limit - 1)
     return [appid for appid in zset if db.redis_conn.exists(f"apps:{appid}")]
 
 
-def get_category(category: str):
+def get_category(category: str, repo: str = "stable"):
     if index := db.redis_conn.smembers(f"categories:{category}"):
         json_appdata = db.redis_conn.mget(index)
         appdata = [json.loads(app) for app in json_appdata]
 
-        return [(app["id"]) for app in appdata]
+        return [(app[repo]["id"]) for app in appdata if repo in app]
+    else:
+        return None
+
+
+def get_developer(developer: str, repo: str = "stable"):
+    if index := db.redis_conn.smembers(f"developers:{developer}"):
+        json_appdata = db.redis_conn.mget(index)
+        appdata = [json.loads(app) for app in json_appdata]
+
+        return [(app[repo]["id"]) for app in appdata if repo in app]
     else:
-        return []
+        return None
 
 
-def search(query: str):
+def search(query: str, repo: str = "stable"):
     if results := db.search(query):
         appids = tuple(doc_id.replace("fts", "apps") for doc_id in results)
         apps = [json.loads(x) for x in db.redis_conn.mget(appids)]
 
         ret = []
         for app in apps:
-            entry = {
-                "id": app["id"],
-                "name": app["name"],
-                "summary": app["summary"],
-                "icon": app.get("icon"),
-            }
-            ret.append(entry)
+            if repo in app:
+                entry = {
+                    "id": app[repo]["id"],
+                    "name": app[repo]["name"],
+                    "summary": app[repo]["summary"],
+                    "icon": app[repo].get("icon"),
+                }
+                ret.append(entry)
 
         return ret
 

app/db.py

@@ -126,5 +126,27 @@ def get_json_key(key: str):
     return None
 
 
-def get_app_count():
-    return redis_conn.scard("apps:index")
+def get_app_count(repo: str = "stable") -> int:
+    if repo == "stable_and_beta":
+        return len(get_apps(repo))
+    if repo == "stable":
+        return redis_conn.scard("apps:index")
+    if repo == "beta":
+        return redis_conn.scard("apps:index_beta")
+
+
+def get_apps(repo: str = "stable"):
+    if repo == "stable_and_beta":
+        apps = {app[5:] for app in redis_conn.smembers("apps:index")}
+        apps_beta = {app[5:] for app in redis_conn.smembers("apps:index_beta")}
+        return apps | apps_beta
+    elif repo == "stable":
+        apps = {app[5:] for app in redis_conn.smembers("apps:index")}
+        return apps
+    elif repo == "beta":
+        apps = {app[5:] for app in redis_conn.smembers("apps:index_beta")}
+        return apps
+
+
+def get_developers():
+    return {developer for developer in redis_conn.smembers("developers:index")}

app/feeds.py

@@ -4,17 +4,27 @@
 from . import db
 
 
-def generate_feed(key: str, title: str, description: str, link: str):
+def generate_feed(
+    key: str,
+    title: str,
+    description: str,
+    link: str,
+    repo: str,
+):
     feed = FeedGenerator()
     feed.title(title)
     feed.description(description)
     feed.link(href=link)
     feed.language("en")
 
-    appids = db.redis_conn.zrevrange(key, 0, 10, withscores=True)
-    apps = [(db.get_json_key(f"apps:{appid[0]}"), appid[1]) for appid in appids]
+    # get 15, as we might have some that don't exist, due to summary knowing about consale-applications
+    appids = db.redis_conn.zrevrange(key, 0, 15, withscores=True)
+    app_score_tuple = [
+        (db.get_json_key(f"apps:{appid[0]}"), appid[1]) for appid in appids
+    ]
+    apps = [(item[0][repo], item[1]) for item in app_score_tuple if item[0] is not None]
 
-    for app, timestamp in reversed(apps):
+    for app, timestamp in reversed(apps[:10]):
         if not app.get("name"):
             continue
 
@@ -29,9 +39,7 @@ def generate_feed(key: str, title: str, description: str, link: str):
         entry.pubDate(f"{entry_date} UTC")
 
         content = [
-            '<img src="https://dl.flathub.org/repo/appstream/x86_64/icons/128x128/{}.png">'.format(
-                app["id"]
-            ),
+            '<img src="{}">'.format(app["icon"]),
             f"<p>{app['summary']}</p>",
             f"<p>{app['description']}</p>",
             "<h3>Additional information:</h3>",
@@ -63,19 +71,21 @@ def generate_feed(key: str, title: str, description: str, link: str):
     return feed.rss_str()
 
 
-def get_recently_updated_apps_feed():
+def get_recently_updated_apps_feed(repo: str = "stable"):
     return generate_feed(
-        "recently_updated_zset",
+        "recently_updated_zset" if repo == "stable" else "recently_updated_beta_zset",
         "Flathub – recently updated applications",
         "Recently updated applications published on Flathub",
         "https://flathub.org/apps/collection/recently-updated",
+        repo,
     )
 
 
-def get_new_apps_feed():
+def get_new_apps_feed(repo: str = "stable"):
     return generate_feed(
-        "new_apps_zset",
+        "new_apps_zset" if repo == "stable" else "new_apps_beta_zset",
         "Flathub – recently added applications",
         "Applications recently published on Flathub",
         "https://flathub.org/apps/collection/new",
+        repo,
     )