WIP

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
 
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.9"
+          python-version: "3.10"
 
       - uses: actions/cache@v2
         with:

Dockerfile

@@ -1,24 +1,26 @@
-FROM python:3.9 as builder
+FROM python:3.10 as builder
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-        build-essential libcairo2-dev libgirepository1.0-dev \
-        gir1.2-ostree-1.0 flatpak
+    build-essential libcairo2-dev libgirepository1.0-dev \
+    gir1.2-ostree-1.0 flatpak
 
 ADD requirements.txt /requirements.txt
 RUN python -m venv /venv && \
     /venv/bin/pip install -r requirements.txt \
     && rm -f /requirements.txt
 
-FROM python:3.9-slim
+FROM python:3.10-slim
 
 EXPOSE 8000
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-        libcairo2 gir1.2-ostree-1.0 flatpak && \
+    libcairo2 gir1.2-ostree-1.0 flatpak && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
-    flatpak --user remote-add flathub https://flathub.org/repo/flathub.flatpakrepo
+    flatpak --user remote-add flathub https://flathub.org/repo/flathub.flatpakrepo && \
+    flatpak --user remote-add flathub-beta https://flathub.org/beta-repo/flathub-beta.flatpakrepo
+
 
 COPY ./app /app
 COPY ./data /data

app/apps.py

@@ -11,7 +11,8 @@ def load_appstream():
 
     apps = merge_apps(apps_beta, apps_stable)
 
-    current_apps = {app[5:] for app in db.redis_conn.smembers("apps:index")}
+    current_apps = db.get_apps("stable")
+    current_apps_beta = db.get_apps("beta")
     current_categories = db.redis_conn.smembers("categories:index")
 
     db.initialize()
@@ -60,13 +61,17 @@ def load_appstream():
         if not len(new_apps):
             new_apps = None
 
+        new_apps_beta = set(apps_beta) - current_apps_beta
+        if not len(new_apps_beta):
+            new_apps_beta = None
+
         p.delete("apps:index")
         p.sadd("apps:index", *[f"apps:{appid}" for appid in apps_stable])
         p.delete("apps:index_beta")
         p.sadd("apps:index_beta", *[f"apps:{appid}" for appid in apps_beta])
         p.execute()
 
-    return new_apps
+    return new_apps, new_apps_beta
 
 
 def merge_apps(apps_beta, apps_stable):
@@ -84,16 +89,15 @@ def merge_apps(apps_beta, apps_stable):
 
 
 def list_appstream(repo: str = "stable"):
-    if repo == "stable":
-        apps = {app[5:] for app in db.redis_conn.smembers("apps:index")}
-        return sorted(apps)
-    else:
-        apps = {app[5:] for app in db.redis_conn.smembers("apps:index_beta")}
-        return sorted(apps)
+    return sorted(db.get_apps(repo))
 
 
-def get_recently_updated(limit: int = 100):
-    zset = db.redis_conn.zrevrange("recently_updated_zset", 0, limit - 1)
+def get_recently_updated(limit: int = 100, repo: str = "stable"):
+    if repo == "stable":
+        zset = db.redis_conn.zrevrange("recently_updated_zset", 0, limit - 1)
+    if repo == "beta":
+        zset = db.redis_conn.zrevrange("recently_updated_beta_zset", 0, limit - 1)
+
     return [appid for appid in zset if db.redis_conn.exists(f"apps:{appid}")]
 
 

app/db.py

@@ -126,8 +126,23 @@ def get_json_key(key: str):
     return None
 
 
-def get_app_count(repo: str = "stable"):
+def get_app_count(repo: str = "stable") -> int:
+    if repo == "stable_and_beta":
+        return len(get_apps(repo))
     if repo == "stable":
         return redis_conn.scard("apps:index")
     if repo == "beta":
         return redis_conn.scard("apps:index_beta")
+
+
+def get_apps(repo: str = "stable"):
+    if repo == "stable_and_beta":
+        apps = {app[5:] for app in redis_conn.smembers("apps:index")}
+        apps_beta = {app[5:] for app in redis_conn.smembers("apps:index_beta")}
+        return apps | apps_beta
+    elif repo == "stable":
+        apps = {app[5:] for app in redis_conn.smembers("apps:index")}
+        return apps
+    elif repo == "beta":
+        apps = {app[5:] for app in redis_conn.smembers("apps:index_beta")}
+        return apps

app/feeds.py

@@ -5,15 +5,19 @@
 
 
 def generate_feed(
-    key: str, title: str, description: str, link: str, repo: str = "stable"
+    key: str,
+    title: str,
+    description: str,
+    link: str,
+    repo: str,
 ):
     feed = FeedGenerator()
     feed.title(title)
     feed.description(description)
     feed.link(href=link)
     feed.language("en")
 
-    # get 15, as we might have some that don't exist, due to summary being out of sync
+    # get 15, as we might have some that don't exist, due to summary knowing about consale-applications
     appids = db.redis_conn.zrevrange(key, 0, 15, withscores=True)
     app_score_tuple = [
         (db.get_json_key(f"apps:{appid[0]}"), appid[1]) for appid in appids
@@ -35,9 +39,7 @@ def generate_feed(
         entry.pubDate(f"{entry_date} UTC")
 
         content = [
-            '<img src="https://dl.flathub.org/repo/appstream/x86_64/icons/128x128/{}.png">'.format(
-                app["id"]
-            ),
+            '<img src="{}">'.format(app["icon"]),
             f"<p>{app['summary']}</p>",
             f"<p>{app['description']}</p>",
             "<h3>Additional information:</h3>",
@@ -69,19 +71,21 @@ def generate_feed(
     return feed.rss_str()
 
 
-def get_recently_updated_apps_feed():
+def get_recently_updated_apps_feed(repo: str = "stable"):
     return generate_feed(
-        "recently_updated_zset",
+        "recently_updated_zset" if repo == "stable" else "recently_updated_beta_zset",
         "Flathub – recently updated applications",
         "Recently updated applications published on Flathub",
         "https://flathub.org/apps/collection/recently-updated",
+        repo,
     )
 
 
-def get_new_apps_feed():
+def get_new_apps_feed(repo: str = "stable"):
     return generate_feed(
-        "new_apps_zset",
+        "new_apps_zset" if repo == "stable" else "new_apps_beta_zset",
         "Flathub – recently added applications",
         "Applications recently published on Flathub",
         "https://flathub.org/apps/collection/new",
+        repo,
     )

app/main.py

@@ -22,7 +22,7 @@ def startup_event():
 
 @app.post("/update")
 def update():
-    new_apps = apps.load_appstream()
+    new_apps, new_apps_beta = apps.load_appstream()
     summary.update()
     picks.update()
     stats.update()
@@ -31,9 +31,18 @@ def update():
         new_apps_zset = {}
         for appid in new_apps:
             if metadata := db.get_json_key(f"summary:{appid}"):
-                new_apps_zset[appid] = metadata.get("timestamp", 0)
+                if "stable" in metadata:
+                    new_apps_zset[appid] = metadata["stable"].get("timestamp", 0)
         db.redis_conn.zadd("new_apps_zset", new_apps_zset)
 
+    if new_apps_beta:
+        new_apps_beta_zset = {}
+        for appid in new_apps_beta:
+            if metadata := db.get_json_key(f"summary:{appid}"):
+                if "beta" in metadata:
+                    new_apps_beta_zset[appid] = metadata["beta"].get("timestamp", 0)
+        db.redis_conn.zadd("new_apps_beta_zset", new_apps_beta_zset)
+
     get_recently_updated.cache_clear()
 
 
@@ -62,13 +71,9 @@ def list_appstream(type: schemas.AppstreamType = "stable"):
 def get_appstream(
     appid: str,
     response: Response,
-    type: schemas.AppstreamType = "stable",
 ):
     if value := db.get_json_key(f"apps:{appid}"):
-        if "stable" in value and type == "stable":
-            return value["stable"]
-        if "beta" in value and type == "beta":
-            return value["beta"]
+        return value
 
     response.status_code = 404
     return None
@@ -82,8 +87,8 @@ def search(userquery: str):
 @app.get("/collection/recently-updated")
 @app.get("/collection/recently-updated/{limit}")
 @lru_cache()
-def get_recently_updated(limit: int = 100):
-    return apps.get_recently_updated(limit)
+def get_recently_updated(limit: int = 100, type: schemas.AppstreamType = "stable"):
+    return apps.get_recently_updated(limit, type)
 
 
 @app.get("/picks/{pick}")
@@ -105,15 +110,15 @@ def get_popular_days(days: int):
 
 
 @app.get("/feed/recently-updated")
-def get_recently_updated_apps_feed():
+def get_recently_updated_apps_feed(type: schemas.AppstreamType = "stable"):
     return Response(
-        content=feeds.get_recently_updated_apps_feed(), media_type="application/rss+xml"
+        content=feeds.get_recently_updated_apps_feed(type), media_type="application/rss+xml"
     )
 
 
 @app.get("/feed/new")
-def get_new_apps_feed():
-    return Response(content=feeds.get_new_apps_feed(), media_type="application/rss+xml")
+def get_new_apps_feed(type: schemas.AppstreamType = "stable"):
+    return Response(content=feeds.get_new_apps_feed(type), media_type="application/rss+xml")
 
 
 @app.get("/status", status_code=200)

app/schemas.py

@@ -19,3 +19,4 @@ class Category(str, Enum):
 class AppstreamType(str, Enum):
     Stable = "stable"
     Beta = "beta"
+    Stable_And_Beta = "stable_and_beta"

app/summary.py

@@ -25,7 +25,7 @@ def validate_ref(ref: str, enforce_arch=True):
     if enforce_arch and arch != "x86_64":
         return False
 
-    if branch != "stable":
+    if branch != "stable" and branch != "beta":
         return False
 
     return True
@@ -91,9 +91,6 @@ def parse_metadata(ini: str):
 
 
 def update():
-    summary_dict = defaultdict(lambda: {"arches": []})
-    recently_updated_zset = {}
-
     repo_file = Gio.File.new_for_path(f"{config.settings.flatpak_user_dir}/repo")
     repo = OSTree.Repo.new(repo_file)
     repo.open(None)
@@ -103,6 +100,30 @@ def update():
         GLib.VariantType.new(OSTree.SUMMARY_GVARIANT_STRING), summary, True
     )
 
+    status_beta, summary_beta, signatures_beta = repo.remote_fetch_summary(
+        "flathub-beta", None
+    )
+    data_beta = GLib.Variant.new_from_bytes(
+        GLib.VariantType.new(OSTree.SUMMARY_GVARIANT_STRING), summary_beta, True
+    )
+
+    summary_dict, recently_updated_zset = parse_summary_data(data)
+    summary_dict_beta, recently_updated_beta_zset = parse_summary_data(data_beta)
+    summary = merge_summary(summary_dict_beta, summary_dict)
+
+    db.redis_conn.zadd("recently_updated_zset", recently_updated_zset)
+    db.redis_conn.zadd("recently_updated_beta_zset", recently_updated_beta_zset)
+    db.redis_conn.mset(
+        {f"summary:{appid}": json.dumps(summary[appid]) for appid in summary}
+    )
+
+    return len(recently_updated_zset), len(recently_updated_beta_zset)
+
+
+def parse_summary_data(data):
+    summary_dict = defaultdict(lambda: {"arches": []})
+    recently_updated_zset = {}
+
     refs, metadata = data.unpack()
     xa_cache = metadata["xa.cache"]
 
@@ -143,9 +164,21 @@ def update():
 
         summary_dict[appid]["arches"].append(arch)
 
-    db.redis_conn.zadd("recently_updated_zset", recently_updated_zset)
-    db.redis_conn.mset(
-        {f"summary:{appid}": json.dumps(summary_dict[appid]) for appid in summary_dict}
-    )
+    return summary_dict, recently_updated_zset
+
+
+def merge_summary(summary_beta, summary_stable):
+    summary = {}
+    for appid in summary_stable:
+        summary[appid] = {"stable": summary_stable[appid]}
+
+    for appid in summary_beta:
+        if appid not in summary:
+            summary[appid] = {"beta": summary_beta[appid]}
+        else:
+            summary[appid] = {
+                "stable": summary_stable[appid],
+                "beta": summary_beta[appid],
+            }
 
-    return len(recently_updated_zset)
+    return summary