Unify usage of requests and cache them

flatpak_builder_lint/cli.py

@@ -7,10 +7,18 @@
 import sys
 from typing import Dict, List, Optional, Union
 
-import requests
 import sentry_sdk
 
-from . import __version__, appstream, builddir, checks, manifest, ostree, staticfiles
+from . import (
+    __version__,
+    appstream,
+    builddir,
+    checks,
+    domainutils,
+    manifest,
+    ostree,
+    staticfiles,
+)
 
 if sentry_dsn := os.getenv("SENTRY_DSN"):
     sentry_sdk.init(sentry_dsn)
@@ -43,19 +51,6 @@ def get_local_exceptions(appid: str) -> set:
     return set()
 
 
-def get_remote_exceptions(
-    appid: str, api_url: str = "https://flathub.org/api/v2/exceptions"
-) -> set:
-    try:
-        r = requests.get(f"{api_url}/{appid}")
-        r.raise_for_status()
-        ret = set(r.json())
-    except requests.exceptions.RequestException:
-        ret = set()
-
-    return ret
-
-
 def run_checks(
     kind: str, path: str, enable_exceptions: bool = False, appid: Optional[str] = None
 ) -> dict:
@@ -105,7 +100,7 @@ def run_checks(
             appid = infer_appid_func(path)
 
         if appid:
-            exceptions = get_remote_exceptions(appid)
+            exceptions = domainutils.get_remote_exceptions(appid)
             if not exceptions:
                 exceptions = get_local_exceptions(appid)
 

flatpak_builder_lint/domainutils.py

@@ -18,7 +18,13 @@
     "org.freedesktop.gitlab.",
 )
 
-session = CachedSession("cache", backend="sqlite", use_temp=True, cache_control=True)
+REQUEST_TIMEOUT = (120.05, None)
+
+FLATHUB_API_URL = "https://flathub.org/api/v2"
+FLATHUB_STABLE_REPO_URL = "https://dl.flathub.org/repo"
+FLATHUB_BETA_REPO_URL = "https://dl.flathub.org/beta-repo"
+
+session = CachedSession("cache", backend="sqlite", use_temp=True, expire_after=3600)
 
 
 def ignore_ref(ref: str) -> bool:
@@ -37,13 +43,15 @@ def ignore_ref(ref: str) -> bool:
 @cache
 def fetch_summary_bytes(url: str) -> bytes:
     summary_bytes = b""
-
-    r = session.get(url, allow_redirects=False, timeout=(120.05, None))
-    if (
-        r.status_code == 200
-        and r.headers.get("Content-Type") == "application/octet-stream"
-    ):
-        summary_bytes = r.content
+    try:
+        r = session.get(url, allow_redirects=False, timeout=REQUEST_TIMEOUT)
+        if (
+            r.status_code == 200
+            and r.headers.get("Content-Type") == "application/octet-stream"
+        ):
+            summary_bytes = r.content
+    except requests.exceptions.RequestException:
+        pass
 
     if not summary_bytes:
         raise Exception("Failed to fetch summary")
@@ -76,15 +84,17 @@ def get_appids_from_summary(url: str) -> set:
 @cache
 def get_all_apps_on_flathub() -> set:
     return get_appids_from_summary(
-        "https://dl.flathub.org/repo/summary"
-    ) | get_appids_from_summary("https://dl.flathub.org/beta-repo/summary")
+        f"{FLATHUB_STABLE_REPO_URL}/summary"
+    ) | get_appids_from_summary(f"{FLATHUB_BETA_REPO_URL}/summary")
 
 
+@cache
 def check_url(url: str, strict: bool) -> bool:
     assert url.startswith(("https://", "http://"))
+
     ret = False
     try:
-        r = requests.get(url, allow_redirects=False, timeout=10)
+        r = session.get(url, allow_redirects=False, timeout=REQUEST_TIMEOUT)
         if r.ok and not strict:
             ret = True
         # For known code hosting sites
@@ -94,6 +104,26 @@ def check_url(url: str, strict: bool) -> bool:
             ret = True
     except requests.exceptions.RequestException:
         pass
+
+    return ret
+
+
+@cache
+def get_remote_exceptions(appid: str) -> set[str]:
+
+    ret = set()
+    try:
+        # exception updates should be reflected immediately
+        r = requests.get(
+            f"{FLATHUB_API_URL}/exceptions/{appid}",
+            allow_redirects=False,
+            timeout=REQUEST_TIMEOUT,
+        )
+        if r.status_code == 200 and r.headers.get("Content-Type") == "application/json":
+            ret = set(r.json())
+    except requests.exceptions.RequestException:
+        pass
+
     return ret
 
 
@@ -205,8 +235,9 @@ def get_domain(appid: str) -> str | None:
     return domain
 
 
+@cache
 def is_app_on_flathub_api(appid: str) -> bool:
-    return check_url(f"https://flathub.org/api/v2/summary/{appid}", strict=True)
+    return check_url(f"{FLATHUB_API_URL}/summary/{appid}", strict=True)
 
 
 @cache