Save last offset for member pages.

migrations/20240112_01_Njk7j-create-initial-tables.sql

@@ -12,6 +12,11 @@ CREATE TABLE site (
  language TEXT NOT NULL
 );
 
+CREATE TABLE site_progress (
+ site_slug TEXT PRIMARY KEY REFERENCES site(site_slug),
+ last_member_offset INTEGER NOT NULL DEFAULT 1
+);
+
 CREATE TABLE "user" (
  user_slug TEXT PRIMARY KEY,
  user_name TEXT NOT NULL,

queries/progress.sql

@@ -0,0 +1,14 @@
+-- :name add_site_progress :insert
+INSERT INTO site_progress (site_slug)
+ VALUES (:site_slug)
+ ON CONFLICT (site_slug)
+ DO NOTHING;
+
+-- :name get_last_member_offset :one
+SELECT get_last_member_offset FROM site_progress
+ WHERE site_slug = :site_slug;
+
+-- :name update_last_member_offset :affected
+UPDATE site_progress
+ SET last_member_offset = MAX(last_member_offset, :last_offset)
+ WHERE site_slug = :site_slug;

yellowstone/core.py

@@ -16,7 +16,6 @@
  JobManager,
  get_site,
 )
-from .job.index_site_members import START_OFFSET as START_MEMBER_OFFSET
 from .s3 import S3
 from .wikidot import Wikidot
 

yellowstone/job/get_site.py

@@ -52,6 +52,7 @@ def insert_site(
  language=site.language,
  home_slug=site.home_page_slug,
  )
+ database.add_site_progress(site_slug=site.slug)
 
  # TODO insert basic home data
  _ = site.home_page_id

yellowstone/job/index_site_members.py

@@ -13,8 +13,6 @@
 if TYPE_CHECKING:
  from ..core import BackupDispatcher
 
-START_OFFSET = 1
-
 logger = logging.getLogger(__name__)
 
 
@@ -44,6 +42,7 @@ def run(core: "BackupDispatcher", data: SiteMemberJob) -> None:
  use_admin=core.config.uses_admin_members(site_slug),
  )
 
+ # Save member data
  if members:
  with core.database.transaction():
  # Queue the next offset, for iterating over pages using the job queue
@@ -57,3 +56,6 @@ def run(core: "BackupDispatcher", data: SiteMemberJob) -> None:
  joined_at=member.joined_at,
  )
  core.job.fetch_user({"user_id": member.id})
+
+ # Save member page progress
+ core.database.update_last_member_offset(last_offset=offset)