Merge pull request #5064 from systeminit/zack/update-module-cache-on-…

…boot-in-dev-mode

feat: migrate local module cache on sdf boot, in dev mode

bin/sdf/src/main.rs

@@ -132,6 +132,7 @@ async fn run_server(
     telemetry_shutdown: TelemetryShutdownGuard,
 ) -> Result<()> {
     let migration_mode_is_run = config.migration_mode().is_run();
+    let is_dev_mode = config.dev_mode();
 
     let server = Server::from_config(
         config,
@@ -146,7 +147,7 @@ async fn run_server(
         //
         // Note that signals are not yet listened for, so a `SIGTERM`/`SIGINT` will cancel this
         // operation and simply exit.
-        server.migrator().run_migrations().await?;
+        server.migrator().run_migrations(is_dev_mode).await?;
     }
 
     main_tracker.spawn(async move {
@@ -180,7 +181,7 @@ async fn migrate_and_quit(
     let migrator =
         Migrator::from_config(config, &helping_tasks_tracker, helping_tasks_token.clone()).await?;
 
-    let handle = main_tracker.spawn(migrator.run_migrations());
+    let handle = main_tracker.spawn(migrator.run_migrations(false));
 
     shutdown::graceful_with_handle(handle)
         .group(main_tracker, main_token)

lib/sdf-server/src/config.rs

@@ -146,6 +146,9 @@ pub struct Config {
 
     #[builder(default)]
     audit: AuditDatabaseConfig,
+
+    #[builder(default)]
+    dev_mode: bool,
 }
 
 impl StandardConfig for Config {
@@ -264,6 +267,10 @@ impl Config {
     pub fn audit(&self) -> &AuditDatabaseConfig {
         &self.audit
     }
+
+    pub fn dev_mode(&self) -> bool {
+        self.dev_mode
+    }
 }
 
 impl ConfigBuilder {
@@ -287,6 +294,8 @@ pub struct ConfigFile {
     #[serde(default)]
     pub migration_mode: MigrationMode,
     #[serde(default)]
+    pub dev_mode: bool,
+    #[serde(default)]
     pub jwt_signing_public_key: JwtConfig,
     #[serde(default)]
     pub crypto: VeritechCryptoConfig,
@@ -340,6 +349,7 @@ impl Default for ConfigFile {
             create_workspace_allowlist: Default::default(),
             spicedb: Default::default(),
             audit: Default::default(),
+            dev_mode: false,
         }
     }
 }
@@ -375,6 +385,7 @@ impl TryFrom<ConfigFile> for Config {
             create_workspace_allowlist: value.create_workspace_allowlist,
             spicedb: value.spicedb,
             audit: value.audit,
+            dev_mode: value.dev_mode,
         })
     }
 }
@@ -513,6 +524,7 @@ fn buck2_development(config: &mut ConfigFile) -> Result<()> {
     config.spicedb.enabled = true;
     config.audit.pg.certificate_path = Some(postgres_cert.clone().try_into()?);
     config.audit.pg.dbname = audit_logs::database::DBNAME.to_string();
+    config.dev_mode = true;
 
     Ok(())
 }
@@ -575,6 +587,7 @@ fn cargo_development(dir: String, config: &mut ConfigFile) -> Result<()> {
     config.spicedb.enabled = true;
     config.audit.pg.certificate_path = Some(postgres_cert.clone().try_into()?);
     config.audit.pg.dbname = audit_logs::database::DBNAME.to_string();
+    config.dev_mode = true;
 
     Ok(())
 }

lib/sdf-server/src/migrations.rs

@@ -4,7 +4,7 @@ use audit_logs::database::{
     AuditDatabaseContext, AuditDatabaseContextError, AuditDatabaseMigrationError,
 };
 use dal::{
-    cached_module::CachedModuleError, slow_rt::SlowRuntimeError,
+    cached_module::CachedModule, slow_rt::SlowRuntimeError,
     workspace_snapshot::migrator::SnapshotGraphMigrator, ServicesContext,
 };
 use telemetry::prelude::*;
@@ -25,14 +25,14 @@ pub enum MigratorError {
     Join(#[from] JoinError),
     #[error("error while migrating audit database: {0}")]
     MigrateAuditDatabase(#[source] AuditDatabaseMigrationError),
+    #[error("error while migrating cached modules: {0}")]
+    MigrateCachedModules(#[source] Box<dyn std::error::Error + 'static + Sync + Send>),
     #[error("error while migrating dal database: {0}")]
     MigrateDalDatabase(#[source] dal::ModelError),
     #[error("error while migrating layer db database: {0}")]
     MigrateLayerDbDatabase(#[source] si_layer_cache::LayerDbError),
     #[error("error while migrating snapshots: {0}")]
     MigrateSnapshots(#[source] Box<dyn std::error::Error + 'static + Sync + Send>),
-    #[error("module cache error: {0}")]
-    ModuleCache(#[from] CachedModuleError),
     #[error("module index url not set")]
     ModuleIndexNotSet,
     #[error("slow runtime: {0}")]
@@ -46,6 +46,13 @@ impl MigratorError {
     {
         Self::MigrateSnapshots(Box::new(err))
     }
+
+    fn migrate_cached_modules<E>(err: E) -> Self
+    where
+        E: std::error::Error + 'static + Sync + Send,
+    {
+        Self::MigrateCachedModules(Box::new(err))
+    }
 }
 
 type MigratorResult<T> = std::result::Result<T, MigratorError>;
@@ -97,7 +104,7 @@ impl Migrator {
             otel.status_message = Empty,
         )
     )]
-    pub async fn run_migrations(self) -> MigratorResult<()> {
+    pub async fn run_migrations(self, update_module_cache: bool) -> MigratorResult<()> {
         let span = current_span_for_instrument_at!("info");
 
         self.migrate_audit_database()
@@ -116,6 +123,12 @@ impl Migrator {
             .await
             .map_err(|err| span.record_err(err))?;
 
+        if update_module_cache {
+            self.migrate_module_cache()
+                .await
+                .map_err(|err| span.record_err(err))?;
+        }
+
         span.record_ok();
         Ok(())
     }
@@ -165,4 +178,25 @@ impl Migrator {
             .map_err(MigratorError::migrate_snapshots)?;
         Ok(())
     }
+
+    #[instrument(name = "sdf.migrator.migrate_module_cache", level = "info", skip_all)]
+    async fn migrate_module_cache(&self) -> MigratorResult<()> {
+        let dal_context = self.services_context.clone().into_builder(true);
+        let ctx = dal_context
+            .build_default()
+            .await
+            .map_err(MigratorError::migrate_cached_modules)?;
+
+        info!("Updating local module cache");
+
+        let new_modules = CachedModule::update_cached_modules(&ctx)
+            .await
+            .map_err(MigratorError::migrate_cached_modules)?;
+        info!(
+            "{} new builtin assets found in module index",
+            new_modules.len()
+        );
+
+        Ok(())
+    }
 }