diff --git a/bin/sdf/src/main.rs b/bin/sdf/src/main.rs index 983db20129..868cd15862 100644 --- a/bin/sdf/src/main.rs +++ b/bin/sdf/src/main.rs @@ -132,6 +132,7 @@ async fn run_server( telemetry_shutdown: TelemetryShutdownGuard, ) -> Result<()> { let migration_mode_is_run = config.migration_mode().is_run(); + let is_dev_mode = config.dev_mode(); let server = Server::from_config( config, @@ -146,7 +147,7 @@ async fn run_server( // // Note that signals are not yet listened for, so a `SIGTERM`/`SIGINT` will cancel this // operation and simply exit. - server.migrator().run_migrations().await?; + server.migrator().run_migrations(is_dev_mode).await?; } main_tracker.spawn(async move { @@ -180,7 +181,7 @@ async fn migrate_and_quit( let migrator = Migrator::from_config(config, &helping_tasks_tracker, helping_tasks_token.clone()).await?; - let handle = main_tracker.spawn(migrator.run_migrations()); + let handle = main_tracker.spawn(migrator.run_migrations(false)); shutdown::graceful_with_handle(handle) .group(main_tracker, main_token) diff --git a/lib/sdf-server/src/config.rs b/lib/sdf-server/src/config.rs index 7d7502f951..dbe7e3249b 100644 --- a/lib/sdf-server/src/config.rs +++ b/lib/sdf-server/src/config.rs @@ -146,6 +146,9 @@ pub struct Config { #[builder(default)] audit: AuditDatabaseConfig, + + #[builder(default)] + dev_mode: bool, } impl StandardConfig for Config { @@ -264,6 +267,10 @@ impl Config { pub fn audit(&self) -> &AuditDatabaseConfig { &self.audit } + + pub fn dev_mode(&self) -> bool { + self.dev_mode + } } impl ConfigBuilder { @@ -287,6 +294,8 @@ pub struct ConfigFile { #[serde(default)] pub migration_mode: MigrationMode, #[serde(default)] + pub dev_mode: bool, + #[serde(default)] pub jwt_signing_public_key: JwtConfig, #[serde(default)] pub crypto: VeritechCryptoConfig, @@ -340,6 +349,7 @@ impl Default for ConfigFile { create_workspace_allowlist: Default::default(), spicedb: Default::default(), audit: Default::default(), + dev_mode: false, } } } @@ -375,6 +385,7 @@ impl TryFrom for Config { create_workspace_allowlist: value.create_workspace_allowlist, spicedb: value.spicedb, audit: value.audit, + dev_mode: value.dev_mode, }) } } @@ -513,6 +524,7 @@ fn buck2_development(config: &mut ConfigFile) -> Result<()> { config.spicedb.enabled = true; config.audit.pg.certificate_path = Some(postgres_cert.clone().try_into()?); config.audit.pg.dbname = audit_logs::database::DBNAME.to_string(); + config.dev_mode = true; Ok(()) } @@ -575,6 +587,7 @@ fn cargo_development(dir: String, config: &mut ConfigFile) -> Result<()> { config.spicedb.enabled = true; config.audit.pg.certificate_path = Some(postgres_cert.clone().try_into()?); config.audit.pg.dbname = audit_logs::database::DBNAME.to_string(); + config.dev_mode = true; Ok(()) } diff --git a/lib/sdf-server/src/migrations.rs b/lib/sdf-server/src/migrations.rs index 99cec223b8..ca63c5dc68 100644 --- a/lib/sdf-server/src/migrations.rs +++ b/lib/sdf-server/src/migrations.rs @@ -4,7 +4,7 @@ use audit_logs::database::{ AuditDatabaseContext, AuditDatabaseContextError, AuditDatabaseMigrationError, }; use dal::{ - cached_module::CachedModuleError, slow_rt::SlowRuntimeError, + cached_module::CachedModule, slow_rt::SlowRuntimeError, workspace_snapshot::migrator::SnapshotGraphMigrator, ServicesContext, }; use telemetry::prelude::*; @@ -25,14 +25,14 @@ pub enum MigratorError { Join(#[from] JoinError), #[error("error while migrating audit database: {0}")] MigrateAuditDatabase(#[source] AuditDatabaseMigrationError), + #[error("error while migrating cached modules: {0}")] + MigrateCachedModules(#[source] Box), #[error("error while migrating dal database: {0}")] MigrateDalDatabase(#[source] dal::ModelError), #[error("error while migrating layer db database: {0}")] MigrateLayerDbDatabase(#[source] si_layer_cache::LayerDbError), #[error("error while migrating snapshots: {0}")] MigrateSnapshots(#[source] Box), - #[error("module cache error: {0}")] - ModuleCache(#[from] CachedModuleError), #[error("module index url not set")] ModuleIndexNotSet, #[error("slow runtime: {0}")] @@ -46,6 +46,13 @@ impl MigratorError { { Self::MigrateSnapshots(Box::new(err)) } + + fn migrate_cached_modules(err: E) -> Self + where + E: std::error::Error + 'static + Sync + Send, + { + Self::MigrateCachedModules(Box::new(err)) + } } type MigratorResult = std::result::Result; @@ -97,7 +104,7 @@ impl Migrator { otel.status_message = Empty, ) )] - pub async fn run_migrations(self) -> MigratorResult<()> { + pub async fn run_migrations(self, update_module_cache: bool) -> MigratorResult<()> { let span = current_span_for_instrument_at!("info"); self.migrate_audit_database() @@ -116,6 +123,12 @@ impl Migrator { .await .map_err(|err| span.record_err(err))?; + if update_module_cache { + self.migrate_module_cache() + .await + .map_err(|err| span.record_err(err))?; + } + span.record_ok(); Ok(()) } @@ -165,4 +178,25 @@ impl Migrator { .map_err(MigratorError::migrate_snapshots)?; Ok(()) } + + #[instrument(name = "sdf.migrator.migrate_module_cache", level = "info", skip_all)] + async fn migrate_module_cache(&self) -> MigratorResult<()> { + let dal_context = self.services_context.clone().into_builder(true); + let ctx = dal_context + .build_default() + .await + .map_err(MigratorError::migrate_cached_modules)?; + + info!("Updating local module cache"); + + let new_modules = CachedModule::update_cached_modules(&ctx) + .await + .map_err(MigratorError::migrate_cached_modules)?; + info!( + "{} new builtin assets found in module index", + new_modules.len() + ); + + Ok(()) + } }