Disk cache for Debian

Cargo.toml

@@ -57,6 +57,7 @@ rune-modules = "0.13.4"
 rust-ini = "0.21.0"
 scopeguard = "1.2.0"
 serde = "1.0.204"
+serde_bytes = "0.11.15"
 serde_json = "1.0.120"
 smallvec = { version = "1.13.2", features = [
     "const_generics",

crates/konfigkoll/Cargo.toml

@@ -63,6 +63,7 @@ tokio = { workspace = true, features = [
 tracing-log.workspace = true
 tracing-subscriber = { workspace = true, features = ["env-filter", "parking_lot"] }
 tracing.workspace = true
+dashmap.workspace = true
 
 [target.'cfg(target_env = "musl")'.dependencies]
 # The allocator on musl is attrociously slow, so we use a custom one.

crates/konfigkoll/src/fs_scan.rs

@@ -4,7 +4,10 @@ use std::sync::Arc;
 
 use anyhow::Context;
 use compact_str::CompactString;
+use dashmap::DashMap;
+use itertools::Itertools;
 use ouroboros::self_referencing;
+use rayon::prelude::*;
 
 use konfigkoll_types::FsInstruction;
 use paketkoll_core::config::{
@@ -13,7 +16,7 @@ use paketkoll_core::config::{
 use paketkoll_core::file_ops::{
     canonicalize_file_entries, create_path_map, mismatching_and_unexpected_files,
 };
-use paketkoll_types::backend::Files;
+use paketkoll_types::backend::{Files, PackageMap};
 use paketkoll_types::files::FileEntry;
 use paketkoll_types::files::PathMap;
 use paketkoll_types::intern::Interner;
@@ -30,17 +33,31 @@ pub(crate) struct ScanResult {
 pub(crate) fn scan_fs(
     interner: &Arc<Interner>,
     backend: &Arc<dyn Files>,
+    package_map: &PackageMap,
     ignores: &[CompactString],
     trust_mtime: bool,
 ) -> anyhow::Result<(ScanResult, Vec<FsInstruction>)> {
     tracing::debug!("Scanning filesystem");
     let mut fs_instructions_sys = vec![];
-    let mut files = backend.files(interner).with_context(|| {
-        format!(
-            "Failed to collect information from backend {}",
-            backend.name()
-        )
-    })?;
+    let mut files = if backend.prefer_files_from_archive() {
+        let all = package_map.keys().cloned().collect::<Vec<_>>();
+        let files = backend.files_from_archives(&all, package_map, interner)?;
+        let file_map = DashMap::new();
+        files
+            .into_par_iter()
+            .flat_map_iter(|(_pkg, files)| files)
+            .for_each(|entry| {
+                file_map.insert(entry.path.clone(), entry);
+            });
+        file_map.into_iter().map(|(_, v)| v).collect_vec()
+    } else {
+        backend.files(interner).with_context(|| {
+            format!(
+                "Failed to collect information from backend {}",
+                backend.name()
+            )
+        })?
+    };
     if backend.may_need_canonicalization() {
         tracing::debug!("Canonicalizing file entries");
         canonicalize_file_entries(&mut files);

crates/konfigkoll/src/main.rs

@@ -21,6 +21,7 @@ use konfigkoll_core::state::DiffGoal;
 use konfigkoll_script::Phase;
 #[cfg(target_env = "musl")]
 use mimalloc::MiMalloc;
+use paketkoll_cache::FromArchiveCache;
 use paketkoll_cache::OriginalFilesCache;
 use paketkoll_core::backend::ConcreteBackend;
 use paketkoll_core::paketkoll_types::intern::Interner;
@@ -118,8 +119,19 @@ async fn main() -> anyhow::Result<()> {
         let backend = b
             .create_files(&backend_cfg, &interner)
             .with_context(|| format!("Failed to create backend {b}"))?;
+        let backend = if backend.prefer_files_from_archive() {
+            tracing::info!("Using archive cache for backend {}", backend.name());
+            // This is slow so we need to cache it
+            Box::new(
+                FromArchiveCache::from_path(backend, proj_dirs.cache_dir())
+                    .context("Failed to create archive disk cache")?,
+            )
+        } else {
+            // This is fast so we don't need to cache it
+            backend
+        };
         let backend = OriginalFilesCache::from_path(backend, proj_dirs.cache_dir())
-            .context("Failed to create disk cache")?;
+            .context("Failed to create original files disk cache")?;
         Arc::new(backend)
     };
 
@@ -133,6 +145,10 @@ async fn main() -> anyhow::Result<()> {
     // Script: Get FS ignores
     script_engine.run_phase(Phase::Ignores).await?;
 
+    tracing::info!("Waiting for package loading results...");
+    let (pkgs_sys, package_maps) = package_loader.await??;
+    tracing::info!("Got package loading results");
+
     // Do FS scan
     tracing::info!("Starting filesystem scan background job");
     let fs_instructions_sys = {
@@ -146,18 +162,24 @@ async fn main() -> anyhow::Result<()> {
         let trust_mtime = cli.trust_mtime;
         let interner = interner.clone();
         let backends_files = backend_files.clone();
+        let package_map = package_maps
+            .get(&backend_files.as_backend_enum())
+            .expect("No matching package backend?")
+            .clone();
         tokio::task::spawn_blocking(move || {
-            fs_scan::scan_fs(&interner, &backends_files, &ignores, trust_mtime)
+            fs_scan::scan_fs(
+                &interner,
+                &backends_files,
+                &package_map,
+                &ignores,
+                trust_mtime,
+            )
         })
     };
 
     // Script: Do early package phase
     script_engine.run_phase(Phase::ScriptDependencies).await?;
 
-    tracing::info!("Waiting for package loading results...");
-    let (pkgs_sys, package_maps) = package_loader.await??;
-    tracing::info!("Got package loading results");
-
     // Create the set of package managers for use by the script
     script_engine.state_mut().setup_package_managers(
         &backends_pkg,

crates/konfigkoll/src/pkgs.rs

@@ -10,6 +10,7 @@ use konfigkoll_types::PkgInstructions;
 use paketkoll_types::{
     backend::{Backend, PackageBackendMap, PackageMap, PackageMapMap},
     intern::Interner,
+    package::PackageInstallStatus,
 };
 
 #[tracing::instrument(skip_all)]
@@ -31,9 +32,11 @@ pub(crate) fn load_packages(
                         backend.name()
                     )
                 })
-                .map(|backend_pkgs| {
+                .map(|mut backend_pkgs| {
+                    // Because we can have partially installed packages on Debian...
+                    backend_pkgs.retain(|pkg| pkg.status == PackageInstallStatus::Installed);
                     let pkg_map = Arc::new(paketkoll_types::backend::packages_to_package_map(
-                        backend_pkgs.clone(),
+                        backend_pkgs.iter(),
                     ));
                     let pkg_instructions =
                         konfigkoll_core::conversion::convert_packages_to_pkg_instructions(

crates/paketkoll/src/cli.rs

@@ -59,6 +59,11 @@ pub enum Commands {
         /// Path to query
         path: String,
     },
+    #[clap(hide = true)]
+    DebugPackageFileData {
+        /// Package to query
+        package: String,
+    },
 }
 
 /// Output format to use

crates/paketkoll/src/conversion.rs

@@ -60,6 +60,7 @@ impl TryFrom<&Cli> for paketkoll_core::backend::BackendConfiguration {
             Commands::InstalledPackages => {}
             Commands::OriginalFile { .. } => {}
             Commands::Owns { .. } => {}
+            Commands::DebugPackageFileData { .. } => {}
         }
         Ok(builder.build()?)
     }

crates/paketkoll/src/main.rs

@@ -118,6 +118,31 @@ fn main() -> anyhow::Result<Exit> {
             }
             Ok(Exit::new(Code::SUCCESS))
         }
+        Commands::DebugPackageFileData { ref package } => {
+            let interner = Interner::new();
+            let backend: paketkoll_core::backend::ConcreteBackend = cli.backend.try_into()?;
+            let backend_impl = backend
+                .create_full(&(&cli).try_into()?, &interner)
+                .context("Failed to create backend")?;
+
+            let package_map = backend_impl
+                .package_map_complete(&interner)
+                .with_context(|| format!("Failed to collect information from backend {backend}"))?;
+
+            let pkg_ref = PackageRef::get_or_intern(&interner, package);
+
+            let files = backend_impl
+                .files_from_archives(&[pkg_ref], &package_map, &interner)
+                .with_context(|| {
+                    format!(
+                        "Failed to collect file information for package {package} from backend {backend}"
+                    )
+                })?;
+
+            println!("{:?}", files);
+
+            Ok(Exit::new(Code::SUCCESS))
+        }
     }
 }
 

crates/paketkoll_cache/Cargo.toml

@@ -18,7 +18,11 @@ cached = { workspace = true, features = [
 ], default-features = false }
 compact_str.workspace = true
 dashmap.workspace = true
-paketkoll_types = { version = "0.1.0", path = "../paketkoll_types" }
+paketkoll_types = { version = "0.1.0", path = "../paketkoll_types", features = [
+    "serde",
+] }
+rayon.workspace = true
+serde = { workspace = true, features = ["derive"] }
 tracing.workspace = true
 
 [lints]