From a261f32d65930ffa9106cb327d8a3341c127c71c Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Thu, 14 Sep 2023 16:24:33 +0530 Subject: [PATCH] incremental build (#1146) * no_static -> build_static_files * refactored out incremental_build() * moved the caching utilities to fastn_core::utils * cache stubs * better cache plumbing * some refactor * better replacement when constructing file names * utility should live in utils.rs * removed edition 2021 support from fastn build * removing un-needed functions * build-dir plumbing * not sure what was happening there, cleanuped code * fastn build working * some caching implementation * get_build_content() implementation * cache working, after ~3m, subsequent builds taking ~10s * refactored out is_cached() * implemented ftd content hash check * cleaned up some code * virtual file handling, ftd paths are incorrect * some changes, not working yet * some debug statements * implement dependency resolver for incremental build * created overwrite function to update ftd files * fixed infinite loop in dependency resolution * Fixed dependency resolver * Handle cache miss * minor fixes * removed unneccessary code * build path write fix for images * fixed unresolved_dependencies files index * testing stuff * testing stuff 2 * Minor fix * Log status for testing * check for circular dependency * removed all bugs in incremental build * refactored code * Minor fixes * build assets * handle deleted documents * minor fixes * proper document name check * Remove deleted docs from build * file exist check before deleting * better path construction * clippy fixes * Merged main * remove package name from dependencies * no need to remove package name from doc id * commented out print statements * Fixed dependency naming bug * delete empty folders from build * Path separator fix * conditional ftd.redirect * fix windows cache dir * id to cache key fix * Delete ftd/t/js/test.html Signed-off-by: Harsh Singh <64768386+harshdoesdev@users.noreply.github.com> * fixed windows bugs * fixed conflicts * updated files * clippy fixes * refactored code --------- Signed-off-by: Harsh Singh <64768386+harshdoesdev@users.noreply.github.com> Co-authored-by: Harsh Singh Co-authored-by: heulitig Co-authored-by: Harsh Singh <64768386+harshdoesdev@users.noreply.github.com> --- fastn-core/src/commands/build.rs | 765 +++++++++++++++++--------- fastn-core/src/config/mod.rs | 2 + fastn-core/src/doc.rs | 71 +-- fastn-core/src/lib.rs | 149 ----- fastn-core/src/library2022/mod.rs | 49 +- fastn-core/src/package/package_doc.rs | 59 +- fastn-core/src/utils.rs | 124 ++++- 7 files changed, 692 insertions(+), 527 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 80e0f32566..912d5cb7b3 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -8,7 +8,6 @@ pub async fn build( check_build: bool, ) -> fastn_core::Result<()> { tokio::fs::create_dir_all(config.build_dir()).await?; - let documents = get_documents_for_current_package(config).await?; // Default css and js default_build_files( @@ -18,9 +17,17 @@ pub async fn build( ) .await?; - if let Some(id) = only_id { - handle_only_id(id, config, base_url, ignore_failed, test, documents).await?; - return Ok(()); + { + let documents = get_documents_for_current_package(config).await?; + + match only_id { + Some(id) => { + return handle_only_id(id, config, base_url, ignore_failed, test, documents).await + } + None => { + incremental_build(config, &documents, base_url, ignore_failed, test).await?; + } + } } // All redirect html files under .build @@ -49,15 +56,355 @@ pub async fn build( } } - for document in documents.values() { - handle_file(document, config, base_url, ignore_failed, test, false).await?; - } - config.download_fonts().await?; if check_build { return fastn_core::post_build_check(config).await; } + + Ok(()) +} + +mod build_dir { + pub(crate) fn get_build_content() -> std::io::Result> + { + let mut b = std::collections::BTreeMap::new(); + + for f in find_all_files_recursively(".build") { + b.insert( + f.to_string_lossy().to_string().replacen(".build/", "", 1), + fastn_core::utils::generate_hash(std::fs::read(&f)?), + ); + } + + Ok(b) + } + + fn find_all_files_recursively( + dir: impl AsRef + std::fmt::Debug, + ) -> Vec { + let mut files = vec![]; + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_dir() { + files.extend(find_all_files_recursively(&path)); + } else { + files.push(path) + } + } + files + } +} + +mod cache { + use super::is_virtual_dep; + + const FILE_NAME: &str = "fastn.cache"; + + pub(crate) fn get() -> std::io::Result<(bool, Cache)> { + let (cache_hit, mut v) = match fastn_core::utils::get_cached(FILE_NAME) { + Some(v) => { + tracing::debug!("cached hit"); + (true, v) + } + None => { + tracing::debug!("cached miss"); + ( + false, + Cache { + build_content: std::collections::BTreeMap::new(), + ftd_cache: std::collections::BTreeMap::new(), + documents: std::collections::BTreeMap::new(), + file_checksum: std::collections::BTreeMap::new(), + }, + ) + } + }; + v.build_content = super::build_dir::get_build_content()?; + Ok((cache_hit, v)) + } + + #[derive(serde::Serialize, serde::Deserialize, Debug)] + pub(crate) struct Cache { + // fastn_version: String, // TODO + #[serde(skip)] + pub(crate) build_content: std::collections::BTreeMap, + #[serde(skip)] + pub(crate) ftd_cache: std::collections::BTreeMap>, + pub(crate) documents: std::collections::BTreeMap, + pub(crate) file_checksum: std::collections::BTreeMap, + } + + impl Cache { + pub(crate) fn cache_it(&self) -> fastn_core::Result<()> { + fastn_core::utils::cache_it(FILE_NAME, self)?; + Ok(()) + } + pub(crate) fn get_file_hash(&mut self, path: &str) -> fastn_core::Result { + if is_virtual_dep(path) { + // these are virtual file, they don't exist on disk, and hash only changes when + // fastn source changes + return Ok("hello".to_string()); + } + match self.ftd_cache.get(path) { + Some(Some(v)) => Ok(v.to_owned()), + Some(None) => Err(fastn_core::Error::GenericError(path.to_string())), + None => { + let hash = match fastn_core::utils::get_ftd_hash(path) { + Ok(v) => v, + Err(e) => { + self.ftd_cache.insert(path.to_string(), None); + return Err(e); + } + }; + self.ftd_cache.insert(path.to_string(), Some(hash.clone())); + Ok(hash) + } + } + } + } + + #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] + pub(crate) struct File { + pub(crate) path: String, + pub(crate) checksum: String, + } + + #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] + pub(crate) struct Document { + pub(crate) html_checksum: String, + pub(crate) dependencies: Vec, + } +} + +fn get_dependency_name_without_package_name(package_name: &str, dependency_name: &str) -> String { + if let Some(remaining) = dependency_name.strip_prefix(&format!("{}/", package_name)) { + remaining.to_string() + } else { + dependency_name.to_string() + } + .trim_end_matches('/') + .to_string() +} + +fn is_virtual_dep(path: &str) -> bool { + let path = std::path::Path::new(path); + + path.starts_with("$fastn$/") + || path.ends_with("/-/fonts.ftd") + || path.ends_with("/-/assets.ftd") +} + +async fn handle_dependency_file( + config: &mut fastn_core::Config, + cache: &mut cache::Cache, + documents: &std::collections::BTreeMap, + base_url: &str, + ignore_failed: bool, + test: bool, + name_without_package_name: String, +) -> fastn_core::Result<()> { + for document in documents.values() { + if remove_extension(document.get_id()).eq(name_without_package_name.as_str()) + || remove_extension(&document.get_id_with_package()) + .eq(name_without_package_name.as_str()) + { + handle_file( + document, + config, + base_url, + ignore_failed, + test, + true, + Some(cache), + ) + .await?; + } + } + + Ok(()) +} + +// removes deleted documents from cache and build folder +fn remove_deleted_documents( + config: &mut fastn_core::Config, + c: &mut cache::Cache, + documents: &std::collections::BTreeMap, +) -> fastn_core::Result<()> { + use itertools::Itertools; + + let removed_documents = c + .documents + .keys() + .filter(|cached_document_id| { + for document in documents.values() { + if remove_extension(document.get_id()).eq(cached_document_id.as_str()) + || remove_extension(&document.get_id_with_package()) + .eq(cached_document_id.as_str()) + { + return false; + } + } + + true + }) + .map(|id| id.to_string()) + .collect_vec(); + + for removed_doc_id in &removed_documents { + let folder_path = config.build_dir().join(removed_doc_id); + let folder_parent = folder_path.parent(); + let file_path = &folder_path.with_extension("ftd"); + + if file_path.exists() { + std::fs::remove_file(file_path)?; + } + + std::fs::remove_dir_all(&folder_path)?; + + // If the parent folder of the file's output folder is also empty, delete it as well. + if let Some(folder_parent) = folder_parent { + if folder_parent.read_dir()?.count().eq(&0) { + std::fs::remove_dir_all(folder_parent)?; + } + } + + c.documents.remove(removed_doc_id); + } + + Ok(()) +} + +#[tracing::instrument(skip(config, documents))] +async fn incremental_build( + config: &mut fastn_core::Config, + documents: &std::collections::BTreeMap, + base_url: &str, + ignore_failed: bool, + test: bool, +) -> fastn_core::Result<()> { + // https://fastn.com/rfc/incremental-build/ + use itertools::Itertools; + + let (cache_hit, mut c) = cache::get()?; + + if cache_hit { + let mut unresolved_dependencies: Vec = documents + .iter() + .filter(|(_, f)| f.is_ftd()) + .map(|(_, f)| remove_extension(f.get_id())) + .collect_vec(); + let mut resolved_dependencies: Vec = vec![]; + let mut resolving_dependencies: Vec = vec![]; + + while let Some(unresolved_dependency) = unresolved_dependencies.pop() { + // println!("Current UR: {}", unresolved_dependency.as_str()); + if let Some(doc) = c.documents.get(unresolved_dependency.as_str()) { + // println!( + // "[INCREMENTAL BUILD][CACHE FOUND] Processing: {}", + // &unresolved_dependency + // ); + + let mut own_resolved_dependencies: Vec = vec![]; + + let dependencies: Vec = doc + .dependencies + .iter() + .map(|dep| get_dependency_name_without_package_name(&config.package.name, dep)) + .collect_vec(); + + for dep in &dependencies { + if resolved_dependencies.contains(dep) || dep.eq(&unresolved_dependency) { + own_resolved_dependencies.push(dep.to_string()); + continue; + } + unresolved_dependencies.push(dep.to_string()); + } + + // println!( + // "[INCREMENTAL] [R]: {} [RV]: {} [UR]: {} [ORD]: {}", + // &resolved_dependencies.len(), + // &resolving_dependencies.len(), + // &unresolved_dependencies.len(), + // own_resolved_dependencies.len(), + // ); + + if own_resolved_dependencies.eq(&dependencies) { + handle_dependency_file( + config, + &mut c, + documents, + base_url, + ignore_failed, + test, + unresolved_dependency.to_string(), + ) + .await?; + + resolved_dependencies.push(unresolved_dependency.to_string()); + if unresolved_dependencies.is_empty() { + if let Some(resolving_dependency) = resolving_dependencies.pop() { + if resolving_dependency.eq(&unresolved_dependency.as_str()) { + // println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); + continue; + } + unresolved_dependencies.push(resolving_dependency); + } + } + } else { + // println!("Adding to RD: {}", unresolved_dependency.as_str()); + resolving_dependencies.push(unresolved_dependency.to_string()); + } + } else { + if is_virtual_dep(&unresolved_dependency) { + resolved_dependencies.push(unresolved_dependency.clone()); + } else { + // println!("Not found in cache UR: {}", unresolved_dependency.as_str()); + + handle_dependency_file( + config, + &mut c, + documents, + base_url, + ignore_failed, + test, + unresolved_dependency.to_string(), + ) + .await?; + + resolved_dependencies.push(unresolved_dependency.clone()); + } + if unresolved_dependencies.is_empty() { + if let Some(resolving_dependency) = resolving_dependencies.pop() { + if resolving_dependency.eq(&unresolved_dependency.as_str()) { + // println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); + continue; + } + unresolved_dependencies.push(resolving_dependency); + } + } + } + } + + remove_deleted_documents(config, &mut c, documents)?; + } else { + for document in documents.values() { + handle_file( + document, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; + } + } + + c.cache_it()?; + Ok(()) } @@ -72,7 +419,7 @@ async fn handle_only_id( ) -> fastn_core::Result<()> { for doc in documents.values() { if doc.get_id().eq(id) || doc.get_id_with_package().eq(id) { - return handle_file(doc, config, base_url, ignore_failed, test, true).await; + return handle_file(doc, config, base_url, ignore_failed, test, false, None).await; } } @@ -89,14 +436,22 @@ async fn handle_file( base_url: &str, ignore_failed: bool, test: bool, - no_static: bool, + build_static_files: bool, + cache: Option<&mut cache::Cache>, ) -> fastn_core::Result<()> { let start = std::time::Instant::now(); print!("Processing {} ... ", document.get_id_with_package()); - let process_status = - handle_file_(document, config, base_url, ignore_failed, test, no_static).await; - + let process_status = handle_file_( + document, + config, + base_url, + ignore_failed, + test, + build_static_files, + cache, + ) + .await; if process_status.is_ok() { fastn_core::utils::print_end( format!( @@ -123,48 +478,159 @@ async fn handle_file( Ok(()) } -#[tracing::instrument(skip(document, config))] +fn is_cached<'a>( + cache: Option<&'a mut cache::Cache>, + doc: &fastn_core::Document, + file_path: &str, +) -> (Option<&'a mut cache::Cache>, bool) { + let cache: &mut cache::Cache = match cache { + Some(c) => c, + None => { + // println!("cache miss: no have cache"); + return (cache, false); + } + }; + + let id = remove_extension(doc.id.as_str()); + + let cached_doc: cache::Document = match cache.documents.get(id.as_str()).cloned() { + Some(cached_doc) => cached_doc, + None => { + // println!("cache miss: no cache entry for {}", id.as_str()); + return (Some(cache), false); + } + }; + + // if it exists, check if the checksums match + // if they do, return + // dbg!(&cached_doc); + let doc_hash = match cache.build_content.get(file_path) { + Some(doc_hash) => doc_hash, + None => { + // println!("cache miss: document not present in .build: {}", file_path); + return (Some(cache), false); + } + }; + + // dbg!(doc_hash); + + if doc_hash != &cached_doc.html_checksum { + // println!("cache miss: html file checksums don't match"); + return (Some(cache), false); + } + + let file_checksum = match cache.file_checksum.get(id.as_str()).cloned() { + Some(file_checksum) => file_checksum, + None => { + // println!("cache miss: no cache entry for {}", id.as_str()); + return (Some(cache), false); + } + }; + + if file_checksum != fastn_core::utils::generate_hash(doc.content.as_str()) { + // println!("cache miss: ftd file checksums don't match"); + return (Some(cache), false); + } + + for dep in &cached_doc.dependencies { + let file_checksum = match cache.file_checksum.get(dep) { + None => { + // println!("cache miss: file {} not present in cache", dep); + return (Some(cache), false); + } + Some(file_checksum) => file_checksum.clone(), + }; + + let current_hash = match cache.get_file_hash(dep.as_str()) { + Ok(hash) => hash, + Err(_) => { + // println!("cache miss: dependency {} not present current folder", dep); + return (Some(cache), false); + } + }; + + if file_checksum != current_hash { + // println!("cache miss: dependency {} checksums don't match", dep); + return (Some(cache), false); + } + } + + // println!("cache hit"); + (Some(cache), true) +} + +fn remove_extension(id: &str) -> String { + if id.ends_with("/index.ftd") { + fastn_core::utils::replace_last_n(id, 1, "/index.ftd", "") + } else { + fastn_core::utils::replace_last_n(id, 1, ".ftd", "") + } +} + +#[tracing::instrument(skip(document, config, cache))] async fn handle_file_( document: &fastn_core::File, config: &mut fastn_core::Config, base_url: &str, ignore_failed: bool, test: bool, - no_static: bool, + build_static_files: bool, + cache: Option<&mut cache::Cache>, ) -> fastn_core::Result<()> { config.current_document = Some(document.get_id().to_string()); + config.dependencies_during_render = vec![]; match document { fastn_core::File::Ftd(doc) => { - if !config - .ftd_edition - .eq(&fastn_core::config::FTDEdition::FTD2021) - { - // Ignore redirect paths - if let Some(r) = config.package.redirects.as_ref() { - if fastn_core::package::redirects::find_redirect(r, doc.id.as_str()).is_some() { - println!("Ignored by redirect {}", doc.id.as_str()); - return Ok(()); - } - } + let file_path = if doc.id.eq("404.ftd") { + "404.html".to_string() + } else if doc.id.ends_with("index.ftd") { + fastn_core::utils::replace_last_n(doc.id.as_str(), 1, "index.ftd", "index.html") + } else { + fastn_core::utils::replace_last_n(doc.id.as_str(), 1, ".ftd", "/index.html") + }; - fastn_core::utils::copy( - config.root.join(doc.id.as_str()), - config.root.join(".build").join(doc.id.as_str()), - ) - .await - .ok(); + let (cache, is_cached) = is_cached(cache, doc, file_path.as_str()); + if is_cached { + return Ok(()); + } - if doc.id.eq("FASTN.ftd") { - return Ok(()); - } + fastn_core::utils::copy( + config.root.join(doc.id.as_str()), + config.root.join(".build").join(doc.id.as_str()), + ) + .await + .ok(); + + if doc.id.eq("FASTN.ftd") { + return Ok(()); } + let resp = fastn_core::package::package_doc::process_ftd( - config, doc, base_url, no_static, test, + config, + doc, + base_url, + build_static_files, + test, + file_path.as_str(), ) .await; match (resp, ignore_failed) { - (Ok(_), _) => (), + (Ok(r), _) => { + if let Some(cache) = cache { + cache.documents.insert( + remove_extension(doc.id.as_str()), + cache::Document { + html_checksum: r.checksum(), + dependencies: config.dependencies_during_render.clone(), + }, + ); + cache.file_checksum.insert( + remove_extension(doc.id.as_str()), + fastn_core::utils::generate_hash(doc.content.as_str()), + ); + } + } (_, true) => { print!("Failed "); return Ok(()); @@ -175,74 +641,26 @@ async fn handle_file_( } } fastn_core::File::Static(sa) => process_static(sa, &config.root, &config.package).await?, - fastn_core::File::Markdown(doc) => { - if !config - .ftd_edition - .eq(&fastn_core::config::FTDEdition::FTD2021) - { - // TODO: bring this feature back - print!("Skipped "); - return Ok(()); - } - let resp = process_markdown(config, doc, base_url, no_static, test).await; - match (resp, ignore_failed) { - (Ok(r), _) => r, - (_, true) => { - print!("Failed "); - return Ok(()); - } - (e, _) => { - return e; - } - } + fastn_core::File::Markdown(_doc) => { + // TODO: bring this feature back + print!("Skipped "); + return Ok(()); } fastn_core::File::Image(main_doc) => { process_static(main_doc, &config.root, &config.package).await?; - if config - .ftd_edition - .eq(&fastn_core::config::FTDEdition::FTD2021) - { - let resp = process_image(config, main_doc, base_url, no_static, test).await; - match (resp, ignore_failed) { - (Ok(r), _) => r, - (_, true) => { - print!("Failed "); - return Ok(()); - } - (e, _) => { - return e; - } - } - } } fastn_core::File::Code(doc) => { process_static( &fastn_core::Static { package_name: config.package.name.to_string(), id: doc.id.to_string(), - content: vec![], + content: doc.content.clone().into_bytes(), base_path: camino::Utf8PathBuf::from(doc.parent_path.as_str()), }, &config.root, &config.package, ) .await?; - if config - .ftd_edition - .eq(&fastn_core::config::FTDEdition::FTD2021) - { - let resp = process_code(config, doc, base_url, no_static, test).await; - match (resp, ignore_failed) { - (Ok(r), _) => r, - (_, true) => { - print!("Failed "); - return Ok(()); - } - (e, _) => { - return e; - } - } - } } } @@ -383,165 +801,20 @@ async fn process_static( .join("-") .join(package.name.as_str()); - std::fs::create_dir_all(&build_path)?; - if let Some((dir, _)) = sa.id.rsplit_once('/') { - std::fs::create_dir_all(build_path.join(dir))?; - } - std::fs::copy( - sa.base_path.join(sa.id.as_str()), - build_path.join(sa.id.as_str()), - )?; - + let full_file_path = build_path.join(sa.id.as_str()); + let (file_root, _file_name) = if let Some((file_root, file_name)) = full_file_path + .as_str() + .rsplit_once(std::path::MAIN_SEPARATOR) { - // TODO: need to remove this once download_base_url is removed - std::fs::create_dir_all(base_path.join(".build"))?; - if let Some((dir, _)) = sa.id.rsplit_once('/') { - std::fs::create_dir_all(base_path.join(".build").join(dir))?; - } - - std::fs::copy( - sa.base_path.join(sa.id.as_str()), - base_path.join(".build").join(sa.id.as_str()), - )?; - } - Ok(()) - } -} - -async fn process_image( - config: &mut fastn_core::Config, - main: &fastn_core::Static, - base_url: &str, - no_static: bool, - test: bool, -) -> fastn_core::Result<()> { - let main = convert_to_ftd(config, main)?; - - fastn_core::package::package_doc::process_ftd(config, &main, base_url, no_static, test).await?; - return Ok(()); - - fn convert_to_ftd( - config: &fastn_core::Config, - doc: &fastn_core::Static, - ) -> fastn_core::Result { - Ok(fastn_core::Document { - package_name: config.package.name.to_string(), - id: convert_to_ftd_extension(doc.id.as_str())?, - content: fastn_core::package_info_image(config, doc, &config.package)?, - parent_path: doc.base_path.to_string(), - }) - } - - fn convert_to_ftd_extension(name: &str) -> fastn_core::Result { - Ok(format!("{}.ftd", name)) - } -} - -async fn process_code( - config: &mut fastn_core::Config, - main: &fastn_core::Document, - base_url: &str, - no_static: bool, - test: bool, -) -> fastn_core::Result<()> { - let main = if let Some(main) = convert_to_ftd(config, main)? { - main - } else { - return Ok(()); - }; - - fastn_core::package::package_doc::process_ftd(config, &main, base_url, no_static, test).await?; - return Ok(()); - - fn convert_to_ftd( - config: &fastn_core::Config, - doc: &fastn_core::Document, - ) -> fastn_core::Result> { - let id = convert_to_ftd_extension(doc.id.as_str())?; - let ext = fastn_core::utils::get_extension(doc.id.as_str())?; - let new_content = - fastn_core::package_info_code(config, id.as_str(), doc.content.as_str(), ext.as_str())?; - - let new_doc = { - let mut new_doc = doc.to_owned(); - new_doc.content = new_content; - new_doc.id = id; - new_doc - }; - - Ok(Some(new_doc)) - } - - fn convert_to_ftd_extension(name: &str) -> fastn_core::Result { - Ok(format!("{}.ftd", name)) - } -} - -async fn process_markdown( - config: &mut fastn_core::Config, - main: &fastn_core::Document, - base_url: &str, - no_static: bool, - test: bool, -) -> fastn_core::Result<()> { - let main = if let Some(main) = convert_md_to_ftd(config, main)? { - main - } else { - return Ok(()); - }; - fastn_core::package::package_doc::process_ftd(config, &main, base_url, no_static, test).await?; - return Ok(()); - - fn convert_md_to_ftd( - config: &fastn_core::Config, - doc: &fastn_core::Document, - ) -> fastn_core::Result> { - let doc_id = if doc.id == "README.md" - && !(camino::Utf8Path::new(format!(".{}index.ftd", std::path::MAIN_SEPARATOR).as_str()) - .exists() - || camino::Utf8Path::new( - format!(".{}index.md", std::path::MAIN_SEPARATOR).as_str(), - ) - .exists()) - { - "index.md".to_string() - } else if !camino::Utf8Path::new( - format!( - ".{}{}", - std::path::MAIN_SEPARATOR, - convert_md_to_ftd_extension(doc.id.as_str())? - ) - .as_str(), - ) - .exists() - { - doc.id.to_string() + (file_root.to_string(), file_name.to_string()) } else { - return Ok(None); - }; - let id = convert_md_to_ftd_extension(doc_id.as_str())?; - let new_content = - fastn_core::package_info_markdown(config, id.as_str(), doc.content.as_str())?; - - let new_doc = { - let mut new_doc = doc.to_owned(); - new_doc.content = new_content; - new_doc.id = id; - new_doc + ("".to_string(), full_file_path.to_string()) }; - Ok(Some(new_doc)) - } - - fn convert_md_to_ftd_extension(name: &str) -> fastn_core::Result { - let file_name = if let Some(p1) = name.strip_suffix(".md") { - p1 - } else { - return Err(fastn_core::Error::UsageError { - message: format!("expected md file found: `{}`", name), - }); - }; - - Ok(format!("{}.ftd", file_name)) + if !base_path.join(&file_root).exists() { + std::fs::create_dir_all(base_path.join(&file_root))?; + } + std::fs::write(full_file_path, &sa.content)?; + Ok(()) } } diff --git a/fastn-core/src/config/mod.rs b/fastn-core/src/config/mod.rs index 841a9400a3..6b3a309cde 100644 --- a/fastn-core/src/config/mod.rs +++ b/fastn-core/src/config/mod.rs @@ -39,6 +39,7 @@ pub struct Config { pub named_parameters: Vec<(String, ftd::Value)>, pub extra_data: std::collections::BTreeMap, pub current_document: Option, + pub dependencies_during_render: Vec, pub request: Option, // TODO: It should only contain reference pub ftd_edition: FTDEdition, pub ftd_external_js: Vec, @@ -1291,6 +1292,7 @@ impl Config { ftd_inline_js: Default::default(), ftd_external_css: Default::default(), ftd_inline_css: Default::default(), + dependencies_during_render: Default::default(), }; // Update global_ids map from the current package files diff --git a/fastn-core/src/doc.rs b/fastn-core/src/doc.rs index 5f67c38929..8eb649e016 100644 --- a/fastn-core/src/doc.rs +++ b/fastn-core/src/doc.rs @@ -11,7 +11,7 @@ fn cached_parse( let hash = fastn_core::utils::generate_hash(source); - if let Some(c) = get_cached::(id) { + if let Some(c) = fastn_core::utils::get_cached::(id) { if c.hash == hash { tracing::debug!("cache hit"); return Ok(c.doc); @@ -22,50 +22,7 @@ fn cached_parse( } let doc = ftd::interpreter::ParsedDocument::parse_with_line_number(id, source, line_number)?; - cache_it(id, C { doc, hash }).map(|v| v.doc) -} - -fn id_to_cache_key(id: &str) -> String { - id.replace('/', "_") -} - -fn get_cached(id: &str) -> Option -where - T: serde::de::DeserializeOwned, -{ - let cache_file = dirs::cache_dir()? - .join("fastn.com/ast-cache/") - .join(id_to_cache_key(id)); - serde_json::from_str( - &std::fs::read_to_string(cache_file) - .map_err(|e| { - tracing::debug!("file read error: {}", e.to_string()); - e - }) - .ok()?, - ) - .map_err(|e| { - tracing::debug!("not valid json: {}", e.to_string()); - e - }) - .ok() -} - -fn cache_it(id: &str, d: T) -> ftd::interpreter::Result -where - T: serde::ser::Serialize, -{ - let cache_file = dirs::cache_dir() - .ok_or_else(|| ftd::interpreter::Error::OtherError("cache dir not found".to_string()))? - .join("fastn.com/ast-cache/") - .join(id_to_cache_key(id)); - std::fs::create_dir_all(cache_file.parent().unwrap()).map_err(|e| { - ftd::interpreter::Error::OtherError(format!("failed to create cache dir: {}", e)) - })?; - std::fs::write(cache_file, serde_json::to_string(&d)?).map_err(|e| { - ftd::interpreter::Error::OtherError(format!("failed to write cache file: {}", e)) - })?; - Ok(d) + fastn_core::utils::cache_it(id, C { doc, hash }).map(|v| v.doc) } #[tracing::instrument(skip_all)] @@ -96,9 +53,10 @@ pub async fn interpret_helper<'a>( state: mut st, caller_module, } => { - let (source, foreign_variable, foreign_function, ignore_line_numbers) = + let (source, path, foreign_variable, foreign_function, ignore_line_numbers) = resolve_import_2022(lib, &mut st, module.as_str(), caller_module.as_str()) .await?; + lib.config.dependencies_during_render.push(path); let doc = cached_parse(module.as_str(), source.as_str(), ignore_line_numbers)?; s = st.continue_after_import( module.as_str(), @@ -205,13 +163,20 @@ pub async fn resolve_import_2022<'a>( _state: &mut ftd::interpreter::InterpreterState, module: &str, caller_module: &str, -) -> ftd::interpreter::Result<(String, Vec, Vec, usize)> { +) -> ftd::interpreter::Result<(String, String, Vec, Vec, usize)> { let current_package = lib.get_current_package(caller_module)?; let source = if module.eq("fastn/time") { - ("".to_string(), vec!["time".to_string()], vec![], 0) + ( + "".to_string(), + "$fastn$/time.ftd".to_string(), + vec!["time".to_string()], + vec![], + 0, + ) } else if module.eq("fastn/processors") { ( fastn_core::processor_ftd().to_string(), + "$fastn$/processors.ftd".to_string(), vec![], vec![ "figma-typo-token".to_string(), @@ -249,12 +214,14 @@ pub async fn resolve_import_2022<'a>( if module.starts_with(current_package.name.as_str()) { ( current_package.get_font_ftd().unwrap_or_default(), + format!("{name}/-/assets.ftd", name = current_package.name), foreign_variable, vec![], 0, ) } else { let mut font_ftd = "".to_string(); + let mut path = "".to_string(); for (alias, package) in current_package.aliases() { if module.starts_with(alias) { lib.push_package_under_process(module, package).await?; @@ -266,16 +233,18 @@ pub async fn resolve_import_2022<'a>( .unwrap() .get_font_ftd() .unwrap_or_default(); + path = format!("{name}/-/fonts.ftd", name = package.name); break; } } - (font_ftd, foreign_variable, vec![], 0) + (font_ftd, path, foreign_variable, vec![], 0) } } else { - let (content, ignore_line_numbers) = lib.get_with_result(module, caller_module).await?; - + let (content, path, ignore_line_numbers) = + lib.get_with_result(module, caller_module).await?; ( content, + path, vec![], vec![ "figma-typo-token".to_string(), diff --git a/fastn-core/src/lib.rs b/fastn-core/src/lib.rs index 58e5b7b717..e51b2de802 100644 --- a/fastn-core/src/lib.rs +++ b/fastn-core/src/lib.rs @@ -91,38 +91,6 @@ fn fastn_lib_ftd() -> &'static str { include_str!("../ftd/fastn-lib.ftd") } -fn package_info_image( - config: &fastn_core::Config, - doc: &fastn_core::Static, - package: &fastn_core::Package, -) -> fastn_core::Result { - let path = config.root.join("fastn").join("image.ftd"); - Ok(if path.is_file() { - std::fs::read_to_string(path)? - } else { - let body_prefix = match config.package.generate_prefix_string(false) { - Some(bp) => bp, - None => String::new(), - }; - indoc::formatdoc! {" - {body_prefix} - - -- import: {package_info_package}/image as pi - - -- ftd.image-src src: {src} - dark: {src} - - -- pi.image-page: {file_name} - src: $src - ", - body_prefix = body_prefix, - file_name = doc.id, - package_info_package = config.package_info_package(), - src = format!("-/{}/{}", package.name.as_str(), doc.id.as_str()), - } - }) -} - fn package_info_about(config: &fastn_core::Config) -> fastn_core::Result { let path = config.root.join("fastn").join("cr.ftd"); Ok(if path.is_file() { @@ -229,123 +197,6 @@ fn package_info_create_cr(config: &fastn_core::Config) -> fastn_core::Result fastn_core::Result { - let path = config.root.join("fastn").join("code.ftd"); - Ok(if path.is_file() { - std::fs::read_to_string(path)? - } else { - let body_prefix = match config.package.generate_prefix_string(false) { - Some(bp) => bp, - None => String::new(), - }; - if content.trim().is_empty() { - format!( - indoc::indoc! {" - {body_prefix} - - -- import: {package_info_package}/code as pi - - -- pi.code-page: {file_name} - lang: {ext} - - "}, - body_prefix = body_prefix, - package_info_package = config.package_info_package(), - file_name = file_name, - ext = extension, - ) - } else { - format!( - indoc::indoc! {" - {body_prefix} - - -- import: {package_info_package}/code as pi - - -- pi.code-page: {file_name} - lang: {ext} - - {content} - - "}, - body_prefix = body_prefix, - package_info_package = config.package_info_package(), - file_name = file_name, - ext = extension, - content = content, - ) - } - }) -} - -fn package_info_markdown( - config: &fastn_core::Config, - file_name: &str, - content: &str, -) -> fastn_core::Result { - let path = config.root.join("fastn").join("markdown.ftd"); - Ok(if path.is_file() { - std::fs::read_to_string(path)? - } else if !config - .ftd_edition - .eq(&fastn_core::config::FTDEdition::FTD2021) - { - if content.trim().is_empty() { - content.to_string() - } else { - format!( - indoc::indoc! {" - -- ftd.text: - - {content} - "}, - content = content, - ) - } - } else { - let body_prefix = match config.package.generate_prefix_string(false) { - Some(bp) => bp, - None => String::new(), - }; - if content.trim().is_empty() { - format!( - indoc::indoc! {" - {body_prefix} - - -- import: {package_info_package}/markdown as pi - - -- pi.markdown-page: {file_name} - - "}, - body_prefix = body_prefix, - package_info_package = config.package_info_package(), - file_name = file_name, - ) - } else { - format!( - indoc::indoc! {" - {body_prefix} - - -- import: {package_info_package}/markdown as pi - - -- pi.markdown-page: {file_name} - - {content} - - "}, - body_prefix = body_prefix, - package_info_package = config.package_info_package(), - content = content, - file_name = file_name, - ) - } - }) -} - #[allow(dead_code)] fn original_package_status(config: &fastn_core::Config) -> fastn_core::Result { let path = config diff --git a/fastn-core/src/library2022/mod.rs b/fastn-core/src/library2022/mod.rs index 17ba88e4a2..3d9182cff1 100644 --- a/fastn-core/src/library2022/mod.rs +++ b/fastn-core/src/library2022/mod.rs @@ -30,7 +30,7 @@ impl Library2022 { &mut self, name: &str, current_processing_module: &str, - ) -> ftd::p1::Result<(String, usize)> { + ) -> ftd::p1::Result<(String, String, usize)> { match self.get(name, current_processing_module).await { Some(v) => Ok(v), None => ftd::p1::utils::parse_error(format!("library not found 1: {}", name), "", 0), @@ -67,9 +67,13 @@ impl Library2022 { &mut self, name: &str, current_processing_module: &str, - ) -> Option<(String, usize)> { + ) -> Option<(String, String, usize)> { if name == "fastn" { - return Some((fastn_core::library::fastn_dot_ftd::get2022(self).await, 0)); + return Some(( + fastn_core::library::fastn_dot_ftd::get2022(self).await, + "$fastn$/fastn.ftd".to_string(), + 0, + )); } return get_for_package( @@ -83,33 +87,31 @@ impl Library2022 { name: &str, lib: &mut fastn_core::Library2022, current_processing_module: &str, - ) -> Option<(String, usize)> { + ) -> Option<(String, String, usize)> { let package = lib.get_current_package(current_processing_module).ok()?; if name.starts_with(package.name.as_str()) { - if let Some(r) = get_data_from_package(name, &package, lib).await { - return Some(r); + if let Some((content, size)) = get_data_from_package(name, &package, lib).await { + return Some((content, name.to_string(), size)); } } // Self package referencing if package.name.ends_with(name.trim_end_matches('/')) { let package_index = format!("{}/", package.name.as_str()); - if let Some(r) = get_data_from_package(package_index.as_str(), &package, lib).await + if let Some((content, size)) = + get_data_from_package(package_index.as_str(), &package, lib).await { - return Some(r); + return Some((content, format!("{package_index}index.ftd"), size)); } } for (alias, package) in package.aliases() { lib.push_package_under_process(name, package).await.ok()?; if name.starts_with(alias) { - if let Some(r) = get_data_from_package( - name.replacen(alias, &package.name, 1).as_str(), - package, - lib, - ) - .await + let name = name.replacen(alias, &package.name, 1); + if let Some((content, size)) = + get_data_from_package(name.as_str(), package, lib).await { - return Some(r); + return Some((content, name.to_string(), size)); } } } @@ -121,21 +123,20 @@ impl Library2022 { let name = name.replacen(package.name.as_str(), translation_of.name.as_str(), 1); if name.starts_with(translation_of.name.as_str()) { - if let Some(r) = get_data_from_package(name.as_str(), &translation_of, lib).await { - return Some(r); + if let Some((content, size)) = + get_data_from_package(name.as_str(), &translation_of, lib).await + { + return Some((content, name.to_string(), size)); } } for (alias, package) in translation_of.aliases() { if name.starts_with(alias) { - if let Some(r) = get_data_from_package( - name.replacen(alias, &package.name, 1).as_str(), - package, - lib, - ) - .await + let name = name.replacen(alias, &package.name, 1); + if let Some((content, size)) = + get_data_from_package(name.as_str(), package, lib).await { - return Some(r); + return Some((content, name.to_string(), size)); } } } diff --git a/fastn-core/src/package/package_doc.rs b/fastn-core/src/package/package_doc.rs index 956b3be95e..f76aa0372d 100644 --- a/fastn-core/src/package/package_doc.rs +++ b/fastn-core/src/package/package_doc.rs @@ -345,6 +345,10 @@ impl FTDResult { } } } + + pub fn checksum(&self) -> String { + fastn_core::utils::generate_hash(self.html()) + } } impl From for fastn_core::http::Response { @@ -380,7 +384,6 @@ pub(crate) async fn read_ftd( } } -#[allow(clippy::await_holding_refcell_ref)] #[tracing::instrument(name = "read_ftd_2022", skip_all)] pub(crate) async fn read_ftd_2022( config: &mut fastn_core::Config, @@ -390,7 +393,7 @@ pub(crate) async fn read_ftd_2022( test: bool, ) -> fastn_core::Result { let lib_config = config.clone(); - let mut all_packages = config.all_packages.borrow_mut(); + let all_packages = config.all_packages.borrow(); let current_package = all_packages .get(main.package_name.as_str()) .unwrap_or(&config.package); @@ -409,6 +412,7 @@ pub(crate) async fn read_ftd_2022( current_package.get_prefixed_body(main.content.as_str(), main.id.as_str(), true); // Fix aliased imports to full path (if any) doc_content = current_package.fix_imports_in_body(doc_content.as_str(), main.id.as_str())?; + drop(all_packages); let line_number = doc_content.split('\n').count() - main.content.split('\n').count(); let main_ftd_doc = match fastn_core::doc::interpret_helper( @@ -429,6 +433,7 @@ pub(crate) async fn read_ftd_2022( }); } }; + config.dependencies_during_render = lib.config.dependencies_during_render; if let Some((url, code)) = main_ftd_doc.get_redirect()? { return Ok(FTDResult::Redirect { url, code }); } @@ -436,9 +441,6 @@ pub(crate) async fn read_ftd_2022( let node = ftd::node::NodeData::from_rt(executor); let html_ui = ftd::html::HtmlUI::from_node_data(node, "main", test)?; - all_packages.extend(lib.config.all_packages.into_inner()); - drop(all_packages); - config .downloaded_assets .extend(lib.config.downloaded_assets); @@ -504,7 +506,7 @@ pub(crate) async fn read_ftd_2023( }); } }; - + config.dependencies_during_render = lib.config.dependencies_during_render; if let Some((url, code)) = main_ftd_doc.get_redirect()? { return Ok(FTDResult::Redirect { url, code }); } @@ -551,49 +553,12 @@ pub(crate) async fn process_ftd( config: &mut fastn_core::Config, main: &fastn_core::Document, base_url: &str, - no_static: bool, + build_static_files: bool, test: bool, + file_path: &str, ) -> fastn_core::Result { - if main.id.eq("FASTN.ftd") { - tokio::fs::copy( - config.root.join(main.id.as_str()), - config.root.join(".build").join(main.id.as_str()), - ) - .await?; - } - - let main = { - let mut main = main.to_owned(); - if main.id.eq("FASTN.ftd") { - main.id = "-.ftd".to_string(); - let path = config.root.join("fastn").join("info.ftd"); - main.content = if path.is_file() { - std::fs::read_to_string(path)? - } else { - format!( - "-- import: {}/package-info as pi\n\n-- pi.package-info-page:", - config.package_info_package() - ) - } - } - main - }; - - let file_rel_path = if main.id.eq("404.ftd") { - "404.html".to_string() - } else if main.id.contains("index.ftd") { - main.id.replace("index.ftd", "index.html") - } else { - main.id.replace(".ftd", "/index.html") - }; - - let response = read_ftd(config, &main, base_url, !no_static, test).await?; - fastn_core::utils::write( - &config.build_dir(), - file_rel_path.as_str(), - &response.html(), - ) - .await?; + let response = read_ftd(config, main, base_url, build_static_files, test).await?; + fastn_core::utils::overwrite(&config.build_dir(), file_path, &response.html()).await?; Ok(response) } diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index e225dab3ff..2a237d35dc 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -29,6 +29,76 @@ macro_rules! warning { }}; } +fn id_to_cache_key(id: &str) -> String { + // TODO: use MAIN_SEPARATOR here + id.replace(['/', '\\'], "_") +} + +pub fn get_ftd_hash(path: &str) -> fastn_core::Result { + let path = fastn_core::utils::replace_last_n(path, 1, "/", ""); + Ok(fastn_core::utils::generate_hash( + std::fs::read(format!("{path}.ftd")) + .or_else(|_| std::fs::read(format!("{path}/index.ftd")))?, + )) +} + +pub fn get_cache_file(id: &str) -> Option { + let cache_dir = dirs::cache_dir()?; + let base_path = cache_dir.join("fastn.com"); + + if !base_path.exists() { + if let Err(err) = std::fs::create_dir_all(&base_path) { + eprintln!("Failed to create cache directory: {}", err); + return None; + } + } + + Some( + base_path + .join(id_to_cache_key( + &std::env::current_dir() + .expect("cant read current dir") + .to_string_lossy(), + )) + .join(id_to_cache_key(id)), + ) +} + +pub fn get_cached(id: &str) -> Option +where + T: serde::de::DeserializeOwned, +{ + let cache_file = get_cache_file(id)?; + serde_json::from_str( + &std::fs::read_to_string(cache_file) + .map_err(|e| { + tracing::debug!("file read error: {}", e.to_string()); + e + }) + .ok()?, + ) + .map_err(|e| { + tracing::debug!("not valid json: {}", e.to_string()); + e + }) + .ok() +} + +pub fn cache_it(id: &str, d: T) -> ftd::interpreter::Result +where + T: serde::ser::Serialize, +{ + let cache_file = get_cache_file(id) + .ok_or_else(|| ftd::interpreter::Error::OtherError("cache dir not found".to_string()))?; + std::fs::create_dir_all(cache_file.parent().unwrap()).map_err(|e| { + ftd::interpreter::Error::OtherError(format!("failed to create cache dir: {}", e)) + })?; + std::fs::write(cache_file, serde_json::to_string(&d)?).map_err(|e| { + ftd::interpreter::Error::OtherError(format!("failed to write cache file: {}", e)) + })?; + Ok(d) +} + pub fn redirect_page_html(url: &str) -> String { include_str!("../redirect.html").replace("__REDIRECT_URL__", url) } @@ -49,6 +119,41 @@ pub fn print_end(msg: &str, start: std::time::Instant) { } } +/// replace_last_n("a.b.c.d.e.f", 2, ".", "/") => "a.b.c.d/e/f" +pub fn replace_last_n(s: &str, n: usize, pattern: &str, replacement: &str) -> String { + use itertools::Itertools; + + s.rsplitn(n + 1, pattern) + .collect_vec() + .into_iter() + .rev() + .join(replacement) +} + +#[cfg(test)] +mod test { + #[test] + fn replace_last_n() { + assert_eq!( + super::replace_last_n("a.b.c.d.e.f", 2, ".", "/"), + "a.b.c.d/e/f" + ); + assert_eq!( + super::replace_last_n("a.b.c.d.e.", 2, ".", "/"), + "a.b.c.d/e/" + ); + assert_eq!(super::replace_last_n("d-e.f", 2, ".", "/"), "d-e/f"); + assert_eq!( + super::replace_last_n("a.ftd/b.ftd", 1, ".ftd", "/index.html"), + "a.ftd/b/index.html" + ); + assert_eq!( + super::replace_last_n("index.ftd/b/index.ftd", 1, "index.ftd", "index.html"), + "index.ftd/b/index.html" + ); + } +} + pub fn print_error(msg: &str, start: std::time::Instant) { use colored::Colorize; @@ -255,15 +360,6 @@ pub(crate) async fn get_number_of_documents( Ok(no_of_docs) } -pub(crate) fn get_extension(file_name: &str) -> fastn_core::Result { - if let Some((_, ext)) = file_name.rsplit_once('.') { - return Ok(ext.to_string()); - } - Err(fastn_core::Error::UsageError { - message: format!("extension not found, `{}`", file_name), - }) -} - pub(crate) async fn get_current_document_last_modified_on( config: &fastn_core::Config, document_id: &str, @@ -718,6 +814,14 @@ pub(crate) async fn write( update1(root, file_path, data).await } +pub(crate) async fn overwrite( + root: &camino::Utf8PathBuf, + file_path: &str, + data: &[u8], +) -> fastn_core::Result<()> { + update1(root, file_path, data).await +} + // TODO: remove this function use update instead pub(crate) async fn update1( root: &camino::Utf8PathBuf, @@ -866,7 +970,7 @@ pub fn query(uri: &str) -> fastn_core::Result> { .collect_vec(), ) } -pub fn generate_hash(content: &str) -> String { +pub fn generate_hash(content: impl AsRef<[u8]>) -> String { use sha2::digest::FixedOutput; use sha2::Digest; let mut hasher = sha2::Sha256::new();