From 8b8fe18c7faf6a4d398ed09a6a37afd5750bfd82 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Fri, 4 Aug 2023 10:17:59 +0530 Subject: [PATCH 01/63] no_static -> build_static_files --- fastn-core/src/commands/build.rs | 84 ++++++++++++++++++++------- fastn-core/src/package/package_doc.rs | 4 +- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 83330a5541..b99fa77dde 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -7,14 +7,23 @@ pub async fn build( test: bool, ) -> fastn_core::Result<()> { tokio::fs::create_dir_all(config.build_dir()).await?; - let documents = get_documents_for_current_package(config).await?; // Default css and js default_build_files(config.root.join(".build"), &config.ftd_edition).await?; - if let Some(id) = only_id { - handle_only_id(id, config, base_url, ignore_failed, test, documents).await?; - return Ok(()); + { + let documents = get_documents_for_current_package(config).await?; + + match only_id { + Some(id) => { + return handle_only_id(id, config, base_url, ignore_failed, test, documents).await + } + None => { + for document in documents.values() { + handle_file(document, config, base_url, ignore_failed, test, true).await?; + } + } + } } // All redirect html files under .build @@ -43,10 +52,6 @@ pub async fn build( } } - for document in documents.values() { - handle_file(document, config, base_url, ignore_failed, test, false).await?; - } - config.download_fonts().await } @@ -61,7 +66,7 @@ async fn handle_only_id( ) -> fastn_core::Result<()> { for doc in documents.values() { if doc.get_id().eq(id) || doc.get_id_with_package().eq(id) { - return handle_file(doc, config, base_url, ignore_failed, test, true).await; + return handle_file(doc, config, base_url, ignore_failed, test, false).await; } } @@ -78,12 +83,21 @@ async fn handle_file( base_url: &str, ignore_failed: bool, test: bool, - no_static: bool, + build_static_files: bool, ) -> fastn_core::Result<()> { let start = std::time::Instant::now(); print!("Processing {} ... ", document.get_id_with_package()); - if let Ok(()) = handle_file_(document, config, base_url, ignore_failed, test, no_static).await { + if let Ok(()) = handle_file_( + document, + config, + base_url, + ignore_failed, + test, + build_static_files, + ) + .await + { fastn_core::utils::print_end( format!( "Processed {}/{}", @@ -105,7 +119,7 @@ async fn handle_file_( base_url: &str, ignore_failed: bool, test: bool, - no_static: bool, + build_static_files: bool, ) -> fastn_core::Result<()> { config.current_document = Some(document.get_id().to_string()); @@ -135,7 +149,11 @@ async fn handle_file_( } } let resp = fastn_core::package::package_doc::process_ftd( - config, doc, base_url, no_static, test, + config, + doc, + base_url, + build_static_files, + test, ) .await; match (resp, ignore_failed) { @@ -159,7 +177,7 @@ async fn handle_file_( print!("Skipped "); return Ok(()); } - let resp = process_markdown(config, doc, base_url, no_static, test).await; + let resp = process_markdown(config, doc, base_url, build_static_files, test).await; match (resp, ignore_failed) { (Ok(r), _) => r, (_, true) => { @@ -177,7 +195,8 @@ async fn handle_file_( .ftd_edition .eq(&fastn_core::config::FTDEdition::FTD2021) { - let resp = process_image(config, main_doc, base_url, no_static, test).await; + let resp = + process_image(config, main_doc, base_url, build_static_files, test).await; match (resp, ignore_failed) { (Ok(r), _) => r, (_, true) => { @@ -206,7 +225,7 @@ async fn handle_file_( .ftd_edition .eq(&fastn_core::config::FTDEdition::FTD2021) { - let resp = process_code(config, doc, base_url, no_static, test).await; + let resp = process_code(config, doc, base_url, build_static_files, test).await; match (resp, ignore_failed) { (Ok(r), _) => r, (_, true) => { @@ -386,12 +405,19 @@ async fn process_image( config: &mut fastn_core::Config, main: &fastn_core::Static, base_url: &str, - no_static: bool, + build_static_files: bool, test: bool, ) -> fastn_core::Result<()> { let main = convert_to_ftd(config, main)?; - fastn_core::package::package_doc::process_ftd(config, &main, base_url, no_static, test).await?; + fastn_core::package::package_doc::process_ftd( + config, + &main, + base_url, + build_static_files, + test, + ) + .await?; return Ok(()); fn convert_to_ftd( @@ -415,7 +441,7 @@ async fn process_code( config: &mut fastn_core::Config, main: &fastn_core::Document, base_url: &str, - no_static: bool, + build_static_files: bool, test: bool, ) -> fastn_core::Result<()> { let main = if let Some(main) = convert_to_ftd(config, main)? { @@ -424,7 +450,14 @@ async fn process_code( return Ok(()); }; - fastn_core::package::package_doc::process_ftd(config, &main, base_url, no_static, test).await?; + fastn_core::package::package_doc::process_ftd( + config, + &main, + base_url, + build_static_files, + test, + ) + .await?; return Ok(()); fn convert_to_ftd( @@ -455,7 +488,7 @@ async fn process_markdown( config: &mut fastn_core::Config, main: &fastn_core::Document, base_url: &str, - no_static: bool, + build_static_files: bool, test: bool, ) -> fastn_core::Result<()> { let main = if let Some(main) = convert_md_to_ftd(config, main)? { @@ -463,7 +496,14 @@ async fn process_markdown( } else { return Ok(()); }; - fastn_core::package::package_doc::process_ftd(config, &main, base_url, no_static, test).await?; + fastn_core::package::package_doc::process_ftd( + config, + &main, + base_url, + build_static_files, + test, + ) + .await?; return Ok(()); fn convert_md_to_ftd( diff --git a/fastn-core/src/package/package_doc.rs b/fastn-core/src/package/package_doc.rs index 0241e3b9ae..725855d607 100644 --- a/fastn-core/src/package/package_doc.rs +++ b/fastn-core/src/package/package_doc.rs @@ -541,7 +541,7 @@ pub(crate) async fn process_ftd( config: &mut fastn_core::Config, main: &fastn_core::Document, base_url: &str, - no_static: bool, + build_static_files: bool, test: bool, ) -> fastn_core::Result { if main.id.eq("FASTN.ftd") { @@ -577,7 +577,7 @@ pub(crate) async fn process_ftd( main.id.replace(".ftd", "/index.html") }; - let response = read_ftd(config, &main, base_url, !no_static, test).await?; + let response = read_ftd(config, &main, base_url, build_static_files, test).await?; fastn_core::utils::write( &config.build_dir(), file_rel_path.as_str(), From bd7369a9ee66d78bf16709f25ae429db43914fe0 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Fri, 4 Aug 2023 17:00:15 +0530 Subject: [PATCH 02/63] refactored out incremental_build() --- fastn-core/src/commands/build.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index b99fa77dde..ddaa75b2a5 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -19,9 +19,7 @@ pub async fn build( return handle_only_id(id, config, base_url, ignore_failed, test, documents).await } None => { - for document in documents.values() { - handle_file(document, config, base_url, ignore_failed, test, true).await?; - } + incremental_build(config, documents, base_url, ignore_failed, test).await?; } } } @@ -55,6 +53,21 @@ pub async fn build( config.download_fonts().await } +#[tracing::instrument(skip(config, documents))] +async fn incremental_build( + config: &mut fastn_core::Config, + documents: std::collections::BTreeMap, + base_url: &str, + ignore_failed: bool, + test: bool, +) -> fastn_core::Result<()> { + for document in documents.values() { + handle_file(document, config, base_url, ignore_failed, test, true).await?; + } + + Ok(()) +} + #[tracing::instrument(skip(config, documents))] async fn handle_only_id( id: &str, From 869a855c230c3265cbc2612156b7a4f2db62aa19 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Fri, 4 Aug 2023 17:01:15 +0530 Subject: [PATCH 03/63] moved the caching utilities to fastn_core::utils --- fastn-core/src/doc.rs | 47 ++--------------------------------------- fastn-core/src/utils.rs | 43 +++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/fastn-core/src/doc.rs b/fastn-core/src/doc.rs index 5f67c38929..8a69e08647 100644 --- a/fastn-core/src/doc.rs +++ b/fastn-core/src/doc.rs @@ -11,7 +11,7 @@ fn cached_parse( let hash = fastn_core::utils::generate_hash(source); - if let Some(c) = get_cached::(id) { + if let Some(c) = fastn_core::utils::get_cached::(id) { if c.hash == hash { tracing::debug!("cache hit"); return Ok(c.doc); @@ -22,50 +22,7 @@ fn cached_parse( } let doc = ftd::interpreter::ParsedDocument::parse_with_line_number(id, source, line_number)?; - cache_it(id, C { doc, hash }).map(|v| v.doc) -} - -fn id_to_cache_key(id: &str) -> String { - id.replace('/', "_") -} - -fn get_cached(id: &str) -> Option -where - T: serde::de::DeserializeOwned, -{ - let cache_file = dirs::cache_dir()? - .join("fastn.com/ast-cache/") - .join(id_to_cache_key(id)); - serde_json::from_str( - &std::fs::read_to_string(cache_file) - .map_err(|e| { - tracing::debug!("file read error: {}", e.to_string()); - e - }) - .ok()?, - ) - .map_err(|e| { - tracing::debug!("not valid json: {}", e.to_string()); - e - }) - .ok() -} - -fn cache_it(id: &str, d: T) -> ftd::interpreter::Result -where - T: serde::ser::Serialize, -{ - let cache_file = dirs::cache_dir() - .ok_or_else(|| ftd::interpreter::Error::OtherError("cache dir not found".to_string()))? - .join("fastn.com/ast-cache/") - .join(id_to_cache_key(id)); - std::fs::create_dir_all(cache_file.parent().unwrap()).map_err(|e| { - ftd::interpreter::Error::OtherError(format!("failed to create cache dir: {}", e)) - })?; - std::fs::write(cache_file, serde_json::to_string(&d)?).map_err(|e| { - ftd::interpreter::Error::OtherError(format!("failed to write cache file: {}", e)) - })?; - Ok(d) + fastn_core::utils::cache_it(id, C { doc, hash }).map(|v| v.doc) } #[tracing::instrument(skip_all)] diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index 5c8d4af387..32c6e8769b 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -29,6 +29,49 @@ macro_rules! warning { }}; } +fn id_to_cache_key(id: &str) -> String { + id.replace('/', "_") +} + +pub fn get_cached(id: &str) -> Option +where + T: serde::de::DeserializeOwned, +{ + let cache_file = dirs::cache_dir()? + .join("fastn.com/ast-cache/") + .join(id_to_cache_key(id)); + serde_json::from_str( + &std::fs::read_to_string(cache_file) + .map_err(|e| { + tracing::debug!("file read error: {}", e.to_string()); + e + }) + .ok()?, + ) + .map_err(|e| { + tracing::debug!("not valid json: {}", e.to_string()); + e + }) + .ok() +} + +pub fn cache_it(id: &str, d: T) -> ftd::interpreter::Result +where + T: serde::ser::Serialize, +{ + let cache_file = dirs::cache_dir() + .ok_or_else(|| ftd::interpreter::Error::OtherError("cache dir not found".to_string()))? + .join("fastn.com/ast-cache/") + .join(id_to_cache_key(id)); + std::fs::create_dir_all(cache_file.parent().unwrap()).map_err(|e| { + ftd::interpreter::Error::OtherError(format!("failed to create cache dir: {}", e)) + })?; + std::fs::write(cache_file, serde_json::to_string(&d)?).map_err(|e| { + ftd::interpreter::Error::OtherError(format!("failed to write cache file: {}", e)) + })?; + Ok(d) +} + pub fn redirect_page_html(url: &str) -> String { include_str!("../redirect.html").replace("__REDIRECT_URL__", url) } From c11b0dff51a4dac55817fc66feb44c81ce518538 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Fri, 4 Aug 2023 20:44:24 +0530 Subject: [PATCH 04/63] cache stubs --- fastn-core/src/commands/build.rs | 58 ++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index ddaa75b2a5..e4368e6638 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -53,6 +53,28 @@ pub async fn build( config.download_fonts().await } +mod cache { + #[derive(serde::Serialize, serde::Deserialize)] + pub(crate) struct Cache { + // fastn_version: String, // TODO: Add this + pub(crate) documents: Vec, + pub(crate) assets: std::collections::BTreeMap, + } + + #[derive(serde::Serialize, serde::Deserialize)] + pub(crate) struct File { + pub(crate) path: String, + pub(crate) checksum: String, + } + + #[derive(serde::Serialize, serde::Deserialize)] + pub(crate) struct Document { + pub(crate) file: File, + pub(crate) html_checksum: String, + pub(crate) dependencies: Vec, + } +} + #[tracing::instrument(skip(config, documents))] async fn incremental_build( config: &mut fastn_core::Config, @@ -61,8 +83,30 @@ async fn incremental_build( ignore_failed: bool, test: bool, ) -> fastn_core::Result<()> { - for document in documents.values() { - handle_file(document, config, base_url, ignore_failed, test, true).await?; + // https://fastn.com/rfc/incremental-build/ + + if let Some(_c) = fastn_core::utils::get_cached::("build_cache") { + tracing::debug!("cached hash mismatch"); + } else { + tracing::debug!("cached miss"); + let mut c = cache::Cache { + // fastn_version: fastn_core::utils::get_fastn_version(), + documents: vec![], + assets: std::collections::BTreeMap::new(), + }; + for document in documents.values() { + handle_file( + document, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; + } + fastn_core::utils::cache_it("build_cache", &c)?; } Ok(()) @@ -79,7 +123,7 @@ async fn handle_only_id( ) -> fastn_core::Result<()> { for doc in documents.values() { if doc.get_id().eq(id) || doc.get_id_with_package().eq(id) { - return handle_file(doc, config, base_url, ignore_failed, test, false).await; + return handle_file(doc, config, base_url, ignore_failed, test, false, None).await; } } @@ -97,6 +141,7 @@ async fn handle_file( ignore_failed: bool, test: bool, build_static_files: bool, + cache: Option<&mut cache::Cache>, ) -> fastn_core::Result<()> { let start = std::time::Instant::now(); print!("Processing {} ... ", document.get_id_with_package()); @@ -108,6 +153,7 @@ async fn handle_file( ignore_failed, test, build_static_files, + cache, ) .await { @@ -125,7 +171,7 @@ async fn handle_file( Ok(()) } -#[tracing::instrument(skip(document, config))] +#[tracing::instrument(skip(document, config, _cache))] async fn handle_file_( document: &fastn_core::File, config: &mut fastn_core::Config, @@ -133,9 +179,9 @@ async fn handle_file_( ignore_failed: bool, test: bool, build_static_files: bool, + _cache: Option<&mut cache::Cache>, ) -> fastn_core::Result<()> { config.current_document = Some(document.get_id().to_string()); - match document { fastn_core::File::Ftd(doc) => { if !config @@ -227,7 +273,7 @@ async fn handle_file_( &fastn_core::Static { package_name: config.package.name.to_string(), id: doc.id.to_string(), - content: vec![], + content: doc.content.clone().into_bytes(), base_path: camino::Utf8PathBuf::from(doc.parent_path.as_str()), }, &config.root, From 3b1e9f83e890bfabae0b14e60d5c9d34b4c78b0f Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Fri, 4 Aug 2023 20:47:35 +0530 Subject: [PATCH 05/63] better cache plumbing --- fastn-core/src/commands/build.rs | 47 ++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index e4368e6638..2662270bc2 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -85,29 +85,34 @@ async fn incremental_build( ) -> fastn_core::Result<()> { // https://fastn.com/rfc/incremental-build/ - if let Some(_c) = fastn_core::utils::get_cached::("build_cache") { - tracing::debug!("cached hash mismatch"); - } else { - tracing::debug!("cached miss"); - let mut c = cache::Cache { - // fastn_version: fastn_core::utils::get_fastn_version(), - documents: vec![], - assets: std::collections::BTreeMap::new(), - }; - for document in documents.values() { - handle_file( - document, - config, - base_url, - ignore_failed, - test, - true, - Some(&mut c), - ) - .await?; + let mut c = match fastn_core::utils::get_cached::("build_cache") { + Some(v) => { + tracing::debug!("cached hit"); + v + } + None => { + tracing::debug!("cached miss"); + cache::Cache { + // fastn_version: fastn_core::utils::get_fastn_version(), + documents: vec![], + assets: std::collections::BTreeMap::new(), + } } - fastn_core::utils::cache_it("build_cache", &c)?; + }; + + for document in documents.values() { + handle_file( + document, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; } + fastn_core::utils::cache_it("build_cache", &c)?; Ok(()) } From 3d6a0c990e866ea37a3ccb4bc32f682882889361 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Fri, 4 Aug 2023 20:55:05 +0530 Subject: [PATCH 06/63] some refactor --- fastn-core/src/commands/build.rs | 45 +++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 2662270bc2..579c5141f8 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -54,6 +54,24 @@ pub async fn build( } mod cache { + const FILE_NAME: &str = "fastn.cache"; + + pub(crate) fn get() -> Cache { + match fastn_core::utils::get_cached(FILE_NAME) { + Some(v) => { + tracing::debug!("cached hit"); + v + } + None => { + tracing::debug!("cached miss"); + Cache { + documents: vec![], + assets: std::collections::BTreeMap::new(), + } + } + } + } + #[derive(serde::Serialize, serde::Deserialize)] pub(crate) struct Cache { // fastn_version: String, // TODO: Add this @@ -61,6 +79,13 @@ mod cache { pub(crate) assets: std::collections::BTreeMap, } + impl Cache { + pub(crate) fn cache_it(&self) -> fastn_core::Result<()> { + fastn_core::utils::cache_it(FILE_NAME, self)?; + Ok(()) + } + } + #[derive(serde::Serialize, serde::Deserialize)] pub(crate) struct File { pub(crate) path: String, @@ -85,20 +110,7 @@ async fn incremental_build( ) -> fastn_core::Result<()> { // https://fastn.com/rfc/incremental-build/ - let mut c = match fastn_core::utils::get_cached::("build_cache") { - Some(v) => { - tracing::debug!("cached hit"); - v - } - None => { - tracing::debug!("cached miss"); - cache::Cache { - // fastn_version: fastn_core::utils::get_fastn_version(), - documents: vec![], - assets: std::collections::BTreeMap::new(), - } - } - }; + let mut c = cache::get(); for document in documents.values() { handle_file( @@ -112,7 +124,10 @@ async fn incremental_build( ) .await?; } - fastn_core::utils::cache_it("build_cache", &c)?; + + // TODO: Handle deleted files (files present in cache/.build but not in documents) + + c.cache_it()?; Ok(()) } From fa4346bfb797e6345f10e654f2895e6558bb2ec7 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Sat, 5 Aug 2023 08:45:17 +0530 Subject: [PATCH 07/63] better replacement when constructing file names --- fastn-core/src/package/package_doc.rs | 41 +++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/fastn-core/src/package/package_doc.rs b/fastn-core/src/package/package_doc.rs index 725855d607..489571b460 100644 --- a/fastn-core/src/package/package_doc.rs +++ b/fastn-core/src/package/package_doc.rs @@ -571,10 +571,10 @@ pub(crate) async fn process_ftd( let file_rel_path = if main.id.eq("404.ftd") { "404.html".to_string() - } else if main.id.contains("index.ftd") { - main.id.replace("index.ftd", "index.html") + } else if main.id.ends_with("index.ftd") { + replace_last_n(main.id.as_str(), 1, "index.ftd", "index.html") } else { - main.id.replace(".ftd", "/index.html") + replace_last_n(main.id.as_str(), 1, ".ftd", "/index.html") }; let response = read_ftd(config, &main, base_url, build_static_files, test).await?; @@ -587,3 +587,38 @@ pub(crate) async fn process_ftd( Ok(response) } + +/// replace_last_n("a.b.c.d.e.f", 2, ".", "/") => "a.b.c.d/e/f" +fn replace_last_n(s: &str, n: usize, pattern: &str, replacement: &str) -> String { + use itertools::Itertools; + + s.rsplitn(n + 1, pattern) + .collect_vec() + .into_iter() + .rev() + .join(replacement) +} + +#[cfg(test)] +mod test { + #[test] + fn replace_last_n() { + assert_eq!( + super::replace_last_n("a.b.c.d.e.f", 2, ".", "/"), + "a.b.c.d/e/f" + ); + assert_eq!( + super::replace_last_n("a.b.c.d.e.", 2, ".", "/"), + "a.b.c.d/e/" + ); + assert_eq!(super::replace_last_n("d-e.f", 2, ".", "/"), "d-e/f"); + assert_eq!( + super::replace_last_n("a.ftd/b.ftd", 1, ".ftd", "/index.html"), + "a.ftd/b/index.html" + ); + assert_eq!( + super::replace_last_n("index.ftd/b/index.ftd", 1, "index.ftd", "index.html"), + "index.ftd/b/index.html" + ); + } +} From 57a61973c0b52e81bc0fd69aa121c508a25b7318 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Sat, 5 Aug 2023 08:58:25 +0530 Subject: [PATCH 08/63] utility should live in utils.rs --- fastn-core/src/package/package_doc.rs | 39 ++------------------------- fastn-core/src/utils.rs | 35 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/fastn-core/src/package/package_doc.rs b/fastn-core/src/package/package_doc.rs index 489571b460..929574252c 100644 --- a/fastn-core/src/package/package_doc.rs +++ b/fastn-core/src/package/package_doc.rs @@ -572,9 +572,9 @@ pub(crate) async fn process_ftd( let file_rel_path = if main.id.eq("404.ftd") { "404.html".to_string() } else if main.id.ends_with("index.ftd") { - replace_last_n(main.id.as_str(), 1, "index.ftd", "index.html") + fastn_core::utils::replace_last_n(main.id.as_str(), 1, "index.ftd", "index.html") } else { - replace_last_n(main.id.as_str(), 1, ".ftd", "/index.html") + fastn_core::utils::replace_last_n(main.id.as_str(), 1, ".ftd", "/index.html") }; let response = read_ftd(config, &main, base_url, build_static_files, test).await?; @@ -587,38 +587,3 @@ pub(crate) async fn process_ftd( Ok(response) } - -/// replace_last_n("a.b.c.d.e.f", 2, ".", "/") => "a.b.c.d/e/f" -fn replace_last_n(s: &str, n: usize, pattern: &str, replacement: &str) -> String { - use itertools::Itertools; - - s.rsplitn(n + 1, pattern) - .collect_vec() - .into_iter() - .rev() - .join(replacement) -} - -#[cfg(test)] -mod test { - #[test] - fn replace_last_n() { - assert_eq!( - super::replace_last_n("a.b.c.d.e.f", 2, ".", "/"), - "a.b.c.d/e/f" - ); - assert_eq!( - super::replace_last_n("a.b.c.d.e.", 2, ".", "/"), - "a.b.c.d/e/" - ); - assert_eq!(super::replace_last_n("d-e.f", 2, ".", "/"), "d-e/f"); - assert_eq!( - super::replace_last_n("a.ftd/b.ftd", 1, ".ftd", "/index.html"), - "a.ftd/b/index.html" - ); - assert_eq!( - super::replace_last_n("index.ftd/b/index.ftd", 1, "index.ftd", "index.html"), - "index.ftd/b/index.html" - ); - } -} diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index 32c6e8769b..94ceb81f04 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -92,6 +92,41 @@ pub fn print_end(msg: &str, start: std::time::Instant) { } } +/// replace_last_n("a.b.c.d.e.f", 2, ".", "/") => "a.b.c.d/e/f" +pub fn replace_last_n(s: &str, n: usize, pattern: &str, replacement: &str) -> String { + use itertools::Itertools; + + s.rsplitn(n + 1, pattern) + .collect_vec() + .into_iter() + .rev() + .join(replacement) +} + +#[cfg(test)] +mod test { + #[test] + fn replace_last_n() { + assert_eq!( + super::replace_last_n("a.b.c.d.e.f", 2, ".", "/"), + "a.b.c.d/e/f" + ); + assert_eq!( + super::replace_last_n("a.b.c.d.e.", 2, ".", "/"), + "a.b.c.d/e/" + ); + assert_eq!(super::replace_last_n("d-e.f", 2, ".", "/"), "d-e/f"); + assert_eq!( + super::replace_last_n("a.ftd/b.ftd", 1, ".ftd", "/index.html"), + "a.ftd/b/index.html" + ); + assert_eq!( + super::replace_last_n("index.ftd/b/index.ftd", 1, "index.ftd", "index.html"), + "index.ftd/b/index.html" + ); + } +} + pub fn value_to_colored_string(value: &serde_json::Value, indent_level: u32) -> String { use colored::Colorize; From 0f4c755055d86f943169745ba47b6b30a8290cad Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Sat, 5 Aug 2023 09:20:25 +0530 Subject: [PATCH 09/63] removed edition 2021 support from fastn build --- fastn-core/src/commands/build.rs | 89 +++++++------------------------- 1 file changed, 18 insertions(+), 71 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 579c5141f8..f55ad1e15d 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -204,29 +204,25 @@ async fn handle_file_( config.current_document = Some(document.get_id().to_string()); match document { fastn_core::File::Ftd(doc) => { - if !config - .ftd_edition - .eq(&fastn_core::config::FTDEdition::FTD2021) - { - // Ignore redirect paths - if let Some(r) = config.package.redirects.as_ref() { - if fastn_core::package::redirects::find_redirect(r, doc.id.as_str()).is_some() { - println!("Ignored by redirect {}", doc.id.as_str()); - return Ok(()); - } + // Ignore redirect paths + if let Some(r) = config.package.redirects.as_ref() { + if fastn_core::package::redirects::find_redirect(r, doc.id.as_str()).is_some() { + println!("Ignored by redirect {}", doc.id.as_str()); + return Ok(()); } + } - fastn_core::utils::copy( - config.root.join(doc.id.as_str()), - config.root.join(".build").join(doc.id.as_str()), - ) - .await - .ok(); + fastn_core::utils::copy( + config.root.join(doc.id.as_str()), + config.root.join(".build").join(doc.id.as_str()), + ) + .await + .ok(); - if doc.id.eq("FASTN.ftd") { - return Ok(()); - } + if doc.id.eq("FASTN.ftd") { + return Ok(()); } + let resp = fastn_core::package::package_doc::process_ftd( config, doc, @@ -248,45 +244,12 @@ async fn handle_file_( } fastn_core::File::Static(sa) => process_static(sa, &config.root, &config.package).await?, fastn_core::File::Markdown(doc) => { - if !config - .ftd_edition - .eq(&fastn_core::config::FTDEdition::FTD2021) - { - // TODO: bring this feature back - print!("Skipped "); - return Ok(()); - } - let resp = process_markdown(config, doc, base_url, build_static_files, test).await; - match (resp, ignore_failed) { - (Ok(r), _) => r, - (_, true) => { - print!("Failed "); - return Ok(()); - } - (e, _) => { - return e; - } - } + // TODO: bring this feature back + print!("Skipped "); + return Ok(()); } fastn_core::File::Image(main_doc) => { process_static(main_doc, &config.root, &config.package).await?; - if config - .ftd_edition - .eq(&fastn_core::config::FTDEdition::FTD2021) - { - let resp = - process_image(config, main_doc, base_url, build_static_files, test).await; - match (resp, ignore_failed) { - (Ok(r), _) => r, - (_, true) => { - print!("Failed "); - return Ok(()); - } - (e, _) => { - return e; - } - } - } } fastn_core::File::Code(doc) => { process_static( @@ -300,22 +263,6 @@ async fn handle_file_( &config.package, ) .await?; - if config - .ftd_edition - .eq(&fastn_core::config::FTDEdition::FTD2021) - { - let resp = process_code(config, doc, base_url, build_static_files, test).await; - match (resp, ignore_failed) { - (Ok(r), _) => r, - (_, true) => { - print!("Failed "); - return Ok(()); - } - (e, _) => { - return e; - } - } - } } } From 2aee28cd8077c13cd06fce8272e10af8d9adc4ab Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Sat, 5 Aug 2023 09:23:31 +0530 Subject: [PATCH 10/63] removing un-needed functions --- fastn-core/src/commands/build.rs | 161 +------------------------------ fastn-core/src/lib.rs | 149 ---------------------------- fastn-core/src/utils.rs | 9 -- 3 files changed, 1 insertion(+), 318 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index f55ad1e15d..f8a629ac7d 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -243,7 +243,7 @@ async fn handle_file_( } } fastn_core::File::Static(sa) => process_static(sa, &config.root, &config.package).await?, - fastn_core::File::Markdown(doc) => { + fastn_core::File::Markdown(_doc) => { // TODO: bring this feature back print!("Skipped "); return Ok(()); @@ -426,162 +426,3 @@ async fn process_static( Ok(()) } } - -async fn process_image( - config: &mut fastn_core::Config, - main: &fastn_core::Static, - base_url: &str, - build_static_files: bool, - test: bool, -) -> fastn_core::Result<()> { - let main = convert_to_ftd(config, main)?; - - fastn_core::package::package_doc::process_ftd( - config, - &main, - base_url, - build_static_files, - test, - ) - .await?; - return Ok(()); - - fn convert_to_ftd( - config: &fastn_core::Config, - doc: &fastn_core::Static, - ) -> fastn_core::Result { - Ok(fastn_core::Document { - package_name: config.package.name.to_string(), - id: convert_to_ftd_extension(doc.id.as_str())?, - content: fastn_core::package_info_image(config, doc, &config.package)?, - parent_path: doc.base_path.to_string(), - }) - } - - fn convert_to_ftd_extension(name: &str) -> fastn_core::Result { - Ok(format!("{}.ftd", name)) - } -} - -async fn process_code( - config: &mut fastn_core::Config, - main: &fastn_core::Document, - base_url: &str, - build_static_files: bool, - test: bool, -) -> fastn_core::Result<()> { - let main = if let Some(main) = convert_to_ftd(config, main)? { - main - } else { - return Ok(()); - }; - - fastn_core::package::package_doc::process_ftd( - config, - &main, - base_url, - build_static_files, - test, - ) - .await?; - return Ok(()); - - fn convert_to_ftd( - config: &fastn_core::Config, - doc: &fastn_core::Document, - ) -> fastn_core::Result> { - let id = convert_to_ftd_extension(doc.id.as_str())?; - let ext = fastn_core::utils::get_extension(doc.id.as_str())?; - let new_content = - fastn_core::package_info_code(config, id.as_str(), doc.content.as_str(), ext.as_str())?; - - let new_doc = { - let mut new_doc = doc.to_owned(); - new_doc.content = new_content; - new_doc.id = id; - new_doc - }; - - Ok(Some(new_doc)) - } - - fn convert_to_ftd_extension(name: &str) -> fastn_core::Result { - Ok(format!("{}.ftd", name)) - } -} - -async fn process_markdown( - config: &mut fastn_core::Config, - main: &fastn_core::Document, - base_url: &str, - build_static_files: bool, - test: bool, -) -> fastn_core::Result<()> { - let main = if let Some(main) = convert_md_to_ftd(config, main)? { - main - } else { - return Ok(()); - }; - fastn_core::package::package_doc::process_ftd( - config, - &main, - base_url, - build_static_files, - test, - ) - .await?; - return Ok(()); - - fn convert_md_to_ftd( - config: &fastn_core::Config, - doc: &fastn_core::Document, - ) -> fastn_core::Result> { - let doc_id = if doc.id == "README.md" - && !(camino::Utf8Path::new(format!(".{}index.ftd", std::path::MAIN_SEPARATOR).as_str()) - .exists() - || camino::Utf8Path::new( - format!(".{}index.md", std::path::MAIN_SEPARATOR).as_str(), - ) - .exists()) - { - "index.md".to_string() - } else if !camino::Utf8Path::new( - format!( - ".{}{}", - std::path::MAIN_SEPARATOR, - convert_md_to_ftd_extension(doc.id.as_str())? - ) - .as_str(), - ) - .exists() - { - doc.id.to_string() - } else { - return Ok(None); - }; - let id = convert_md_to_ftd_extension(doc_id.as_str())?; - let new_content = - fastn_core::package_info_markdown(config, id.as_str(), doc.content.as_str())?; - - let new_doc = { - let mut new_doc = doc.to_owned(); - new_doc.content = new_content; - new_doc.id = id; - new_doc - }; - - Ok(Some(new_doc)) - } - - fn convert_md_to_ftd_extension(name: &str) -> fastn_core::Result { - let file_name = if let Some(p1) = name.strip_suffix(".md") { - p1 - } else { - return Err(fastn_core::Error::UsageError { - message: format!("expected md file found: `{}`", name), - }); - }; - - Ok(format!("{}.ftd", file_name)) - } -} diff --git a/fastn-core/src/lib.rs b/fastn-core/src/lib.rs index f9c387c1d2..0f5f9b32f3 100644 --- a/fastn-core/src/lib.rs +++ b/fastn-core/src/lib.rs @@ -89,38 +89,6 @@ fn fastn_lib_ftd() -> &'static str { include_str!("../ftd/fastn-lib.ftd") } -fn package_info_image( - config: &fastn_core::Config, - doc: &fastn_core::Static, - package: &fastn_core::Package, -) -> fastn_core::Result { - let path = config.root.join("fastn").join("image.ftd"); - Ok(if path.is_file() { - std::fs::read_to_string(path)? - } else { - let body_prefix = match config.package.generate_prefix_string(false) { - Some(bp) => bp, - None => String::new(), - }; - indoc::formatdoc! {" - {body_prefix} - - -- import: {package_info_package}/image as pi - - -- ftd.image-src src: {src} - dark: {src} - - -- pi.image-page: {file_name} - src: $src - ", - body_prefix = body_prefix, - file_name = doc.id, - package_info_package = config.package_info_package(), - src = format!("-/{}/{}", package.name.as_str(), doc.id.as_str()), - } - }) -} - fn package_info_about(config: &fastn_core::Config) -> fastn_core::Result { let path = config.root.join("fastn").join("cr.ftd"); Ok(if path.is_file() { @@ -227,123 +195,6 @@ fn package_info_create_cr(config: &fastn_core::Config) -> fastn_core::Result fastn_core::Result { - let path = config.root.join("fastn").join("code.ftd"); - Ok(if path.is_file() { - std::fs::read_to_string(path)? - } else { - let body_prefix = match config.package.generate_prefix_string(false) { - Some(bp) => bp, - None => String::new(), - }; - if content.trim().is_empty() { - format!( - indoc::indoc! {" - {body_prefix} - - -- import: {package_info_package}/code as pi - - -- pi.code-page: {file_name} - lang: {ext} - - "}, - body_prefix = body_prefix, - package_info_package = config.package_info_package(), - file_name = file_name, - ext = extension, - ) - } else { - format!( - indoc::indoc! {" - {body_prefix} - - -- import: {package_info_package}/code as pi - - -- pi.code-page: {file_name} - lang: {ext} - - {content} - - "}, - body_prefix = body_prefix, - package_info_package = config.package_info_package(), - file_name = file_name, - ext = extension, - content = content, - ) - } - }) -} - -fn package_info_markdown( - config: &fastn_core::Config, - file_name: &str, - content: &str, -) -> fastn_core::Result { - let path = config.root.join("fastn").join("markdown.ftd"); - Ok(if path.is_file() { - std::fs::read_to_string(path)? - } else if !config - .ftd_edition - .eq(&fastn_core::config::FTDEdition::FTD2021) - { - if content.trim().is_empty() { - content.to_string() - } else { - format!( - indoc::indoc! {" - -- ftd.text: - - {content} - "}, - content = content, - ) - } - } else { - let body_prefix = match config.package.generate_prefix_string(false) { - Some(bp) => bp, - None => String::new(), - }; - if content.trim().is_empty() { - format!( - indoc::indoc! {" - {body_prefix} - - -- import: {package_info_package}/markdown as pi - - -- pi.markdown-page: {file_name} - - "}, - body_prefix = body_prefix, - package_info_package = config.package_info_package(), - file_name = file_name, - ) - } else { - format!( - indoc::indoc! {" - {body_prefix} - - -- import: {package_info_package}/markdown as pi - - -- pi.markdown-page: {file_name} - - {content} - - "}, - body_prefix = body_prefix, - package_info_package = config.package_info_package(), - content = content, - file_name = file_name, - ) - } - }) -} - #[allow(dead_code)] fn original_package_status(config: &fastn_core::Config) -> fastn_core::Result { let path = config diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index 94ceb81f04..98f7ef2709 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -318,15 +318,6 @@ pub(crate) async fn get_number_of_documents( Ok(no_of_docs) } -pub(crate) fn get_extension(file_name: &str) -> fastn_core::Result { - if let Some((_, ext)) = file_name.rsplit_once('.') { - return Ok(ext.to_string()); - } - Err(fastn_core::Error::UsageError { - message: format!("extension not found, `{}`", file_name), - }) -} - pub(crate) async fn get_current_document_last_modified_on( config: &fastn_core::Config, document_id: &str, From bbf4546bb0520694b15726f9f36bb44a9331a5a2 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Sat, 5 Aug 2023 09:34:49 +0530 Subject: [PATCH 11/63] build-dir plumbing --- fastn-core/src/commands/build.rs | 23 +++++++++++++---------- fastn-core/src/package/package_doc.rs | 25 ------------------------- 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index f8a629ac7d..893ef40826 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -53,11 +53,17 @@ pub async fn build( config.download_fonts().await } +mod build_dir { + pub(crate) fn get_build_content() -> std::collections::BTreeMap { + todo!() + } +} + mod cache { const FILE_NAME: &str = "fastn.cache"; pub(crate) fn get() -> Cache { - match fastn_core::utils::get_cached(FILE_NAME) { + let mut v = match fastn_core::utils::get_cached(FILE_NAME) { Some(v) => { tracing::debug!("cached hit"); v @@ -65,16 +71,21 @@ mod cache { None => { tracing::debug!("cached miss"); Cache { + build_content: std::collections::BTreeMap::new(), documents: vec![], assets: std::collections::BTreeMap::new(), } } - } + }; + v.build_content = super::build_dir::get_build_content(); + v } #[derive(serde::Serialize, serde::Deserialize)] pub(crate) struct Cache { // fastn_version: String, // TODO: Add this + #[serde(skip)] + pub(crate) build_content: std::collections::BTreeMap, pub(crate) documents: Vec, pub(crate) assets: std::collections::BTreeMap, } @@ -204,14 +215,6 @@ async fn handle_file_( config.current_document = Some(document.get_id().to_string()); match document { fastn_core::File::Ftd(doc) => { - // Ignore redirect paths - if let Some(r) = config.package.redirects.as_ref() { - if fastn_core::package::redirects::find_redirect(r, doc.id.as_str()).is_some() { - println!("Ignored by redirect {}", doc.id.as_str()); - return Ok(()); - } - } - fastn_core::utils::copy( config.root.join(doc.id.as_str()), config.root.join(".build").join(doc.id.as_str()), diff --git a/fastn-core/src/package/package_doc.rs b/fastn-core/src/package/package_doc.rs index 929574252c..82970417f4 100644 --- a/fastn-core/src/package/package_doc.rs +++ b/fastn-core/src/package/package_doc.rs @@ -544,31 +544,6 @@ pub(crate) async fn process_ftd( build_static_files: bool, test: bool, ) -> fastn_core::Result { - if main.id.eq("FASTN.ftd") { - tokio::fs::copy( - config.root.join(main.id.as_str()), - config.root.join(".build").join(main.id.as_str()), - ) - .await?; - } - - let main = { - let mut main = main.to_owned(); - if main.id.eq("FASTN.ftd") { - main.id = "-.ftd".to_string(); - let path = config.root.join("fastn").join("info.ftd"); - main.content = if path.is_file() { - std::fs::read_to_string(path)? - } else { - format!( - "-- import: {}/package-info as pi\n\n-- pi.package-info-page:", - config.package_info_package() - ) - } - } - main - }; - let file_rel_path = if main.id.eq("404.ftd") { "404.html".to_string() } else if main.id.ends_with("index.ftd") { From dfc4ae10e34b5733a57f6c0e85203047f3e5683c Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Sat, 5 Aug 2023 13:47:54 +0530 Subject: [PATCH 12/63] not sure what was happening there, cleanuped code --- fastn-core/src/commands/build.rs | 21 +-------------------- fastn-core/src/package/package_doc.rs | 2 +- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 893ef40826..7f0a39a89e 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -406,26 +406,7 @@ async fn process_static( .join(package.name.as_str()); std::fs::create_dir_all(&build_path)?; - if let Some((dir, _)) = sa.id.rsplit_once('/') { - std::fs::create_dir_all(build_path.join(dir))?; - } - std::fs::copy( - sa.base_path.join(sa.id.as_str()), - build_path.join(sa.id.as_str()), - )?; - - { - // TODO: need to remove this once download_base_url is removed - std::fs::create_dir_all(base_path.join(".build"))?; - if let Some((dir, _)) = sa.id.rsplit_once('/') { - std::fs::create_dir_all(base_path.join(".build").join(dir))?; - } - - std::fs::copy( - sa.base_path.join(sa.id.as_str()), - base_path.join(".build").join(sa.id.as_str()), - )?; - } + std::fs::write(build_path.join(sa.id.as_str()), sa.content.as_bytes())?; Ok(()) } } diff --git a/fastn-core/src/package/package_doc.rs b/fastn-core/src/package/package_doc.rs index 82970417f4..7c8f821437 100644 --- a/fastn-core/src/package/package_doc.rs +++ b/fastn-core/src/package/package_doc.rs @@ -552,7 +552,7 @@ pub(crate) async fn process_ftd( fastn_core::utils::replace_last_n(main.id.as_str(), 1, ".ftd", "/index.html") }; - let response = read_ftd(config, &main, base_url, build_static_files, test).await?; + let response = read_ftd(config, main, base_url, build_static_files, test).await?; fastn_core::utils::write( &config.build_dir(), file_rel_path.as_str(), From f8153467b7f834dee0330fa30271cd4140b6588e Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Sat, 5 Aug 2023 14:03:39 +0530 Subject: [PATCH 13/63] fastn build working --- fastn-core/src/commands/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 7f0a39a89e..6f4a287992 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -55,7 +55,7 @@ pub async fn build( mod build_dir { pub(crate) fn get_build_content() -> std::collections::BTreeMap { - todo!() + Default::default() } } @@ -406,7 +406,7 @@ async fn process_static( .join(package.name.as_str()); std::fs::create_dir_all(&build_path)?; - std::fs::write(build_path.join(sa.id.as_str()), sa.content.as_bytes())?; + std::fs::write(build_path.join(sa.id.as_str()), &sa.content)?; Ok(()) } } From bc553d7944e4d8bee9a3defad00a1252e2bbf0e6 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Sun, 6 Aug 2023 10:24:34 +0530 Subject: [PATCH 14/63] some caching implementation --- fastn-core/src/commands/build.rs | 54 ++++++++++++++++++++++++--- fastn-core/src/config/mod.rs | 2 + fastn-core/src/package/package_doc.rs | 20 +++------- fastn-core/src/utils.rs | 2 +- 4 files changed, 57 insertions(+), 21 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 6f4a287992..bd971f125d 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -72,7 +72,7 @@ mod cache { tracing::debug!("cached miss"); Cache { build_content: std::collections::BTreeMap::new(), - documents: vec![], + documents: std::collections::BTreeMap::new(), assets: std::collections::BTreeMap::new(), } } @@ -83,10 +83,10 @@ mod cache { #[derive(serde::Serialize, serde::Deserialize)] pub(crate) struct Cache { - // fastn_version: String, // TODO: Add this + // fastn_version: String, // TODO #[serde(skip)] pub(crate) build_content: std::collections::BTreeMap, - pub(crate) documents: Vec, + pub(crate) documents: std::collections::BTreeMap, pub(crate) assets: std::collections::BTreeMap, } @@ -202,7 +202,7 @@ async fn handle_file( Ok(()) } -#[tracing::instrument(skip(document, config, _cache))] +#[tracing::instrument(skip(document, config, cache))] async fn handle_file_( document: &fastn_core::File, config: &mut fastn_core::Config, @@ -210,11 +210,36 @@ async fn handle_file_( ignore_failed: bool, test: bool, build_static_files: bool, - _cache: Option<&mut cache::Cache>, + cache: Option<&mut cache::Cache>, ) -> fastn_core::Result<()> { config.current_document = Some(document.get_id().to_string()); + config.dependencies_during_render = vec![]; + match document { fastn_core::File::Ftd(doc) => { + let file_path = if doc.id.eq("404.ftd") { + "404.html".to_string() + } else if doc.id.ends_with("index.ftd") { + fastn_core::utils::replace_last_n(doc.id.as_str(), 1, "index.ftd", "index.html") + } else { + fastn_core::utils::replace_last_n(doc.id.as_str(), 1, ".ftd", "/index.html") + }; + + // find the document in cache + if let Some(ref cache) = cache { + if let Some(cached_doc) = cache.documents.get(doc.id.as_str()) { + // if it exists, check if the checksums match + // if they do, return + if let Some(doc_hash) = cache.build_content.get(file_path.as_str()) { + if doc_hash == &cached_doc.html_checksum { + return Ok(()); + } + } + } + // if it exists, check if the checksums match + // if they do, return + } + fastn_core::utils::copy( config.root.join(doc.id.as_str()), config.root.join(".build").join(doc.id.as_str()), @@ -232,10 +257,27 @@ async fn handle_file_( base_url, build_static_files, test, + file_path.as_str(), ) .await; match (resp, ignore_failed) { - (Ok(_), _) => (), + (Ok(r), _) => { + if let Some(cache) = cache { + cache.documents.insert( + doc.id.to_string(), + cache::Document { + file: cache::File { + path: doc.id.to_string(), + checksum: fastn_core::utils::generate_hash( + doc.content.as_str(), + ), + }, + html_checksum: r.checksum(), + dependencies: config.dependencies_during_render.clone(), + }, + ); + } + } (_, true) => { print!("Failed "); return Ok(()); diff --git a/fastn-core/src/config/mod.rs b/fastn-core/src/config/mod.rs index 841a9400a3..6b3a309cde 100644 --- a/fastn-core/src/config/mod.rs +++ b/fastn-core/src/config/mod.rs @@ -39,6 +39,7 @@ pub struct Config { pub named_parameters: Vec<(String, ftd::Value)>, pub extra_data: std::collections::BTreeMap, pub current_document: Option, + pub dependencies_during_render: Vec, pub request: Option, // TODO: It should only contain reference pub ftd_edition: FTDEdition, pub ftd_external_js: Vec, @@ -1291,6 +1292,7 @@ impl Config { ftd_inline_js: Default::default(), ftd_external_css: Default::default(), ftd_inline_css: Default::default(), + dependencies_during_render: Default::default(), }; // Update global_ids map from the current package files diff --git a/fastn-core/src/package/package_doc.rs b/fastn-core/src/package/package_doc.rs index 7c8f821437..5ae01fa1db 100644 --- a/fastn-core/src/package/package_doc.rs +++ b/fastn-core/src/package/package_doc.rs @@ -345,6 +345,10 @@ impl FTDResult { } } } + + pub fn checksum(&self) -> String { + fastn_core::utils::generate_hash(self.html()) + } } impl From for fastn_core::http::Response { @@ -543,22 +547,10 @@ pub(crate) async fn process_ftd( base_url: &str, build_static_files: bool, test: bool, + file_path: &str, ) -> fastn_core::Result { - let file_rel_path = if main.id.eq("404.ftd") { - "404.html".to_string() - } else if main.id.ends_with("index.ftd") { - fastn_core::utils::replace_last_n(main.id.as_str(), 1, "index.ftd", "index.html") - } else { - fastn_core::utils::replace_last_n(main.id.as_str(), 1, ".ftd", "/index.html") - }; - let response = read_ftd(config, main, base_url, build_static_files, test).await?; - fastn_core::utils::write( - &config.build_dir(), - file_rel_path.as_str(), - &response.html(), - ) - .await?; + fastn_core::utils::write(&config.build_dir(), file_path, &response.html()).await?; Ok(response) } diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index 98f7ef2709..d965273dd0 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -896,7 +896,7 @@ pub fn query(uri: &str) -> fastn_core::Result> { .collect_vec(), ) } -pub fn generate_hash(content: &str) -> String { +pub fn generate_hash(content: impl AsRef<[u8]>) -> String { use sha2::digest::FixedOutput; use sha2::Digest; let mut hasher = sha2::Sha256::new(); From 0ab3466492e3a93fc75cc1fb87b298b88759d749 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Sun, 6 Aug 2023 12:22:38 +0530 Subject: [PATCH 15/63] get_build_content() implementation --- fastn-core/src/commands/build.rs | 38 +++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index bd971f125d..c6ae064984 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -54,15 +54,41 @@ pub async fn build( } mod build_dir { - pub(crate) fn get_build_content() -> std::collections::BTreeMap { - Default::default() + pub(crate) fn get_build_content() -> std::io::Result> + { + let mut b = std::collections::BTreeMap::new(); + + for f in find_all_files_recursively(".build") { + b.insert( + f.to_string_lossy().to_string().replacen(".build/", "", 1), + fastn_core::utils::generate_hash(std::fs::read(&f)?), + ); + } + + Ok(dbg!(b)) + } + + fn find_all_files_recursively( + dir: impl AsRef + std::fmt::Debug, + ) -> Vec { + let mut files = vec![]; + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_dir() { + files.extend(find_all_files_recursively(&path)); + } else { + files.push(path) + } + } + files } } mod cache { const FILE_NAME: &str = "fastn.cache"; - pub(crate) fn get() -> Cache { + pub(crate) fn get() -> std::io::Result { let mut v = match fastn_core::utils::get_cached(FILE_NAME) { Some(v) => { tracing::debug!("cached hit"); @@ -77,8 +103,8 @@ mod cache { } } }; - v.build_content = super::build_dir::get_build_content(); - v + v.build_content = super::build_dir::get_build_content()?; + Ok(v) } #[derive(serde::Serialize, serde::Deserialize)] @@ -121,7 +147,7 @@ async fn incremental_build( ) -> fastn_core::Result<()> { // https://fastn.com/rfc/incremental-build/ - let mut c = cache::get(); + let mut c = cache::get()?; for document in documents.values() { handle_file( From 099d3e7c9e814b71e141ac7534408142e058af9a Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Sun, 6 Aug 2023 15:06:59 +0530 Subject: [PATCH 16/63] cache working, after ~3m, subsequent builds taking ~10s --- fastn-core/src/commands/build.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index c6ae064984..456e273007 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -107,7 +107,7 @@ mod cache { Ok(v) } - #[derive(serde::Serialize, serde::Deserialize)] + #[derive(serde::Serialize, serde::Deserialize, Debug)] pub(crate) struct Cache { // fastn_version: String, // TODO #[serde(skip)] @@ -123,13 +123,13 @@ mod cache { } } - #[derive(serde::Serialize, serde::Deserialize)] + #[derive(serde::Serialize, serde::Deserialize, Debug)] pub(crate) struct File { pub(crate) path: String, pub(crate) checksum: String, } - #[derive(serde::Serialize, serde::Deserialize)] + #[derive(serde::Serialize, serde::Deserialize, Debug)] pub(crate) struct Document { pub(crate) file: File, pub(crate) html_checksum: String, @@ -147,7 +147,7 @@ async fn incremental_build( ) -> fastn_core::Result<()> { // https://fastn.com/rfc/incremental-build/ - let mut c = cache::get()?; + let mut c = dbg!(cache::get()?); for document in documents.values() { handle_file( @@ -164,7 +164,7 @@ async fn incremental_build( // TODO: Handle deleted files (files present in cache/.build but not in documents) - c.cache_it()?; + dbg!(c).cache_it()?; Ok(()) } @@ -256,14 +256,23 @@ async fn handle_file_( if let Some(cached_doc) = cache.documents.get(doc.id.as_str()) { // if it exists, check if the checksums match // if they do, return + dbg!(cached_doc); if let Some(doc_hash) = cache.build_content.get(file_path.as_str()) { + dbg!(doc_hash); if doc_hash == &cached_doc.html_checksum { + println!("cache hit, returning"); return Ok(()); + } else { + println!("cache miss"); } } + } else { + println!("no cache entry for {}", doc.id.as_str()); } // if it exists, check if the checksums match // if they do, return + } else { + println!("no have cache"); } fastn_core::utils::copy( @@ -291,7 +300,7 @@ async fn handle_file_( if let Some(cache) = cache { cache.documents.insert( doc.id.to_string(), - cache::Document { + dbg!(cache::Document { file: cache::File { path: doc.id.to_string(), checksum: fastn_core::utils::generate_hash( @@ -300,7 +309,7 @@ async fn handle_file_( }, html_checksum: r.checksum(), dependencies: config.dependencies_during_render.clone(), - }, + }), ); } } From d101f1da04195e7752247f2da4cdab6be5470e0c Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Mon, 7 Aug 2023 17:42:26 +0530 Subject: [PATCH 17/63] refactored out is_cached() --- fastn-core/src/commands/build.rs | 67 +++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 456e273007..5335f4775b 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -228,6 +228,49 @@ async fn handle_file( Ok(()) } +fn is_cached( + cache: &Option<&mut cache::Cache>, + doc: &fastn_core::Document, + file_path: &str, +) -> bool { + let cache = match cache { + Some(c) => c, + None => { + println!("cache miss: no have cache"); + return false; + } + }; + + let cached_doc = match cache.documents.get(doc.id.as_str()) { + Some(cached_doc) => cached_doc, + None => { + println!("cache miss: no cache entry for {}", doc.id.as_str()); + return false; + } + }; + + // if it exists, check if the checksums match + // if they do, return + dbg!(cached_doc); + let doc_hash = match cache.build_content.get(file_path) { + Some(doc_hash) => doc_hash, + None => { + println!("cache miss: document not present in .build: {}", file_path); + return false; + } + }; + + dbg!(doc_hash); + + if doc_hash != &cached_doc.html_checksum { + println!("cache miss: html file checksums don't match"); + } + + // if it exists, check if the checksums match + println!("cache hit"); + true +} + #[tracing::instrument(skip(document, config, cache))] async fn handle_file_( document: &fastn_core::File, @@ -251,28 +294,8 @@ async fn handle_file_( fastn_core::utils::replace_last_n(doc.id.as_str(), 1, ".ftd", "/index.html") }; - // find the document in cache - if let Some(ref cache) = cache { - if let Some(cached_doc) = cache.documents.get(doc.id.as_str()) { - // if it exists, check if the checksums match - // if they do, return - dbg!(cached_doc); - if let Some(doc_hash) = cache.build_content.get(file_path.as_str()) { - dbg!(doc_hash); - if doc_hash == &cached_doc.html_checksum { - println!("cache hit, returning"); - return Ok(()); - } else { - println!("cache miss"); - } - } - } else { - println!("no cache entry for {}", doc.id.as_str()); - } - // if it exists, check if the checksums match - // if they do, return - } else { - println!("no have cache"); + if is_cached(&cache, doc, file_path.as_str()) { + return Ok(()); } fastn_core::utils::copy( From 566c895d8b08ef739d3751e7b0fc4042b728e6a0 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Mon, 7 Aug 2023 18:53:04 +0530 Subject: [PATCH 18/63] implemented ftd content hash check --- fastn-core/src/commands/build.rs | 77 ++++++++++++++++++++++++++------ fastn-core/src/utils.rs | 4 ++ 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 5335f4775b..b043c017d4 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -98,6 +98,7 @@ mod cache { tracing::debug!("cached miss"); Cache { build_content: std::collections::BTreeMap::new(), + ftd_cache: std::collections::BTreeMap::new(), documents: std::collections::BTreeMap::new(), assets: std::collections::BTreeMap::new(), } @@ -112,6 +113,8 @@ mod cache { // fastn_version: String, // TODO #[serde(skip)] pub(crate) build_content: std::collections::BTreeMap, + #[serde(skip)] + pub(crate) ftd_cache: std::collections::BTreeMap>, pub(crate) documents: std::collections::BTreeMap, pub(crate) assets: std::collections::BTreeMap, } @@ -121,15 +124,32 @@ mod cache { fastn_core::utils::cache_it(FILE_NAME, self)?; Ok(()) } + pub(crate) fn get_file_hash(&mut self, path: &str) -> fastn_core::Result { + match self.ftd_cache.get(path) { + Some(Some(v)) => Ok(v.to_owned()), + Some(None) => Err(fastn_core::Error::GenericError(path.to_string())), + None => { + let hash = match fastn_core::utils::get_file_hash(path) { + Ok(v) => v, + Err(e) => { + self.ftd_cache.insert(path.to_string(), None); + return Err(e); + } + }; + self.ftd_cache.insert(path.to_string(), Some(hash.clone())); + Ok(hash) + } + } + } } - #[derive(serde::Serialize, serde::Deserialize, Debug)] + #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] pub(crate) struct File { pub(crate) path: String, pub(crate) checksum: String, } - #[derive(serde::Serialize, serde::Deserialize, Debug)] + #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] pub(crate) struct Document { pub(crate) file: File, pub(crate) html_checksum: String, @@ -228,35 +248,35 @@ async fn handle_file( Ok(()) } -fn is_cached( - cache: &Option<&mut cache::Cache>, +fn is_cached<'a>( + cache: Option<&'a mut cache::Cache>, doc: &fastn_core::Document, file_path: &str, -) -> bool { - let cache = match cache { +) -> (Option<&'a mut cache::Cache>, bool) { + let cache: &mut cache::Cache = match cache { Some(c) => c, None => { println!("cache miss: no have cache"); - return false; + return (cache, false); } }; - let cached_doc = match cache.documents.get(doc.id.as_str()) { + let cached_doc: cache::Document = match cache.documents.get(doc.id.as_str()).cloned() { Some(cached_doc) => cached_doc, None => { println!("cache miss: no cache entry for {}", doc.id.as_str()); - return false; + return (Some(cache), false); } }; // if it exists, check if the checksums match // if they do, return - dbg!(cached_doc); + dbg!(&cached_doc); let doc_hash = match cache.build_content.get(file_path) { Some(doc_hash) => doc_hash, None => { println!("cache miss: document not present in .build: {}", file_path); - return false; + return (Some(cache), false); } }; @@ -264,11 +284,39 @@ fn is_cached( if doc_hash != &cached_doc.html_checksum { println!("cache miss: html file checksums don't match"); + return (Some(cache), false); + } + + if cached_doc.file.checksum != fastn_core::utils::generate_hash(doc.content.as_str()) { + println!("cache miss: ftd file checksums don't match"); + return (Some(cache), false); + } + + for dep in &cached_doc.dependencies { + let dep_doc = match cache.documents.get(dep) { + None => { + println!("cache miss: dependency {} not present in cache", dep); + return (Some(cache), false); + } + Some(dep_doc) => dep_doc.clone(), + }; + + let current_hash = match cache.get_file_hash(dep_doc.file.path.as_str()) { + Ok(hash) => hash, + Err(_) => { + println!("cache miss: dependency {} not present current folder", dep); + return (Some(cache), false); + } + }; + + if dep_doc.file.checksum != current_hash { + println!("cache miss: dependency {} checksums don't match", dep); + return (Some(cache), false); + } } - // if it exists, check if the checksums match println!("cache hit"); - true + (Some(cache), true) } #[tracing::instrument(skip(document, config, cache))] @@ -294,7 +342,8 @@ async fn handle_file_( fastn_core::utils::replace_last_n(doc.id.as_str(), 1, ".ftd", "/index.html") }; - if is_cached(&cache, doc, file_path.as_str()) { + let (cache, is_cached) = is_cached(cache, doc, file_path.as_str()); + if is_cached { return Ok(()); } diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index d965273dd0..e53f184dae 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -33,6 +33,10 @@ fn id_to_cache_key(id: &str) -> String { id.replace('/', "_") } +pub fn get_file_hash(_path: &str) -> fastn_core::Result { + todo!() +} + pub fn get_cached(id: &str) -> Option where T: serde::de::DeserializeOwned, From 7d42ff7b412a7fc106a6429e504cbf63052d583b Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Mon, 7 Aug 2023 19:29:54 +0530 Subject: [PATCH 19/63] cleaned up some code --- fastn-core/src/package/package_doc.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fastn-core/src/package/package_doc.rs b/fastn-core/src/package/package_doc.rs index 5ae01fa1db..6b05aa1161 100644 --- a/fastn-core/src/package/package_doc.rs +++ b/fastn-core/src/package/package_doc.rs @@ -384,7 +384,6 @@ pub(crate) async fn read_ftd( } } -#[allow(clippy::await_holding_refcell_ref)] #[tracing::instrument(name = "read_ftd_2022", skip_all)] pub(crate) async fn read_ftd_2022( config: &mut fastn_core::Config, @@ -394,7 +393,7 @@ pub(crate) async fn read_ftd_2022( test: bool, ) -> fastn_core::Result { let lib_config = config.clone(); - let mut all_packages = config.all_packages.borrow_mut(); + let all_packages = config.all_packages.borrow(); let current_package = all_packages .get(main.package_name.as_str()) .unwrap_or(&config.package); @@ -413,6 +412,7 @@ pub(crate) async fn read_ftd_2022( current_package.get_prefixed_body(main.content.as_str(), main.id.as_str(), true); // Fix aliased imports to full path (if any) doc_content = current_package.fix_imports_in_body(doc_content.as_str(), main.id.as_str())?; + drop(all_packages); let line_number = doc_content.split('\n').count() - main.content.split('\n').count(); let main_ftd_doc = match fastn_core::doc::interpret_helper( @@ -440,9 +440,6 @@ pub(crate) async fn read_ftd_2022( let node = ftd::node::NodeData::from_rt(executor); let html_ui = ftd::html::HtmlUI::from_node_data(node, "main", test)?; - all_packages.extend(lib.config.all_packages.into_inner()); - drop(all_packages); - config .downloaded_assets .extend(lib.config.downloaded_assets); From 7ccb48e0fba244de5f9edbc95e1cb133e3492075 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Mon, 7 Aug 2023 20:00:05 +0530 Subject: [PATCH 20/63] virtual file handling, ftd paths are incorrect --- fastn-core/src/commands/build.rs | 10 ++++++ fastn-core/src/doc.rs | 24 +++++++++---- fastn-core/src/library2022/mod.rs | 49 ++++++++++++++------------- fastn-core/src/package/package_doc.rs | 3 +- 4 files changed, 55 insertions(+), 31 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index b043c017d4..5c5ca95b28 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -125,6 +125,16 @@ mod cache { Ok(()) } pub(crate) fn get_file_hash(&mut self, path: &str) -> fastn_core::Result { + if path == "$fastn$/fastn.ftd" + || path == "$fastn$/processors.ftd" + || path == "$fastn$/time.ftd" + || path.ends_with("/-/fonts.ftd") + || path.ends_with("/-/assets.ftd") + { + // these are virtual file, they don't exist on disk, and hash only changes when + // fastn source changes + return Ok("hello".to_string()); + } match self.ftd_cache.get(path) { Some(Some(v)) => Ok(v.to_owned()), Some(None) => Err(fastn_core::Error::GenericError(path.to_string())), diff --git a/fastn-core/src/doc.rs b/fastn-core/src/doc.rs index 8a69e08647..8eb649e016 100644 --- a/fastn-core/src/doc.rs +++ b/fastn-core/src/doc.rs @@ -53,9 +53,10 @@ pub async fn interpret_helper<'a>( state: mut st, caller_module, } => { - let (source, foreign_variable, foreign_function, ignore_line_numbers) = + let (source, path, foreign_variable, foreign_function, ignore_line_numbers) = resolve_import_2022(lib, &mut st, module.as_str(), caller_module.as_str()) .await?; + lib.config.dependencies_during_render.push(path); let doc = cached_parse(module.as_str(), source.as_str(), ignore_line_numbers)?; s = st.continue_after_import( module.as_str(), @@ -162,13 +163,20 @@ pub async fn resolve_import_2022<'a>( _state: &mut ftd::interpreter::InterpreterState, module: &str, caller_module: &str, -) -> ftd::interpreter::Result<(String, Vec, Vec, usize)> { +) -> ftd::interpreter::Result<(String, String, Vec, Vec, usize)> { let current_package = lib.get_current_package(caller_module)?; let source = if module.eq("fastn/time") { - ("".to_string(), vec!["time".to_string()], vec![], 0) + ( + "".to_string(), + "$fastn$/time.ftd".to_string(), + vec!["time".to_string()], + vec![], + 0, + ) } else if module.eq("fastn/processors") { ( fastn_core::processor_ftd().to_string(), + "$fastn$/processors.ftd".to_string(), vec![], vec![ "figma-typo-token".to_string(), @@ -206,12 +214,14 @@ pub async fn resolve_import_2022<'a>( if module.starts_with(current_package.name.as_str()) { ( current_package.get_font_ftd().unwrap_or_default(), + format!("{name}/-/assets.ftd", name = current_package.name), foreign_variable, vec![], 0, ) } else { let mut font_ftd = "".to_string(); + let mut path = "".to_string(); for (alias, package) in current_package.aliases() { if module.starts_with(alias) { lib.push_package_under_process(module, package).await?; @@ -223,16 +233,18 @@ pub async fn resolve_import_2022<'a>( .unwrap() .get_font_ftd() .unwrap_or_default(); + path = format!("{name}/-/fonts.ftd", name = package.name); break; } } - (font_ftd, foreign_variable, vec![], 0) + (font_ftd, path, foreign_variable, vec![], 0) } } else { - let (content, ignore_line_numbers) = lib.get_with_result(module, caller_module).await?; - + let (content, path, ignore_line_numbers) = + lib.get_with_result(module, caller_module).await?; ( content, + path, vec![], vec![ "figma-typo-token".to_string(), diff --git a/fastn-core/src/library2022/mod.rs b/fastn-core/src/library2022/mod.rs index c3fc2dffa9..e5106dd70c 100644 --- a/fastn-core/src/library2022/mod.rs +++ b/fastn-core/src/library2022/mod.rs @@ -30,7 +30,7 @@ impl Library2022 { &mut self, name: &str, current_processing_module: &str, - ) -> ftd::p1::Result<(String, usize)> { + ) -> ftd::p1::Result<(String, String, usize)> { match self.get(name, current_processing_module).await { Some(v) => Ok(v), None => ftd::p1::utils::parse_error(format!("library not found 1: {}", name), "", 0), @@ -67,9 +67,13 @@ impl Library2022 { &mut self, name: &str, current_processing_module: &str, - ) -> Option<(String, usize)> { + ) -> Option<(String, String, usize)> { if name == "fastn" { - return Some((fastn_core::library::fastn_dot_ftd::get2022(self).await, 0)); + return Some(( + fastn_core::library::fastn_dot_ftd::get2022(self).await, + "$fastn$/fastn.ftd".to_string(), + 0, + )); } return get_for_package( @@ -83,33 +87,31 @@ impl Library2022 { name: &str, lib: &mut fastn_core::Library2022, current_processing_module: &str, - ) -> Option<(String, usize)> { + ) -> Option<(String, String, usize)> { let package = lib.get_current_package(current_processing_module).ok()?; if name.starts_with(package.name.as_str()) { - if let Some(r) = get_data_from_package(name, &package, lib).await { - return Some(r); + if let Some((content, size)) = get_data_from_package(name, &package, lib).await { + return Some((content, format!("{name}.ftd"), size)); } } // Self package referencing if package.name.ends_with(name.trim_end_matches('/')) { let package_index = format!("{}/", package.name.as_str()); - if let Some(r) = get_data_from_package(package_index.as_str(), &package, lib).await + if let Some((content, size)) = + get_data_from_package(package_index.as_str(), &package, lib).await { - return Some(r); + return Some((content, format!("{package_index}index.ftd"), size)); } } for (alias, package) in package.aliases() { lib.push_package_under_process(name, package).await.ok()?; if name.starts_with(alias) { - if let Some(r) = get_data_from_package( - name.replacen(alias, &package.name, 1).as_str(), - package, - lib, - ) - .await + let name = name.replacen(alias, &package.name, 1); + if let Some((content, size)) = + get_data_from_package(name.as_str(), package, lib).await { - return Some(r); + return Some((content, format!("{name}.ftd"), size)); } } } @@ -121,21 +123,20 @@ impl Library2022 { let name = name.replacen(package.name.as_str(), translation_of.name.as_str(), 1); if name.starts_with(translation_of.name.as_str()) { - if let Some(r) = get_data_from_package(name.as_str(), &translation_of, lib).await { - return Some(r); + if let Some((content, size)) = + get_data_from_package(name.as_str(), &translation_of, lib).await + { + return Some((content, format!("{name}.ftd"), size)); } } for (alias, package) in translation_of.aliases() { if name.starts_with(alias) { - if let Some(r) = get_data_from_package( - name.replacen(alias, &package.name, 1).as_str(), - package, - lib, - ) - .await + let name = name.replacen(alias, &package.name, 1); + if let Some((content, size)) = + get_data_from_package(name.as_str(), package, lib).await { - return Some(r); + return Some((content, format!("{name}.ftd"), size)); } } } diff --git a/fastn-core/src/package/package_doc.rs b/fastn-core/src/package/package_doc.rs index 6b05aa1161..a6222e611b 100644 --- a/fastn-core/src/package/package_doc.rs +++ b/fastn-core/src/package/package_doc.rs @@ -433,6 +433,7 @@ pub(crate) async fn read_ftd_2022( }); } }; + config.dependencies_during_render = lib.config.dependencies_during_render; if let Some((url, code)) = main_ftd_doc.get_redirect() { return Ok(FTDResult::Redirect { url, code }); } @@ -505,7 +506,7 @@ pub(crate) async fn read_ftd_2023( }); } }; - + config.dependencies_during_render = lib.config.dependencies_during_render; if let Some((url, code)) = main_ftd_doc.get_redirect() { return Ok(FTDResult::Redirect { url, code }); } From 957017dc893cd8dd3c83978db67ee9f5b2ea122a Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Wed, 9 Aug 2023 10:16:20 +0530 Subject: [PATCH 21/63] some changes, not working yet --- fastn-core/src/commands/build.rs | 63 +++++++++++++++++++------------ fastn-core/src/library2022/mod.rs | 8 ++-- fastn-core/src/utils.rs | 31 ++++++++++----- 3 files changed, 64 insertions(+), 38 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 5c5ca95b28..8aed8ba91a 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -65,7 +65,7 @@ mod build_dir { ); } - Ok(dbg!(b)) + Ok(b) } fn find_all_files_recursively( @@ -100,7 +100,7 @@ mod cache { build_content: std::collections::BTreeMap::new(), ftd_cache: std::collections::BTreeMap::new(), documents: std::collections::BTreeMap::new(), - assets: std::collections::BTreeMap::new(), + file_checksum: std::collections::BTreeMap::new(), } } }; @@ -116,7 +116,7 @@ mod cache { #[serde(skip)] pub(crate) ftd_cache: std::collections::BTreeMap>, pub(crate) documents: std::collections::BTreeMap, - pub(crate) assets: std::collections::BTreeMap, + pub(crate) file_checksum: std::collections::BTreeMap, } impl Cache { @@ -125,9 +125,7 @@ mod cache { Ok(()) } pub(crate) fn get_file_hash(&mut self, path: &str) -> fastn_core::Result { - if path == "$fastn$/fastn.ftd" - || path == "$fastn$/processors.ftd" - || path == "$fastn$/time.ftd" + if path.starts_with("$fastn$/") || path.ends_with("/-/fonts.ftd") || path.ends_with("/-/assets.ftd") { @@ -139,7 +137,7 @@ mod cache { Some(Some(v)) => Ok(v.to_owned()), Some(None) => Err(fastn_core::Error::GenericError(path.to_string())), None => { - let hash = match fastn_core::utils::get_file_hash(path) { + let hash = match fastn_core::utils::get_ftd_hash(path) { Ok(v) => v, Err(e) => { self.ftd_cache.insert(path.to_string(), None); @@ -161,7 +159,6 @@ mod cache { #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] pub(crate) struct Document { - pub(crate) file: File, pub(crate) html_checksum: String, pub(crate) dependencies: Vec, } @@ -271,10 +268,12 @@ fn is_cached<'a>( } }; - let cached_doc: cache::Document = match cache.documents.get(doc.id.as_str()).cloned() { + let id = remove_extension(doc.id.as_str()); + + let cached_doc: cache::Document = match cache.documents.get(id.as_str()).cloned() { Some(cached_doc) => cached_doc, None => { - println!("cache miss: no cache entry for {}", doc.id.as_str()); + println!("cache miss: no cache entry for {}", id.as_str()); return (Some(cache), false); } }; @@ -297,21 +296,29 @@ fn is_cached<'a>( return (Some(cache), false); } - if cached_doc.file.checksum != fastn_core::utils::generate_hash(doc.content.as_str()) { + let file_checksum = match cache.file_checksum.get(id.as_str()).cloned() { + Some(file_checksum) => file_checksum, + None => { + println!("cache miss: no cache entry for {}", id.as_str()); + return (Some(cache), false); + } + }; + + if file_checksum != fastn_core::utils::generate_hash(doc.content.as_str()) { println!("cache miss: ftd file checksums don't match"); return (Some(cache), false); } for dep in &cached_doc.dependencies { - let dep_doc = match cache.documents.get(dep) { + let file_checksum = match cache.file_checksum.get(dep) { None => { - println!("cache miss: dependency {} not present in cache", dep); + println!("cache miss: file {} not present in cache", dep); return (Some(cache), false); } - Some(dep_doc) => dep_doc.clone(), + Some(file_checksum) => file_checksum.clone(), }; - let current_hash = match cache.get_file_hash(dep_doc.file.path.as_str()) { + let current_hash = match cache.get_file_hash(dep.as_str()) { Ok(hash) => hash, Err(_) => { println!("cache miss: dependency {} not present current folder", dep); @@ -319,7 +326,7 @@ fn is_cached<'a>( } }; - if dep_doc.file.checksum != current_hash { + if file_checksum != current_hash { println!("cache miss: dependency {} checksums don't match", dep); return (Some(cache), false); } @@ -329,6 +336,14 @@ fn is_cached<'a>( (Some(cache), true) } +fn remove_extension(id: &str) -> String { + if id.ends_with("/index.ftd") { + fastn_core::utils::replace_last_n(id, 1, "/index.ftd", "") + } else { + fastn_core::utils::replace_last_n(id, 1, ".ftd", "") + } +} + #[tracing::instrument(skip(document, config, cache))] async fn handle_file_( document: &fastn_core::File, @@ -381,17 +396,15 @@ async fn handle_file_( (Ok(r), _) => { if let Some(cache) = cache { cache.documents.insert( - doc.id.to_string(), - dbg!(cache::Document { - file: cache::File { - path: doc.id.to_string(), - checksum: fastn_core::utils::generate_hash( - doc.content.as_str(), - ), - }, + remove_extension(doc.id.as_str()), + cache::Document { html_checksum: r.checksum(), dependencies: config.dependencies_during_render.clone(), - }), + }, + ); + cache.file_checksum.insert( + remove_extension(doc.id.as_str()), + fastn_core::utils::generate_hash(doc.content.as_str()), ); } } diff --git a/fastn-core/src/library2022/mod.rs b/fastn-core/src/library2022/mod.rs index e5106dd70c..e15baaa545 100644 --- a/fastn-core/src/library2022/mod.rs +++ b/fastn-core/src/library2022/mod.rs @@ -91,7 +91,7 @@ impl Library2022 { let package = lib.get_current_package(current_processing_module).ok()?; if name.starts_with(package.name.as_str()) { if let Some((content, size)) = get_data_from_package(name, &package, lib).await { - return Some((content, format!("{name}.ftd"), size)); + return Some((content, name.to_string(), size)); } } // Self package referencing @@ -111,7 +111,7 @@ impl Library2022 { if let Some((content, size)) = get_data_from_package(name.as_str(), package, lib).await { - return Some((content, format!("{name}.ftd"), size)); + return Some((content, name.to_string(), size)); } } } @@ -126,7 +126,7 @@ impl Library2022 { if let Some((content, size)) = get_data_from_package(name.as_str(), &translation_of, lib).await { - return Some((content, format!("{name}.ftd"), size)); + return Some((content, name.to_string(), size)); } } @@ -136,7 +136,7 @@ impl Library2022 { if let Some((content, size)) = get_data_from_package(name.as_str(), package, lib).await { - return Some((content, format!("{name}.ftd"), size)); + return Some((content, name.to_string(), size)); } } } diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index e53f184dae..089447c2ed 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -33,17 +33,32 @@ fn id_to_cache_key(id: &str) -> String { id.replace('/', "_") } -pub fn get_file_hash(_path: &str) -> fastn_core::Result { - todo!() +pub fn get_ftd_hash(path: &str) -> fastn_core::Result { + let path = fastn_core::utils::replace_last_n(path, 1, "/", ""); + Ok(fastn_core::utils::generate_hash( + std::fs::read(format!("{path}.ftd")) + .or_else(|_| std::fs::read(format!("{path}/index.ftd")))?, + )) +} + +pub fn get_cache_file(id: &str) -> Option { + Some( + dirs::cache_dir()? + .join("fastn.com/") + .join(id_to_cache_key( + &std::env::current_dir() + .expect("cant read current dir") + .to_string_lossy(), + )) + .join(id_to_cache_key(id)), + ) } pub fn get_cached(id: &str) -> Option where T: serde::de::DeserializeOwned, { - let cache_file = dirs::cache_dir()? - .join("fastn.com/ast-cache/") - .join(id_to_cache_key(id)); + let cache_file = get_cache_file(id)?; serde_json::from_str( &std::fs::read_to_string(cache_file) .map_err(|e| { @@ -63,10 +78,8 @@ pub fn cache_it(id: &str, d: T) -> ftd::interpreter::Result where T: serde::ser::Serialize, { - let cache_file = dirs::cache_dir() - .ok_or_else(|| ftd::interpreter::Error::OtherError("cache dir not found".to_string()))? - .join("fastn.com/ast-cache/") - .join(id_to_cache_key(id)); + let cache_file = get_cache_file(id) + .ok_or_else(|| ftd::interpreter::Error::OtherError("cache dir not found".to_string()))?; std::fs::create_dir_all(cache_file.parent().unwrap()).map_err(|e| { ftd::interpreter::Error::OtherError(format!("failed to create cache dir: {}", e)) })?; From 03fa8d59725daa4869c273571e61bdc3c660e0f8 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Wed, 9 Aug 2023 10:36:25 +0530 Subject: [PATCH 22/63] some debug statements --- fastn-core/src/commands/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 8aed8ba91a..af7397b14b 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -396,9 +396,9 @@ async fn handle_file_( (Ok(r), _) => { if let Some(cache) = cache { cache.documents.insert( - remove_extension(doc.id.as_str()), + dbg!(remove_extension(doc.id.as_str())), cache::Document { - html_checksum: r.checksum(), + html_checksum: dbg!(r.checksum()), dependencies: config.dependencies_during_render.clone(), }, ); From 5da963bfc660e958762873654a180feaa474ed62 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Wed, 23 Aug 2023 12:46:21 +0530 Subject: [PATCH 23/63] implement dependency resolver for incremental build --- fastn-core/src/commands/build.rs | 171 ++++++++++++++++++++++++++++--- 1 file changed, 158 insertions(+), 13 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index af7397b14b..d2688a9e63 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -19,7 +19,7 @@ pub async fn build( return handle_only_id(id, config, base_url, ignore_failed, test, documents).await } None => { - incremental_build(config, documents, base_url, ignore_failed, test).await?; + incremental_build(config, &documents, base_url, ignore_failed, test).await?; } } } @@ -164,10 +164,20 @@ mod cache { } } +fn get_dependency_name_without_package_name(package_name: &str, dependency_name: &str) -> String { + if let Some(remaining) = dependency_name.strip_prefix(&format!("{}/", package_name)) { + remaining.to_string() + } else { + dependency_name.to_string() + } + .trim_end_matches('/') + .to_string() +} + #[tracing::instrument(skip(config, documents))] async fn incremental_build( config: &mut fastn_core::Config, - documents: std::collections::BTreeMap, + documents: &std::collections::BTreeMap, base_url: &str, ignore_failed: bool, test: bool, @@ -176,17 +186,96 @@ async fn incremental_build( let mut c = dbg!(cache::get()?); - for document in documents.values() { - handle_file( - document, - config, - base_url, - ignore_failed, - test, - true, - Some(&mut c), - ) - .await?; + let mut unresolved_dependencies: Vec = vec!["index".to_string()]; + + let mut resolved_dependencies: Vec = vec![]; + + let mut resolving_dependencies: Vec = vec![]; + + while let Some(unresolved_dependency) = unresolved_dependencies.pop() { + dbg!(unresolved_dependency.as_str()); + + dbg!(&unresolved_dependencies); + + if let Some(doc) = dbg!(c.documents.get( + get_dependency_name_without_package_name( + config.package.name.as_str(), + unresolved_dependency.as_str() + ) + .as_str() + )) { + let mut own_resolved_dependencies: Vec = vec![]; + + for dep in &doc.dependencies { + if resolved_dependencies.contains(dep) { + own_resolved_dependencies.push(dep.to_string()); + continue; + } + + unresolved_dependencies.push(dep.to_string()); + } + + dbg!( + &own_resolved_dependencies, + &doc.dependencies, + &unresolved_dependency + ); + + if own_resolved_dependencies.eq(&doc.dependencies) { + dbg!( + &own_resolved_dependencies, + &doc.dependencies, + &unresolved_dependency + ); + for (doc_id, doc) in documents { + if dbg!(doc_id.eq(format!( + "{}.ftd", + get_dependency_name_without_package_name( + config.package.name.as_str(), + unresolved_dependency.as_str() + ) + ) + .as_str())) + { + handle_file( + doc, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; + + break; + } + } + + resolved_dependencies.push(unresolved_dependency.to_string()); + if unresolved_dependencies.is_empty() { + if let Some(resolving_dependency) = resolving_dependencies.pop() { + unresolved_dependencies.push(resolving_dependency); + } + } + } else { + resolving_dependencies.push(unresolved_dependency.to_string()); + } + } else { + unresolved_dependencies.push(unresolved_dependency.to_string()); + for doc in documents.values() { + handle_file( + doc, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; + } + } } // TODO: Handle deleted files (files present in cache/.build but not in documents) @@ -196,6 +285,62 @@ async fn incremental_build( Ok(()) } +/** + +for (doc_id, document) in c.documents.clone() { + if doc_id.eq(unresolved_document.as_str()) { + unresolved_documents.push(unresolved_document.to_string()); + + if document.dependencies.is_empty() { + for doc in documents.values() { + if doc.get_id_with_package().eq(unresolved_document.as_str()) { + handle_file( + doc, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; + + resolved_dependencies.push(unresolved_document.to_string()); + } + } + } + + for dep in &document.dependencies { + if resolved_dependencies.contains(&dep) { + continue; + } + + unresolved_documents.push(dep.to_string()); + } + + break; + } + } + + for doc in documents.values() { + if doc.get_id_with_package().eq(unresolved_document.as_str()) { + handle_file( + doc, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; + + resolved_dependencies.push(unresolved_document.to_string()); + } + } + + */ + #[tracing::instrument(skip(config, documents))] async fn handle_only_id( id: &str, From 37179262ecbafd53624df6fd681d4f2314aff026 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Wed, 23 Aug 2023 13:48:21 +0530 Subject: [PATCH 24/63] created overwrite function to update ftd files --- fastn-core/src/commands/build.rs | 27 ++++++--------------------- fastn-core/src/package/package_doc.rs | 2 +- fastn-core/src/utils.rs | 8 ++++++++ 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index d2688a9e63..b139bd9f83 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -193,17 +193,13 @@ async fn incremental_build( let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { - dbg!(unresolved_dependency.as_str()); - - dbg!(&unresolved_dependencies); - - if let Some(doc) = dbg!(c.documents.get( + if let Some(doc) = c.documents.get( get_dependency_name_without_package_name( config.package.name.as_str(), - unresolved_dependency.as_str() + unresolved_dependency.as_str(), ) - .as_str() - )) { + .as_str(), + ) { let mut own_resolved_dependencies: Vec = vec![]; for dep in &doc.dependencies { @@ -215,27 +211,16 @@ async fn incremental_build( unresolved_dependencies.push(dep.to_string()); } - dbg!( - &own_resolved_dependencies, - &doc.dependencies, - &unresolved_dependency - ); - if own_resolved_dependencies.eq(&doc.dependencies) { - dbg!( - &own_resolved_dependencies, - &doc.dependencies, - &unresolved_dependency - ); for (doc_id, doc) in documents { - if dbg!(doc_id.eq(format!( + if doc_id.eq(format!( "{}.ftd", get_dependency_name_without_package_name( config.package.name.as_str(), unresolved_dependency.as_str() ) ) - .as_str())) + .as_str()) { handle_file( doc, diff --git a/fastn-core/src/package/package_doc.rs b/fastn-core/src/package/package_doc.rs index a6222e611b..7eb8ae45a5 100644 --- a/fastn-core/src/package/package_doc.rs +++ b/fastn-core/src/package/package_doc.rs @@ -548,7 +548,7 @@ pub(crate) async fn process_ftd( file_path: &str, ) -> fastn_core::Result { let response = read_ftd(config, main, base_url, build_static_files, test).await?; - fastn_core::utils::write(&config.build_dir(), file_path, &response.html()).await?; + fastn_core::utils::overwrite(&config.build_dir(), file_path, &response.html()).await?; Ok(response) } diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index 089447c2ed..9488282633 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -765,6 +765,14 @@ pub(crate) async fn write( update1(root, file_path, data).await } +pub(crate) async fn overwrite( + root: &camino::Utf8PathBuf, + file_path: &str, + data: &[u8], +) -> fastn_core::Result<()> { + update1(root, file_path, data).await +} + // TODO: remove this function use update instead pub(crate) async fn update1( root: &camino::Utf8PathBuf, From 6521451656c9adf24620e7296ca55aaace7f68a2 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Wed, 23 Aug 2023 16:42:56 +0530 Subject: [PATCH 25/63] fixed infinite loop in dependency resolution --- fastn-core/src/commands/build.rs | 93 ++++---------------------------- 1 file changed, 11 insertions(+), 82 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index b139bd9f83..624d791ec1 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -193,6 +193,8 @@ async fn incremental_build( let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { + dbg!(&unresolved_dependencies, &unresolved_dependency); + if let Some(doc) = c.documents.get( get_dependency_name_without_package_name( config.package.name.as_str(), @@ -212,30 +214,14 @@ async fn incremental_build( } if own_resolved_dependencies.eq(&doc.dependencies) { - for (doc_id, doc) in documents { - if doc_id.eq(format!( - "{}.ftd", - get_dependency_name_without_package_name( - config.package.name.as_str(), - unresolved_dependency.as_str() - ) - ) - .as_str()) - { - handle_file( - doc, - config, - base_url, - ignore_failed, - test, - true, - Some(&mut c), - ) - .await?; - - break; - } - } + handle_only_id( + unresolved_dependency.as_str(), + config, + base_url, + ignore_failed, + test, + documents.clone(), + ).await?; resolved_dependencies.push(unresolved_dependency.to_string()); if unresolved_dependencies.is_empty() { @@ -247,7 +233,6 @@ async fn incremental_build( resolving_dependencies.push(unresolved_dependency.to_string()); } } else { - unresolved_dependencies.push(unresolved_dependency.to_string()); for doc in documents.values() { handle_file( doc, @@ -270,62 +255,6 @@ async fn incremental_build( Ok(()) } -/** - -for (doc_id, document) in c.documents.clone() { - if doc_id.eq(unresolved_document.as_str()) { - unresolved_documents.push(unresolved_document.to_string()); - - if document.dependencies.is_empty() { - for doc in documents.values() { - if doc.get_id_with_package().eq(unresolved_document.as_str()) { - handle_file( - doc, - config, - base_url, - ignore_failed, - test, - true, - Some(&mut c), - ) - .await?; - - resolved_dependencies.push(unresolved_document.to_string()); - } - } - } - - for dep in &document.dependencies { - if resolved_dependencies.contains(&dep) { - continue; - } - - unresolved_documents.push(dep.to_string()); - } - - break; - } - } - - for doc in documents.values() { - if doc.get_id_with_package().eq(unresolved_document.as_str()) { - handle_file( - doc, - config, - base_url, - ignore_failed, - test, - true, - Some(&mut c), - ) - .await?; - - resolved_dependencies.push(unresolved_document.to_string()); - } - } - - */ - #[tracing::instrument(skip(config, documents))] async fn handle_only_id( id: &str, @@ -336,7 +265,7 @@ async fn handle_only_id( documents: std::collections::BTreeMap, ) -> fastn_core::Result<()> { for doc in documents.values() { - if doc.get_id().eq(id) || doc.get_id_with_package().eq(id) { + if dbg!(doc.get_id().eq(id) || doc.get_id_with_package().eq(id)) { return handle_file(doc, config, base_url, ignore_failed, test, false, None).await; } } From 860efd5592683261b802967b595d162d2c29f673 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 24 Aug 2023 16:43:41 +0530 Subject: [PATCH 26/63] Fixed dependency resolver --- fastn-core/src/commands/build.rs | 56 ++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 624d791ec1..d923e8cd84 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -193,8 +193,6 @@ async fn incremental_build( let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { - dbg!(&unresolved_dependencies, &unresolved_dependency); - if let Some(doc) = c.documents.get( get_dependency_name_without_package_name( config.package.name.as_str(), @@ -214,14 +212,33 @@ async fn incremental_build( } if own_resolved_dependencies.eq(&doc.dependencies) { - handle_only_id( - unresolved_dependency.as_str(), - config, - base_url, - ignore_failed, - test, - documents.clone(), - ).await?; + let name_with_extension = format!( + "{}.ftd", + get_dependency_name_without_package_name( + config.package.name.as_str(), + unresolved_dependency.as_str() + ) + ); + + for document in documents.values() { + if document.get_id().eq(name_with_extension.as_str()) + || document + .get_id_with_package() + .eq(name_with_extension.as_str()) + { + handle_file( + document, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; + break; + } + } resolved_dependencies.push(unresolved_dependency.to_string()); if unresolved_dependencies.is_empty() { @@ -233,18 +250,7 @@ async fn incremental_build( resolving_dependencies.push(unresolved_dependency.to_string()); } } else { - for doc in documents.values() { - handle_file( - doc, - config, - base_url, - ignore_failed, - test, - true, - Some(&mut c), - ) - .await?; - } + resolved_dependencies.push(unresolved_dependency); } } @@ -265,7 +271,7 @@ async fn handle_only_id( documents: std::collections::BTreeMap, ) -> fastn_core::Result<()> { for doc in documents.values() { - if dbg!(doc.get_id().eq(id) || doc.get_id_with_package().eq(id)) { + if doc.get_id().eq(id) || doc.get_id_with_package().eq(id) { return handle_file(doc, config, base_url, ignore_failed, test, false, None).await; } } @@ -339,7 +345,7 @@ fn is_cached<'a>( // if it exists, check if the checksums match // if they do, return - dbg!(&cached_doc); + // dbg!(&cached_doc); let doc_hash = match cache.build_content.get(file_path) { Some(doc_hash) => doc_hash, None => { @@ -348,7 +354,7 @@ fn is_cached<'a>( } }; - dbg!(doc_hash); + // dbg!(doc_hash); if doc_hash != &cached_doc.html_checksum { println!("cache miss: html file checksums don't match"); From 68df5b6e94d59052c4e0715563ac73fd1a5d2a99 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 24 Aug 2023 17:38:09 +0530 Subject: [PATCH 27/63] Handle cache miss --- fastn-core/src/commands/build.rs | 151 ++++++++++++++++++------------- 1 file changed, 87 insertions(+), 64 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index d923e8cd84..d107c264a3 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -88,24 +88,27 @@ mod build_dir { mod cache { const FILE_NAME: &str = "fastn.cache"; - pub(crate) fn get() -> std::io::Result { - let mut v = match fastn_core::utils::get_cached(FILE_NAME) { + pub(crate) fn get() -> std::io::Result<(bool, Cache)> { + let (cache_hit, mut v) = match fastn_core::utils::get_cached(FILE_NAME) { Some(v) => { tracing::debug!("cached hit"); - v + (true, v) } None => { tracing::debug!("cached miss"); - Cache { - build_content: std::collections::BTreeMap::new(), - ftd_cache: std::collections::BTreeMap::new(), - documents: std::collections::BTreeMap::new(), - file_checksum: std::collections::BTreeMap::new(), - } + ( + false, + Cache { + build_content: std::collections::BTreeMap::new(), + ftd_cache: std::collections::BTreeMap::new(), + documents: std::collections::BTreeMap::new(), + file_checksum: std::collections::BTreeMap::new(), + }, + ) } }; v.build_content = super::build_dir::get_build_content()?; - Ok(v) + Ok((cache_hit, v)) } #[derive(serde::Serialize, serde::Deserialize, Debug)] @@ -184,73 +187,93 @@ async fn incremental_build( ) -> fastn_core::Result<()> { // https://fastn.com/rfc/incremental-build/ - let mut c = dbg!(cache::get()?); + let (cache_hit, mut c) = dbg!(cache::get()?); - let mut unresolved_dependencies: Vec = vec!["index".to_string()]; + if cache_hit { + let mut unresolved_dependencies: Vec = vec!["index".to_string()]; - let mut resolved_dependencies: Vec = vec![]; + let mut resolved_dependencies: Vec = vec![]; - let mut resolving_dependencies: Vec = vec![]; + let mut resolving_dependencies: Vec = vec![]; - while let Some(unresolved_dependency) = unresolved_dependencies.pop() { - if let Some(doc) = c.documents.get( - get_dependency_name_without_package_name( - config.package.name.as_str(), - unresolved_dependency.as_str(), - ) - .as_str(), - ) { - let mut own_resolved_dependencies: Vec = vec![]; + while let Some(unresolved_dependency) = unresolved_dependencies.pop() { + if let Some(doc) = c.documents.get( + get_dependency_name_without_package_name( + config.package.name.as_str(), + unresolved_dependency.as_str(), + ) + .as_str(), + ) { + println!( + "[INCREMENTAL BUILD][CACHE FOUND] Processing: {}", + &unresolved_dependency + ); - for dep in &doc.dependencies { - if resolved_dependencies.contains(dep) { - own_resolved_dependencies.push(dep.to_string()); - continue; - } + let mut own_resolved_dependencies: Vec = vec![]; - unresolved_dependencies.push(dep.to_string()); - } + for dep in &doc.dependencies { + if resolved_dependencies.contains(dep) { + own_resolved_dependencies.push(dep.to_string()); + continue; + } - if own_resolved_dependencies.eq(&doc.dependencies) { - let name_with_extension = format!( - "{}.ftd", - get_dependency_name_without_package_name( - config.package.name.as_str(), - unresolved_dependency.as_str() - ) - ); + unresolved_dependencies.push(dep.to_string()); + } - for document in documents.values() { - if document.get_id().eq(name_with_extension.as_str()) - || document - .get_id_with_package() - .eq(name_with_extension.as_str()) - { - handle_file( - document, - config, - base_url, - ignore_failed, - test, - true, - Some(&mut c), + if own_resolved_dependencies.eq(&doc.dependencies) { + let name_with_extension = format!( + "{}.ftd", + get_dependency_name_without_package_name( + config.package.name.as_str(), + unresolved_dependency.as_str() ) - .await?; - break; + ); + + for document in documents.values() { + if document.get_id().eq(name_with_extension.as_str()) + || document + .get_id_with_package() + .eq(name_with_extension.as_str()) + { + handle_file( + document, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; + break; + } } - } - resolved_dependencies.push(unresolved_dependency.to_string()); - if unresolved_dependencies.is_empty() { - if let Some(resolving_dependency) = resolving_dependencies.pop() { - unresolved_dependencies.push(resolving_dependency); + resolved_dependencies.push(unresolved_dependency.to_string()); + if unresolved_dependencies.is_empty() { + if let Some(resolving_dependency) = resolving_dependencies.pop() { + unresolved_dependencies.push(resolving_dependency); + } } + } else { + resolving_dependencies.push(unresolved_dependency.to_string()); } } else { - resolving_dependencies.push(unresolved_dependency.to_string()); + resolved_dependencies.push(unresolved_dependency); } - } else { - resolved_dependencies.push(unresolved_dependency); + } + } else { + for document in documents.values() { + handle_file( + document, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; } } @@ -345,7 +368,7 @@ fn is_cached<'a>( // if it exists, check if the checksums match // if they do, return - // dbg!(&cached_doc); + dbg!(&cached_doc); let doc_hash = match cache.build_content.get(file_path) { Some(doc_hash) => doc_hash, None => { @@ -354,7 +377,7 @@ fn is_cached<'a>( } }; - // dbg!(doc_hash); + dbg!(doc_hash); if doc_hash != &cached_doc.html_checksum { println!("cache miss: html file checksums don't match"); From 87fab7f31acf73bb8ba51822afca84113ece2d65 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 24 Aug 2023 17:51:39 +0530 Subject: [PATCH 28/63] minor fixes --- fastn-core/src/commands/build.rs | 62 +++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index d107c264a3..739d8ee9d8 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -197,6 +197,14 @@ async fn incremental_build( let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { + let name_with_extension = format!( + "{}.ftd", + get_dependency_name_without_package_name( + config.package.name.as_str(), + unresolved_dependency.as_str() + ) + ); + if let Some(doc) = c.documents.get( get_dependency_name_without_package_name( config.package.name.as_str(), @@ -221,14 +229,6 @@ async fn incremental_build( } if own_resolved_dependencies.eq(&doc.dependencies) { - let name_with_extension = format!( - "{}.ftd", - get_dependency_name_without_package_name( - config.package.name.as_str(), - unresolved_dependency.as_str() - ) - ); - for document in documents.values() { if document.get_id().eq(name_with_extension.as_str()) || document @@ -259,6 +259,52 @@ async fn incremental_build( resolving_dependencies.push(unresolved_dependency.to_string()); } } else { + if unresolved_dependency.starts_with("$fastn$/") + || unresolved_dependency.ends_with("/-/fonts.ftd") + || unresolved_dependency.ends_with("/-/assets.ftd") + { + println!( + "[INCREMENTAL BUILD][NEW][VIRTUAL] Processing: {}", + &unresolved_dependency + ); + resolved_dependencies.push(unresolved_dependency); + continue; + } + + println!( + "[INCREMENTAL BUILD][NEW] Processing: {}", + &unresolved_dependency + ); + for document in documents.values() { + if document.get_id().eq(name_with_extension.as_str()) + || document + .get_id_with_package() + .eq(&name_with_extension.as_str()) + { + println!( + "[INCREMENTAL BUILD][NEW][FOUND] Processing: {}", + &unresolved_dependency + ); + handle_file( + document, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; + break; + } + } + + println!( + "[INCREMENTAL BUILD][NEW][NOT FOUND] Processing: {} {}", + &unresolved_dependency.to_string(), + &name_with_extension + ); + resolved_dependencies.push(unresolved_dependency); } } From f78aa0675174e1344854c35f494e4aa2ec6d7b6b Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 24 Aug 2023 19:58:48 +0530 Subject: [PATCH 29/63] removed unneccessary code --- fastn-core/src/commands/build.rs | 62 +++++--------------------------- 1 file changed, 8 insertions(+), 54 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 0b75b5cb1f..bdbfbd35ff 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -204,14 +204,6 @@ async fn incremental_build( let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { - let name_with_extension = format!( - "{}.ftd", - get_dependency_name_without_package_name( - config.package.name.as_str(), - unresolved_dependency.as_str() - ) - ); - if let Some(doc) = c.documents.get( get_dependency_name_without_package_name( config.package.name.as_str(), @@ -236,6 +228,14 @@ async fn incremental_build( } if own_resolved_dependencies.eq(&doc.dependencies) { + let name_with_extension = format!( + "{}.ftd", + get_dependency_name_without_package_name( + config.package.name.as_str(), + unresolved_dependency.as_str() + ) + ); + for document in documents.values() { if document.get_id().eq(name_with_extension.as_str()) || document @@ -266,52 +266,6 @@ async fn incremental_build( resolving_dependencies.push(unresolved_dependency.to_string()); } } else { - if unresolved_dependency.starts_with("$fastn$/") - || unresolved_dependency.ends_with("/-/fonts.ftd") - || unresolved_dependency.ends_with("/-/assets.ftd") - { - println!( - "[INCREMENTAL BUILD][NEW][VIRTUAL] Processing: {}", - &unresolved_dependency - ); - resolved_dependencies.push(unresolved_dependency); - continue; - } - - println!( - "[INCREMENTAL BUILD][NEW] Processing: {}", - &unresolved_dependency - ); - for document in documents.values() { - if document.get_id().eq(name_with_extension.as_str()) - || document - .get_id_with_package() - .eq(&name_with_extension.as_str()) - { - println!( - "[INCREMENTAL BUILD][NEW][FOUND] Processing: {}", - &unresolved_dependency - ); - handle_file( - document, - config, - base_url, - ignore_failed, - test, - true, - Some(&mut c), - ) - .await?; - break; - } - } - - println!( - "[INCREMENTAL BUILD][NEW][NOT FOUND] Processing: {} {}", - &unresolved_dependency.to_string(), - &name_with_extension - ); - resolved_dependencies.push(unresolved_dependency); } } From 3e92d545368dd218b5fd228b9d89f572af6d74a3 Mon Sep 17 00:00:00 2001 From: heulitig Date: Fri, 25 Aug 2023 10:30:49 +0530 Subject: [PATCH 30/63] build path write fix for images --- fastn-core/src/commands/build.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 0b75b5cb1f..6b8aef76b1 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -198,9 +198,7 @@ async fn incremental_build( if cache_hit { let mut unresolved_dependencies: Vec = vec!["index".to_string()]; - let mut resolved_dependencies: Vec = vec![]; - let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { @@ -729,8 +727,18 @@ async fn process_static( .join("-") .join(package.name.as_str()); - std::fs::create_dir_all(&build_path)?; - std::fs::write(build_path.join(sa.id.as_str()), &sa.content)?; + let full_file_path = build_path.join(sa.id.as_str()); + let (file_root, _file_name) = + if let Some((file_root, file_name)) = full_file_path.as_str().rsplit_once('/') { + (file_root.to_string(), file_name.to_string()) + } else { + ("".to_string(), full_file_path.to_string()) + }; + + if !base_path.join(&file_root).exists() { + std::fs::create_dir_all(base_path.join(&file_root))?; + } + std::fs::write(full_file_path, &sa.content)?; Ok(()) } } From 65352e6ecd9ff5dd0cc78244ee6bb00e009fbdb6 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Fri, 25 Aug 2023 11:45:44 +0530 Subject: [PATCH 31/63] fixed unresolved_dependencies files index --- fastn-core/src/commands/build.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 4080736e18..fb7d53b73e 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -1,3 +1,5 @@ +use itertools::Itertools; + // #[tracing::instrument(skip(config))] pub async fn build( config: &mut fastn_core::Config, @@ -197,11 +199,21 @@ async fn incremental_build( let (cache_hit, mut c) = dbg!(cache::get()?); if cache_hit { - let mut unresolved_dependencies: Vec = vec!["index".to_string()]; + let mut unresolved_dependencies: Vec = documents + .iter() + .filter(|(_, f)| f.is_ftd()) + .map(|(_, f)| f.get_id().trim_end_matches(".ftd").to_string()) + .collect_vec(); let mut resolved_dependencies: Vec = vec![]; let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { + dbg!( + &unresolved_dependencies, + &resolving_dependencies, + &resolved_dependencies + ); + if let Some(doc) = c.documents.get( get_dependency_name_without_package_name( config.package.name.as_str(), From 3366cf388d42df3e766ad55c119c78acd8d655db Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Fri, 25 Aug 2023 14:01:20 +0530 Subject: [PATCH 32/63] testing stuff --- fastn-core/src/commands/build.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index fb7d53b73e..397962f486 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -237,6 +237,8 @@ async fn incremental_build( unresolved_dependencies.push(dep.to_string()); } + dbg!( &own_resolved_dependencies, &doc.dependencies); + if own_resolved_dependencies.eq(&doc.dependencies) { let name_with_extension = format!( "{}.ftd", @@ -269,6 +271,10 @@ async fn incremental_build( resolved_dependencies.push(unresolved_dependency.to_string()); if unresolved_dependencies.is_empty() { if let Some(resolving_dependency) = resolving_dependencies.pop() { + dbg!(&unresolved_dependency); + if resolving_dependency.ne(&unresolved_dependency.as_str()) { + break; + } unresolved_dependencies.push(resolving_dependency); } } @@ -281,13 +287,15 @@ async fn incremental_build( } } else { for document in documents.values() { + dbg!(&document.get_id()); + handle_file( document, config, base_url, ignore_failed, test, - true, + false, Some(&mut c), ) .await?; @@ -488,12 +496,14 @@ async fn handle_file_( return Ok(()); } - fastn_core::utils::copy( + println!("[HF][CACHED]{}", document.get_id()); + + dbg!(fastn_core::utils::copy( config.root.join(doc.id.as_str()), config.root.join(".build").join(doc.id.as_str()), ) .await - .ok(); + .ok()); if doc.id.eq("FASTN.ftd") { return Ok(()); From 3391285c51295800c52c1f34f126968ccb92803e Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Fri, 25 Aug 2023 14:27:02 +0530 Subject: [PATCH 33/63] testing stuff 2 --- fastn-core/src/commands/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 397962f486..9556375047 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -272,8 +272,8 @@ async fn incremental_build( if unresolved_dependencies.is_empty() { if let Some(resolving_dependency) = resolving_dependencies.pop() { dbg!(&unresolved_dependency); - if resolving_dependency.ne(&unresolved_dependency.as_str()) { - break; + if resolving_dependency.eq(&unresolved_dependency.as_str()) { + continue; } unresolved_dependencies.push(resolving_dependency); } From 56bcb400ccd89870e923e3ea06a79b399b663629 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Fri, 25 Aug 2023 14:31:49 +0530 Subject: [PATCH 34/63] Minor fix --- fastn-core/src/commands/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 9556375047..1129e49e9f 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -237,7 +237,7 @@ async fn incremental_build( unresolved_dependencies.push(dep.to_string()); } - dbg!( &own_resolved_dependencies, &doc.dependencies); + dbg!(&own_resolved_dependencies, &doc.dependencies); if own_resolved_dependencies.eq(&doc.dependencies) { let name_with_extension = format!( From 2e8cdcbdcbd1501987dfbd0305f32c2dcad89190 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Fri, 25 Aug 2023 14:55:40 +0530 Subject: [PATCH 35/63] Log status for testing --- fastn-core/src/commands/build.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 1129e49e9f..f83f7f9754 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -208,12 +208,6 @@ async fn incremental_build( let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { - dbg!( - &unresolved_dependencies, - &resolving_dependencies, - &resolved_dependencies - ); - if let Some(doc) = c.documents.get( get_dependency_name_without_package_name( config.package.name.as_str(), @@ -237,7 +231,13 @@ async fn incremental_build( unresolved_dependencies.push(dep.to_string()); } - dbg!(&own_resolved_dependencies, &doc.dependencies); + println!( + "[INCREMENTAL] [R]: {} [RV]: {} [UR]: {} [RM]: {}", + &resolved_dependencies.len(), + &resolving_dependencies.len(), + &unresolved_dependencies.len(), + unresolved_dependencies.len() - resolved_dependencies.len() + ); if own_resolved_dependencies.eq(&doc.dependencies) { let name_with_extension = format!( @@ -273,12 +273,13 @@ async fn incremental_build( if let Some(resolving_dependency) = resolving_dependencies.pop() { dbg!(&unresolved_dependency); if resolving_dependency.eq(&unresolved_dependency.as_str()) { + println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); continue; } unresolved_dependencies.push(resolving_dependency); } } - } else { + } else if !resolving_dependencies.contains(&unresolved_dependency.to_string()) { resolving_dependencies.push(unresolved_dependency.to_string()); } } else { @@ -404,7 +405,7 @@ fn is_cached<'a>( // if it exists, check if the checksums match // if they do, return - dbg!(&cached_doc); + // dbg!(&cached_doc); let doc_hash = match cache.build_content.get(file_path) { Some(doc_hash) => doc_hash, None => { From 7dce18840a07df597e9535b0e0f1ccb4bdfadbd1 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Fri, 25 Aug 2023 15:01:22 +0530 Subject: [PATCH 36/63] check for circular dependency --- fastn-core/src/commands/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index f83f7f9754..9097f14fbb 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -223,7 +223,7 @@ async fn incremental_build( let mut own_resolved_dependencies: Vec = vec![]; for dep in &doc.dependencies { - if resolved_dependencies.contains(dep) { + if resolved_dependencies.contains(dep) || dep.eq(&unresolved_dependency) { own_resolved_dependencies.push(dep.to_string()); continue; } From 45d12b167aff2a2068ba91d2ee158adb281735b0 Mon Sep 17 00:00:00 2001 From: heulitig Date: Fri, 25 Aug 2023 16:20:30 +0530 Subject: [PATCH 37/63] removed all bugs in incremental build --- fastn-core/src/commands/build.rs | 113 +++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 36 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 9097f14fbb..caad8b9c98 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -196,7 +196,7 @@ async fn incremental_build( ) -> fastn_core::Result<()> { // https://fastn.com/rfc/incremental-build/ - let (cache_hit, mut c) = dbg!(cache::get()?); + let (cache_hit, mut c) = cache::get()?; if cache_hit { let mut unresolved_dependencies: Vec = documents @@ -208,6 +208,7 @@ async fn incremental_build( let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { + // println!("Current UR: {}", unresolved_dependency.as_str()); if let Some(doc) = c.documents.get( get_dependency_name_without_package_name( config.package.name.as_str(), @@ -215,29 +216,27 @@ async fn incremental_build( ) .as_str(), ) { - println!( - "[INCREMENTAL BUILD][CACHE FOUND] Processing: {}", - &unresolved_dependency - ); + // println!( + // "[INCREMENTAL BUILD][CACHE FOUND] Processing: {}", + // &unresolved_dependency + // ); let mut own_resolved_dependencies: Vec = vec![]; - for dep in &doc.dependencies { if resolved_dependencies.contains(dep) || dep.eq(&unresolved_dependency) { own_resolved_dependencies.push(dep.to_string()); continue; } - unresolved_dependencies.push(dep.to_string()); } - println!( - "[INCREMENTAL] [R]: {} [RV]: {} [UR]: {} [RM]: {}", - &resolved_dependencies.len(), - &resolving_dependencies.len(), - &unresolved_dependencies.len(), - unresolved_dependencies.len() - resolved_dependencies.len() - ); + // println!( + // "[INCREMENTAL] [R]: {} [RV]: {} [UR]: {} [ORD]: {}", + // &resolved_dependencies.len(), + // &resolving_dependencies.len(), + // &unresolved_dependencies.len(), + // own_resolved_dependencies.len(), + // ); if own_resolved_dependencies.eq(&doc.dependencies) { let name_with_extension = format!( @@ -271,25 +270,67 @@ async fn incremental_build( resolved_dependencies.push(unresolved_dependency.to_string()); if unresolved_dependencies.is_empty() { if let Some(resolving_dependency) = resolving_dependencies.pop() { - dbg!(&unresolved_dependency); if resolving_dependency.eq(&unresolved_dependency.as_str()) { - println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); + // println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); continue; } unresolved_dependencies.push(resolving_dependency); } } - } else if !resolving_dependencies.contains(&unresolved_dependency.to_string()) { + } else { + // println!("Adding to RD: {}", unresolved_dependency.as_str()); resolving_dependencies.push(unresolved_dependency.to_string()); } } else { - resolved_dependencies.push(unresolved_dependency); + if unresolved_dependency.starts_with("$fastn$/") + || unresolved_dependency.ends_with("/-/fonts.ftd") + || unresolved_dependency.ends_with("/-/assets.ftd") + { + resolved_dependencies.push(unresolved_dependency.clone()); + } else { + // println!("Not found in cache UR: {}", unresolved_dependency.as_str()); + let name_with_extension = format!( + "{}.ftd", + get_dependency_name_without_package_name( + config.package.name.as_str(), + unresolved_dependency.as_str() + ) + ); + + for document in documents.values() { + if document.get_id().eq(name_with_extension.as_str()) + || document + .get_id_with_package() + .eq(name_with_extension.as_str()) + { + handle_file( + document, + config, + base_url, + ignore_failed, + test, + true, + Some(&mut c), + ) + .await?; + break; + } + } + resolved_dependencies.push(unresolved_dependency.clone()); + } + if unresolved_dependencies.is_empty() { + if let Some(resolving_dependency) = resolving_dependencies.pop() { + if resolving_dependency.eq(&unresolved_dependency.as_str()) { + // println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); + continue; + } + unresolved_dependencies.push(resolving_dependency); + } + } } } } else { for document in documents.values() { - dbg!(&document.get_id()); - handle_file( document, config, @@ -305,7 +346,7 @@ async fn incremental_build( // TODO: Handle deleted files (files present in cache/.build but not in documents) - dbg!(c).cache_it()?; + c.cache_it()?; Ok(()) } @@ -388,7 +429,7 @@ fn is_cached<'a>( let cache: &mut cache::Cache = match cache { Some(c) => c, None => { - println!("cache miss: no have cache"); + // println!("cache miss: no have cache"); return (cache, false); } }; @@ -398,7 +439,7 @@ fn is_cached<'a>( let cached_doc: cache::Document = match cache.documents.get(id.as_str()).cloned() { Some(cached_doc) => cached_doc, None => { - println!("cache miss: no cache entry for {}", id.as_str()); + // println!("cache miss: no cache entry for {}", id.as_str()); return (Some(cache), false); } }; @@ -409,35 +450,35 @@ fn is_cached<'a>( let doc_hash = match cache.build_content.get(file_path) { Some(doc_hash) => doc_hash, None => { - println!("cache miss: document not present in .build: {}", file_path); + // println!("cache miss: document not present in .build: {}", file_path); return (Some(cache), false); } }; - dbg!(doc_hash); + // dbg!(doc_hash); if doc_hash != &cached_doc.html_checksum { - println!("cache miss: html file checksums don't match"); + // println!("cache miss: html file checksums don't match"); return (Some(cache), false); } let file_checksum = match cache.file_checksum.get(id.as_str()).cloned() { Some(file_checksum) => file_checksum, None => { - println!("cache miss: no cache entry for {}", id.as_str()); + // println!("cache miss: no cache entry for {}", id.as_str()); return (Some(cache), false); } }; if file_checksum != fastn_core::utils::generate_hash(doc.content.as_str()) { - println!("cache miss: ftd file checksums don't match"); + // println!("cache miss: ftd file checksums don't match"); return (Some(cache), false); } for dep in &cached_doc.dependencies { let file_checksum = match cache.file_checksum.get(dep) { None => { - println!("cache miss: file {} not present in cache", dep); + // println!("cache miss: file {} not present in cache", dep); return (Some(cache), false); } Some(file_checksum) => file_checksum.clone(), @@ -446,18 +487,18 @@ fn is_cached<'a>( let current_hash = match cache.get_file_hash(dep.as_str()) { Ok(hash) => hash, Err(_) => { - println!("cache miss: dependency {} not present current folder", dep); + // println!("cache miss: dependency {} not present current folder", dep); return (Some(cache), false); } }; if file_checksum != current_hash { - println!("cache miss: dependency {} checksums don't match", dep); + // println!("cache miss: dependency {} checksums don't match", dep); return (Some(cache), false); } } - println!("cache hit"); + // println!("cache hit"); (Some(cache), true) } @@ -499,12 +540,12 @@ async fn handle_file_( println!("[HF][CACHED]{}", document.get_id()); - dbg!(fastn_core::utils::copy( + fastn_core::utils::copy( config.root.join(doc.id.as_str()), config.root.join(".build").join(doc.id.as_str()), ) .await - .ok()); + .ok(); if doc.id.eq("FASTN.ftd") { return Ok(()); @@ -523,9 +564,9 @@ async fn handle_file_( (Ok(r), _) => { if let Some(cache) = cache { cache.documents.insert( - dbg!(remove_extension(doc.id.as_str())), + remove_extension(doc.id.as_str()), cache::Document { - html_checksum: dbg!(r.checksum()), + html_checksum: r.checksum(), dependencies: config.dependencies_during_render.clone(), }, ); From dbbe9da47506ec3385ea96ca81520a6e0ac62f12 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Fri, 25 Aug 2023 16:35:46 +0530 Subject: [PATCH 38/63] refactored code --- fastn-core/src/commands/build.rs | 90 ++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index caad8b9c98..a279955844 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -186,6 +186,37 @@ fn get_dependency_name_without_package_name(package_name: &str, dependency_name: .to_string() } +async fn handle_dependency_file( + config: &mut fastn_core::Config, + cache: &mut cache::Cache, + documents: &std::collections::BTreeMap, + base_url: &str, + ignore_failed: bool, + test: bool, + name_with_extension: String, +) -> fastn_core::Result<()> { + for document in documents.values() { + if document.get_id().eq(name_with_extension.as_str()) + || document + .get_id_with_package() + .eq(name_with_extension.as_str()) + { + handle_file( + document, + config, + base_url, + ignore_failed, + test, + true, + Some(cache), + ) + .await?; + } + } + + Ok(()) +} + #[tracing::instrument(skip(config, documents))] async fn incremental_build( config: &mut fastn_core::Config, @@ -247,25 +278,16 @@ async fn incremental_build( ) ); - for document in documents.values() { - if document.get_id().eq(name_with_extension.as_str()) - || document - .get_id_with_package() - .eq(name_with_extension.as_str()) - { - handle_file( - document, - config, - base_url, - ignore_failed, - test, - true, - Some(&mut c), - ) - .await?; - break; - } - } + handle_dependency_file( + config, + &mut c, + documents, + base_url, + ignore_failed, + test, + name_with_extension, + ) + .await?; resolved_dependencies.push(unresolved_dependency.to_string()); if unresolved_dependencies.is_empty() { @@ -297,25 +319,17 @@ async fn incremental_build( ) ); - for document in documents.values() { - if document.get_id().eq(name_with_extension.as_str()) - || document - .get_id_with_package() - .eq(name_with_extension.as_str()) - { - handle_file( - document, - config, - base_url, - ignore_failed, - test, - true, - Some(&mut c), - ) - .await?; - break; - } - } + handle_dependency_file( + config, + &mut c, + documents, + base_url, + ignore_failed, + test, + name_with_extension, + ) + .await?; + resolved_dependencies.push(unresolved_dependency.clone()); } if unresolved_dependencies.is_empty() { From 4e0323e5e8051efc851bc5bf25aeea5e1bbe9485 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Fri, 25 Aug 2023 19:51:46 +0530 Subject: [PATCH 39/63] Minor fixes --- fastn-core/src/commands/build.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index a279955844..99d727d6a8 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -552,8 +552,6 @@ async fn handle_file_( return Ok(()); } - println!("[HF][CACHED]{}", document.get_id()); - fastn_core::utils::copy( config.root.join(doc.id.as_str()), config.root.join(".build").join(doc.id.as_str()), From bb602e9536e119814548a220b6f49189446cc22f Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Mon, 28 Aug 2023 11:57:57 +0530 Subject: [PATCH 40/63] build assets --- fastn-core/src/commands/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 99d727d6a8..7d80d084bd 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -351,7 +351,7 @@ async fn incremental_build( base_url, ignore_failed, test, - false, + true, Some(&mut c), ) .await?; From 1f6c4412c05c61b61009c5ca6a0740981a4f8751 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Fri, 1 Sep 2023 14:16:08 +0530 Subject: [PATCH 41/63] handle deleted documents --- fastn-core/src/commands/build.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 7d80d084bd..6356d4fa50 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -233,7 +233,7 @@ async fn incremental_build( let mut unresolved_dependencies: Vec = documents .iter() .filter(|(_, f)| f.is_ftd()) - .map(|(_, f)| f.get_id().trim_end_matches(".ftd").to_string()) + .map(|(_, f)| remove_extension(f.get_id())) .collect_vec(); let mut resolved_dependencies: Vec = vec![]; let mut resolving_dependencies: Vec = vec![]; @@ -358,7 +358,26 @@ async fn incremental_build( } } - // TODO: Handle deleted files (files present in cache/.build but not in documents) + let mut removed_documents: Vec = vec![]; + + for cached_document_id in c.documents.keys() { + if !documents.contains_key(format!("{}.ftd", &cached_document_id).as_str()) { + let folder_path = format!("{}/{}", config.build_dir(), &cached_document_id); + let file_path = format!("{}.ftd", &folder_path); + + // println!("Removing file: {}", &file_path); + std::fs::remove_file(file_path)?; + + // println!("Removing dir: {}", &folder_path); + std::fs::remove_dir_all(folder_path)?; + + removed_documents.push(cached_document_id.to_string()); + } + } + + for removed_doc_id in &removed_documents { + c.documents.remove(removed_doc_id); + } c.cache_it()?; From 992a4ebe62472f61d32f940340b99eef57625b3c Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Fri, 1 Sep 2023 15:01:03 +0530 Subject: [PATCH 42/63] minor fixes --- Cargo.lock | 2 +- fastn-core/src/commands/build.rs | 42 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e307ed9c40..5271979e39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1519,7 +1519,7 @@ dependencies = [ [[package]] name = "fastn" -version = "0.3.36" +version = "0.3.37" dependencies = [ "clap", "colored", diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 6356d4fa50..7d6a6b1a60 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -343,6 +343,27 @@ async fn incremental_build( } } } + + let mut removed_documents: Vec = vec![]; + + for cached_document_id in c.documents.keys() { + if !documents.contains_key(format!("{}.ftd", &cached_document_id).as_str()) { + let folder_path = format!("{}/{}", config.build_dir(), &cached_document_id); + let file_path = format!("{}.ftd", &folder_path); + + // println!("Removing file: {}", &file_path); + std::fs::remove_file(file_path)?; + + // println!("Removing dir: {}", &folder_path); + std::fs::remove_dir_all(folder_path)?; + + removed_documents.push(cached_document_id.to_string()); + } + } + + for removed_doc_id in &removed_documents { + c.documents.remove(removed_doc_id); + } } else { for document in documents.values() { handle_file( @@ -358,27 +379,6 @@ async fn incremental_build( } } - let mut removed_documents: Vec = vec![]; - - for cached_document_id in c.documents.keys() { - if !documents.contains_key(format!("{}.ftd", &cached_document_id).as_str()) { - let folder_path = format!("{}/{}", config.build_dir(), &cached_document_id); - let file_path = format!("{}.ftd", &folder_path); - - // println!("Removing file: {}", &file_path); - std::fs::remove_file(file_path)?; - - // println!("Removing dir: {}", &folder_path); - std::fs::remove_dir_all(folder_path)?; - - removed_documents.push(cached_document_id.to_string()); - } - } - - for removed_doc_id in &removed_documents { - c.documents.remove(removed_doc_id); - } - c.cache_it()?; Ok(()) From d03457cc0a98bc9af366521634c393bcf11d49c2 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Tue, 5 Sep 2023 02:11:56 +0530 Subject: [PATCH 43/63] proper document name check --- fastn-core/src/commands/build.rs | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 7d6a6b1a60..7af7f09ffe 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -193,13 +193,12 @@ async fn handle_dependency_file( base_url: &str, ignore_failed: bool, test: bool, - name_with_extension: String, + name_without_package_name: String, ) -> fastn_core::Result<()> { for document in documents.values() { - if document.get_id().eq(name_with_extension.as_str()) - || document - .get_id_with_package() - .eq(name_with_extension.as_str()) + if remove_extension(document.get_id()).eq(name_without_package_name.as_str()) + || remove_extension(&document.get_id_with_package()) + .eq(name_without_package_name.as_str()) { handle_file( document, @@ -239,7 +238,7 @@ async fn incremental_build( let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { - // println!("Current UR: {}", unresolved_dependency.as_str()); + println!("Current UR: {}", unresolved_dependency.as_str()); if let Some(doc) = c.documents.get( get_dependency_name_without_package_name( config.package.name.as_str(), @@ -270,13 +269,11 @@ async fn incremental_build( // ); if own_resolved_dependencies.eq(&doc.dependencies) { - let name_with_extension = format!( - "{}.ftd", + let name_without_package_name: String = get_dependency_name_without_package_name( config.package.name.as_str(), - unresolved_dependency.as_str() - ) - ); + unresolved_dependency.as_str(), + ); handle_dependency_file( config, @@ -285,7 +282,7 @@ async fn incremental_build( base_url, ignore_failed, test, - name_with_extension, + name_without_package_name, ) .await?; @@ -311,12 +308,9 @@ async fn incremental_build( resolved_dependencies.push(unresolved_dependency.clone()); } else { // println!("Not found in cache UR: {}", unresolved_dependency.as_str()); - let name_with_extension = format!( - "{}.ftd", - get_dependency_name_without_package_name( - config.package.name.as_str(), - unresolved_dependency.as_str() - ) + let name_without_package_name = get_dependency_name_without_package_name( + config.package.name.as_str(), + unresolved_dependency.as_str(), ); handle_dependency_file( @@ -326,7 +320,7 @@ async fn incremental_build( base_url, ignore_failed, test, - name_with_extension, + name_without_package_name, ) .await?; From d881590ce07fd8b3134ec6d3280f98689c541f04 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Tue, 5 Sep 2023 12:28:29 +0530 Subject: [PATCH 44/63] Remove deleted docs from build --- fastn-core/src/commands/build.rs | 39 +++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 7af7f09ffe..a73040a742 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -338,24 +338,37 @@ async fn incremental_build( } } - let mut removed_documents: Vec = vec![]; + let removed_documents = c + .documents + .keys() + .filter(|cached_document_id| { + let name_without_package_name = get_dependency_name_without_package_name( + config.package.name.as_str(), + cached_document_id.as_str(), + ); + + for document in documents.values() { + if remove_extension(document.get_id()).eq(name_without_package_name.as_str()) + || remove_extension(&document.get_id_with_package()) + .eq(name_without_package_name.as_str()) + { + return false; + } + } - for cached_document_id in c.documents.keys() { - if !documents.contains_key(format!("{}.ftd", &cached_document_id).as_str()) { - let folder_path = format!("{}/{}", config.build_dir(), &cached_document_id); - let file_path = format!("{}.ftd", &folder_path); + true + }) + .map(|id| id.to_string()) + .collect_vec(); - // println!("Removing file: {}", &file_path); - std::fs::remove_file(file_path)?; + for removed_doc_id in &removed_documents { + let folder_path = format!("{}/{}", config.build_dir(), &removed_doc_id); + let file_path = format!("{}.ftd", &folder_path); - // println!("Removing dir: {}", &folder_path); - std::fs::remove_dir_all(folder_path)?; + std::fs::remove_file(file_path)?; - removed_documents.push(cached_document_id.to_string()); - } - } + std::fs::remove_dir_all(folder_path)?; - for removed_doc_id in &removed_documents { c.documents.remove(removed_doc_id); } } else { From d4b7f240b4b8a95f96295ea8ebea398c5271d3f4 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Tue, 5 Sep 2023 13:07:56 +0530 Subject: [PATCH 45/63] file exist check before deleting --- fastn-core/src/commands/build.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index a73040a742..77f74d94f4 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -238,7 +238,7 @@ async fn incremental_build( let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { - println!("Current UR: {}", unresolved_dependency.as_str()); + // println!("Current UR: {}", unresolved_dependency.as_str()); if let Some(doc) = c.documents.get( get_dependency_name_without_package_name( config.package.name.as_str(), @@ -363,9 +363,12 @@ async fn incremental_build( for removed_doc_id in &removed_documents { let folder_path = format!("{}/{}", config.build_dir(), &removed_doc_id); - let file_path = format!("{}.ftd", &folder_path); + let file_path_str = format!("{}.ftd", &folder_path); + let file_path = std::path::Path::new(&file_path_str); - std::fs::remove_file(file_path)?; + if file_path.exists() { + std::fs::remove_file(file_path)?; + } std::fs::remove_dir_all(folder_path)?; From 6a3d46aaf7d393811f34e1e1d6873e6745f8713d Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Tue, 5 Sep 2023 15:08:16 +0530 Subject: [PATCH 46/63] better path construction --- fastn-core/src/commands/build.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 77f74d94f4..4209aa4377 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -362,9 +362,8 @@ async fn incremental_build( .collect_vec(); for removed_doc_id in &removed_documents { - let folder_path = format!("{}/{}", config.build_dir(), &removed_doc_id); - let file_path_str = format!("{}.ftd", &folder_path); - let file_path = std::path::Path::new(&file_path_str); + let folder_path = config.build_dir().join( &removed_doc_id); + let file_path = &folder_path.with_extension("ftd"); if file_path.exists() { std::fs::remove_file(file_path)?; From 546e5542c70253942ae99e445b06e7459f9de30f Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Tue, 5 Sep 2023 15:09:32 +0530 Subject: [PATCH 47/63] clippy fixes --- fastn-core/src/commands/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 4209aa4377..fb1db57ddb 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -362,7 +362,7 @@ async fn incremental_build( .collect_vec(); for removed_doc_id in &removed_documents { - let folder_path = config.build_dir().join( &removed_doc_id); + let folder_path = config.build_dir().join(removed_doc_id); let file_path = &folder_path.with_extension("ftd"); if file_path.exists() { From 27d301053933211be78f726d05d4d4cd54fbfc0a Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Wed, 6 Sep 2023 12:41:34 +0530 Subject: [PATCH 48/63] Merged main --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 4cc442d2b0..14a6b848b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1519,7 +1519,7 @@ dependencies = [ [[package]] name = "fastn" -version = "0.3.38" +version = "0.3.40" dependencies = [ "clap", "colored", From e43341ca77756067e37d328b48c4587af4fc1e98 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Wed, 6 Sep 2023 13:39:15 +0530 Subject: [PATCH 49/63] remove package name from dependencies --- fastn-core/src/commands/build.rs | 57 ++++++++++++-------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index edb7676573..b672a1fb83 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -243,43 +243,32 @@ async fn incremental_build( let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { - // println!("Current UR: {}", unresolved_dependency.as_str()); - if let Some(doc) = c.documents.get( - get_dependency_name_without_package_name( - config.package.name.as_str(), - unresolved_dependency.as_str(), - ) - .as_str(), - ) { - // println!( - // "[INCREMENTAL BUILD][CACHE FOUND] Processing: {}", - // &unresolved_dependency - // ); + println!("Current UR: {}", unresolved_dependency.as_str()); + if let Some(doc) = c.documents.get(unresolved_dependency.as_str()) { + println!( + "[INCREMENTAL BUILD][CACHE FOUND] Processing: {}", + &unresolved_dependency + ); let mut own_resolved_dependencies: Vec = vec![]; for dep in &doc.dependencies { - if resolved_dependencies.contains(dep) || dep.eq(&unresolved_dependency) { + let dep = get_dependency_name_without_package_name(&config.package.name, dep); + if resolved_dependencies.contains(&dep) || dep.eq(&unresolved_dependency) { own_resolved_dependencies.push(dep.to_string()); continue; } unresolved_dependencies.push(dep.to_string()); } - // println!( - // "[INCREMENTAL] [R]: {} [RV]: {} [UR]: {} [ORD]: {}", - // &resolved_dependencies.len(), - // &resolving_dependencies.len(), - // &unresolved_dependencies.len(), - // own_resolved_dependencies.len(), - // ); + println!( + "[INCREMENTAL] [R]: {} [RV]: {} [UR]: {} [ORD]: {}", + &resolved_dependencies.len(), + &resolving_dependencies.len(), + &unresolved_dependencies.len(), + own_resolved_dependencies.len(), + ); if own_resolved_dependencies.eq(&doc.dependencies) { - let name_without_package_name: String = - get_dependency_name_without_package_name( - config.package.name.as_str(), - unresolved_dependency.as_str(), - ); - handle_dependency_file( config, &mut c, @@ -287,7 +276,7 @@ async fn incremental_build( base_url, ignore_failed, test, - name_without_package_name, + unresolved_dependency.to_string(), ) .await?; @@ -295,14 +284,14 @@ async fn incremental_build( if unresolved_dependencies.is_empty() { if let Some(resolving_dependency) = resolving_dependencies.pop() { if resolving_dependency.eq(&unresolved_dependency.as_str()) { - // println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); + println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); continue; } unresolved_dependencies.push(resolving_dependency); } } } else { - // println!("Adding to RD: {}", unresolved_dependency.as_str()); + println!("Adding to RD: {}", unresolved_dependency.as_str()); resolving_dependencies.push(unresolved_dependency.to_string()); } } else { @@ -312,11 +301,7 @@ async fn incremental_build( { resolved_dependencies.push(unresolved_dependency.clone()); } else { - // println!("Not found in cache UR: {}", unresolved_dependency.as_str()); - let name_without_package_name = get_dependency_name_without_package_name( - config.package.name.as_str(), - unresolved_dependency.as_str(), - ); + println!("Not found in cache UR: {}", unresolved_dependency.as_str()); handle_dependency_file( config, @@ -325,7 +310,7 @@ async fn incremental_build( base_url, ignore_failed, test, - name_without_package_name, + unresolved_dependency.to_string(), ) .await?; @@ -334,7 +319,7 @@ async fn incremental_build( if unresolved_dependencies.is_empty() { if let Some(resolving_dependency) = resolving_dependencies.pop() { if resolving_dependency.eq(&unresolved_dependency.as_str()) { - // println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); + println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); continue; } unresolved_dependencies.push(resolving_dependency); From 4e0856ad51b798607c04c775491882528630c784 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Wed, 6 Sep 2023 13:46:13 +0530 Subject: [PATCH 50/63] no need to remove package name from doc id --- fastn-core/src/commands/build.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index b672a1fb83..770b301ffe 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -332,15 +332,10 @@ async fn incremental_build( .documents .keys() .filter(|cached_document_id| { - let name_without_package_name = get_dependency_name_without_package_name( - config.package.name.as_str(), - cached_document_id.as_str(), - ); - for document in documents.values() { - if remove_extension(document.get_id()).eq(name_without_package_name.as_str()) + if remove_extension(document.get_id()).eq(cached_document_id.as_str()) || remove_extension(&document.get_id_with_package()) - .eq(name_without_package_name.as_str()) + .eq(cached_document_id.as_str()) { return false; } From cc2e17672c1bcb305de5863fc80ddb05fe9de50c Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Wed, 6 Sep 2023 13:56:55 +0530 Subject: [PATCH 51/63] commented out print statements --- fastn-core/src/commands/build.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 770b301ffe..ac3d1f422b 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -243,12 +243,12 @@ async fn incremental_build( let mut resolving_dependencies: Vec = vec![]; while let Some(unresolved_dependency) = unresolved_dependencies.pop() { - println!("Current UR: {}", unresolved_dependency.as_str()); + // println!("Current UR: {}", unresolved_dependency.as_str()); if let Some(doc) = c.documents.get(unresolved_dependency.as_str()) { - println!( - "[INCREMENTAL BUILD][CACHE FOUND] Processing: {}", - &unresolved_dependency - ); + // println!( + // "[INCREMENTAL BUILD][CACHE FOUND] Processing: {}", + // &unresolved_dependency + // ); let mut own_resolved_dependencies: Vec = vec![]; for dep in &doc.dependencies { @@ -260,13 +260,13 @@ async fn incremental_build( unresolved_dependencies.push(dep.to_string()); } - println!( - "[INCREMENTAL] [R]: {} [RV]: {} [UR]: {} [ORD]: {}", - &resolved_dependencies.len(), - &resolving_dependencies.len(), - &unresolved_dependencies.len(), - own_resolved_dependencies.len(), - ); + // println!( + // "[INCREMENTAL] [R]: {} [RV]: {} [UR]: {} [ORD]: {}", + // &resolved_dependencies.len(), + // &resolving_dependencies.len(), + // &unresolved_dependencies.len(), + // own_resolved_dependencies.len(), + // ); if own_resolved_dependencies.eq(&doc.dependencies) { handle_dependency_file( @@ -284,14 +284,14 @@ async fn incremental_build( if unresolved_dependencies.is_empty() { if let Some(resolving_dependency) = resolving_dependencies.pop() { if resolving_dependency.eq(&unresolved_dependency.as_str()) { - println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); + // println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); continue; } unresolved_dependencies.push(resolving_dependency); } } } else { - println!("Adding to RD: {}", unresolved_dependency.as_str()); + // println!("Adding to RD: {}", unresolved_dependency.as_str()); resolving_dependencies.push(unresolved_dependency.to_string()); } } else { @@ -301,7 +301,7 @@ async fn incremental_build( { resolved_dependencies.push(unresolved_dependency.clone()); } else { - println!("Not found in cache UR: {}", unresolved_dependency.as_str()); + // println!("Not found in cache UR: {}", unresolved_dependency.as_str()); handle_dependency_file( config, @@ -319,7 +319,7 @@ async fn incremental_build( if unresolved_dependencies.is_empty() { if let Some(resolving_dependency) = resolving_dependencies.pop() { if resolving_dependency.eq(&unresolved_dependency.as_str()) { - println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); + // println!("[INCREMENTAL][CIRCULAR]: {}", &unresolved_dependency); continue; } unresolved_dependencies.push(resolving_dependency); From 196d85e7354946cd03b00099de810ed602e670ab Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Wed, 6 Sep 2023 16:31:11 +0530 Subject: [PATCH 52/63] Fixed dependency naming bug --- fastn-core/src/commands/build.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index ac3d1f422b..c2b9f00a51 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -251,9 +251,15 @@ async fn incremental_build( // ); let mut own_resolved_dependencies: Vec = vec![]; - for dep in &doc.dependencies { - let dep = get_dependency_name_without_package_name(&config.package.name, dep); - if resolved_dependencies.contains(&dep) || dep.eq(&unresolved_dependency) { + + let dependencies: Vec = doc + .dependencies + .iter() + .map(|dep| get_dependency_name_without_package_name(&config.package.name, dep)) + .collect_vec(); + + for dep in &dependencies { + if resolved_dependencies.contains(dep) || dep.eq(&unresolved_dependency) { own_resolved_dependencies.push(dep.to_string()); continue; } @@ -268,7 +274,7 @@ async fn incremental_build( // own_resolved_dependencies.len(), // ); - if own_resolved_dependencies.eq(&doc.dependencies) { + if own_resolved_dependencies.eq(&dependencies) { handle_dependency_file( config, &mut c, From 8910e00fdc6707051c3177d434b83887121ea7ca Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 7 Sep 2023 11:35:13 +0530 Subject: [PATCH 53/63] delete empty folders from build --- fastn-core/src/commands/build.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index c2b9f00a51..33049d3dea 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -354,13 +354,21 @@ async fn incremental_build( for removed_doc_id in &removed_documents { let folder_path = config.build_dir().join(removed_doc_id); + let folder_parent = folder_path.parent(); let file_path = &folder_path.with_extension("ftd"); if file_path.exists() { std::fs::remove_file(file_path)?; } - std::fs::remove_dir_all(folder_path)?; + std::fs::remove_dir_all(&folder_path)?; + + // If the parent folder of the file's output folder is also empty, delete it as well. + if let Some(folder_parent) = folder_parent { + if folder_parent.read_dir()?.count().eq(&0) { + std::fs::remove_dir_all(folder_parent)?; + } + } c.documents.remove(removed_doc_id); } From 784d642683b2046d3355acff15b29d79616508af Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 7 Sep 2023 23:02:22 +0530 Subject: [PATCH 54/63] Path separator fix --- fastn-core/src/commands/build.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 33049d3dea..23d2791dd8 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -786,12 +786,14 @@ async fn process_static( .join(package.name.as_str()); let full_file_path = build_path.join(sa.id.as_str()); - let (file_root, _file_name) = - if let Some((file_root, file_name)) = full_file_path.as_str().rsplit_once('/') { - (file_root.to_string(), file_name.to_string()) - } else { - ("".to_string(), full_file_path.to_string()) - }; + let (file_root, _file_name) = if let Some((file_root, file_name)) = full_file_path + .as_str() + .rsplit_once(std::path::MAIN_SEPARATOR) + { + (file_root.to_string(), file_name.to_string()) + } else { + ("".to_string(), full_file_path.to_string()) + }; if !base_path.join(&file_root).exists() { std::fs::create_dir_all(base_path.join(&file_root))?; From 82e1c1336630b1892a401282f2f13009fc4fe254 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Fri, 8 Sep 2023 16:38:43 +0530 Subject: [PATCH 55/63] conditional ftd.redirect --- ftd/src/interpreter/main.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ftd/src/interpreter/main.rs b/ftd/src/interpreter/main.rs index 34cb7bc1fa..8c72928a9c 100644 --- a/ftd/src/interpreter/main.rs +++ b/ftd/src/interpreter/main.rs @@ -1102,7 +1102,17 @@ impl Document { pub fn get_redirect(&self) -> Option<(String, i32)> { let components = self.get_instructions("ftd#redirect"); - let c = match components.first() { + let c = match components.iter().find(|v| { + if let Some(expr) = &v.condition.as_ref() { + if let Ok(b) = expr.eval(&self.tdoc()) { + b + } else { + false + } + } else { + true + } + }) { Some(v) => v, None => return None, }; From 99c74385ea1bc3a16e05c7b49c0c57d1999bbe23 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Mon, 11 Sep 2023 12:49:38 +0530 Subject: [PATCH 56/63] fix windows cache dir --- Cargo.lock | 2 +- fastn-core/src/package/mod.rs | 9 +- fastn-core/src/package/redirects.rs | 51 +- fastn-core/src/utils.rs | 13 +- fastn-js/js/dom.js | 60 +- fastn-js/js/fastn.js | 2 +- fastn-js/src/property.rs | 2 + fastn-js/src/to_js.rs | 3 + fastn/Cargo.toml | 2 +- ftd/src/interpreter/things/default.rs | 6 + ftd/src/js/element.rs | 12 + ftd/t/js/01-basic-module.html | 22 +- ftd/t/js/02-property.html | 10 +- ftd/t/js/03-common-properties.html | 735 ++++++++++++++----- ftd/t/js/04-variable.html | 7 +- ftd/t/js/05-dynamic-dom-list.html | 9 +- ftd/t/js/06-dynamic-dom-list-2.html | 6 +- ftd/t/js/08-inherited.html | 37 +- ftd/t/js/09-text-properties.html | 88 ++- ftd/t/js/10-color-test.html | 26 +- ftd/t/js/12-children.html | 5 +- ftd/t/js/13-non-style-properties.html | 24 +- ftd/t/js/14-code.html | 9 +- ftd/t/js/15-function-call-in-property.html | 3 + ftd/t/js/17-clone.html | 13 +- ftd/t/js/18-rive.html | 9 +- ftd/t/js/20-background-properties.html | 43 +- ftd/t/js/23-record-list.html | 8 +- ftd/t/js/24-device.html | 3 +- ftd/t/js/29-dom-list.html | 10 +- ftd/t/js/30-web-component.html | 89 ++- ftd/t/js/31-advance-list.html | 34 +- ftd/t/js/31-ftd-len.html | 5 +- ftd/t/js/32-ftd-len.html | 32 +- ftd/t/js/33-list-indexing.html | 6 +- ftd/t/js/36-single-ui.html | 3 +- ftd/t/js/37-expander.html | 42 +- ftd/t/js/38-background-image-properties.html | 11 +- ftd/t/js/40-code-themes.html | 13 +- ftd/t/js/43-image-object-fit.html | 31 +- ftd/t/js/44-local-storage.html | 3 +- ftd/t/js/44-module.html | 7 +- ftd/t/js/45-re-module.html | 7 +- ftd/t/js/47-ftd-code-syntax.html | 9 +- ftd/t/js/49-align-content.html | 251 ++++++- ftd/t/js/50-iframe-fullscreen.html | 15 +- ftd/t/js/51-markdown-table.html | 13 +- ftd/t/js/52-events.ftd | 96 +++ ftd/t/js/52-events.html | 229 ++++++ ftd/t/js/53-link-color.ftd | 14 + ftd/t/js/53-link-color.html | 90 +++ ftd/t/js/54-class-fix.ftd | 10 + ftd/t/js/54-class-fix.html | 93 +++ 53 files changed, 1890 insertions(+), 442 deletions(-) create mode 100644 ftd/t/js/52-events.ftd create mode 100644 ftd/t/js/52-events.html create mode 100644 ftd/t/js/53-link-color.ftd create mode 100644 ftd/t/js/53-link-color.html create mode 100644 ftd/t/js/54-class-fix.ftd create mode 100644 ftd/t/js/54-class-fix.html diff --git a/Cargo.lock b/Cargo.lock index 08c28736aa..8c933a53b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1519,7 +1519,7 @@ dependencies = [ [[package]] name = "fastn" -version = "0.3.42" +version = "0.3.49" dependencies = [ "clap", "colored", diff --git a/fastn-core/src/package/mod.rs b/fastn-core/src/package/mod.rs index f8e2e75781..b1da0f15bf 100644 --- a/fastn-core/src/package/mod.rs +++ b/fastn-core/src/package/mod.rs @@ -597,7 +597,14 @@ impl Package { package.redirects = { let redirects_temp: Option = fastn_doc.get("fastn#redirects")?; - redirects_temp.map(|r| r.redirects_from_body()) + if let Some(redirects) = redirects_temp { + let result = redirects + .redirects_from_body() + .map_err(|e| fastn_core::Error::GenericError(e.to_string()))?; + Some(result) + } else { + None + } }; package.auto_import = fastn_doc diff --git a/fastn-core/src/package/redirects.rs b/fastn-core/src/package/redirects.rs index 040f5284d3..4978bd4b80 100644 --- a/fastn-core/src/package/redirects.rs +++ b/fastn-core/src/package/redirects.rs @@ -5,19 +5,62 @@ pub struct RedirectsTemp { } impl RedirectsTemp { - pub(crate) fn redirects_from_body(&self) -> ftd::Map { + pub(crate) fn redirects_from_body(&self) -> fastn_core::Result> { let body = self.body.as_str(); let mut redirects: ftd::Map = ftd::Map::new(); for line in body.lines() { - if line.trim_start().starts_with(';') { + if line.is_empty() || line.trim_start().starts_with(';') { + continue; + } + // Supported Redirects Syntax under fastn.redirects + // : + // -> + + if let Some((key, value)) = line.split_once("->") { + Self::assert_and_insert_redirect(key, value, &mut redirects)?; continue; } if let Some((key, value)) = line.split_once(':') { - redirects.insert(key.trim().to_owned(), value.trim().to_owned()); + fastn_core::warning!( + "Redirect syntax: '{key}: {value}' will be deprecated\nPlease use the '{key} \ + -> {value}' redirect syntax instead." + ); + Self::assert_and_insert_redirect(key, value, &mut redirects)?; } } - redirects + Ok(redirects) + } + + // Assert checks on redirects + // - All redirects should be A -> B where A != B (Self loop) + // - If A -> B exists then there can’t be A -> C where B != C + // (No duplicated values starting with the same A) + fn assert_and_insert_redirect( + from: &str, + to: &str, + redirects: &mut ftd::Map, + ) -> fastn_core::Result<()> { + let from = from.trim().to_owned(); + let to = to.trim().to_owned(); + + assert!( + !from.eq(to.as_str()), + "Redirect {} -> {} is invalid", + from, + to + ); + assert!( + !redirects.contains_key(from.as_str()), + "Redirect {} -> {} is invalid, since {} -> {} already exists", + from.as_str(), + to.as_str(), + from.as_str(), + redirects.get(from.as_str()).unwrap(), + ); + + redirects.insert(from, to); + Ok(()) } } diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index d4f81dc11d..efd7d3b76e 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -42,9 +42,18 @@ pub fn get_ftd_hash(path: &str) -> fastn_core::Result { } pub fn get_cache_file(id: &str) -> Option { + let cache_dir = dirs::cache_dir()?; + let base_path = cache_dir.join("fastn.com"); + + if !base_path.exists() { + if let Err(err) = std::fs::create_dir_all(&base_path) { + eprintln!("Failed to create cache directory: {}", err); + return None; + } + } + Some( - dirs::cache_dir()? - .join("fastn.com/") + base_path .join(id_to_cache_key( &std::env::current_dir() .expect("cant read current dir") diff --git a/fastn-js/js/dom.js b/fastn-js/js/dom.js index 5ef5652611..df3d4474e2 100644 --- a/fastn-js/js/dom.js +++ b/fastn-js/js/dom.js @@ -60,6 +60,7 @@ fastn_dom.propertyMap = { "justify-content": "jc", "left": "l", "link": "lk", + "link-color": "lkc", "margin": "m", "margin-bottom": "mb", "margin-horizontal": "mh", @@ -273,6 +274,7 @@ fastn_dom.PropertyKind = { LoopVideo: 113, Controls: 114, Muted: 115, + LinkColor: 116, }; @@ -870,7 +872,7 @@ class Node2 { attachCss(property, value, createClass, className) { let propertyShort = fastn_dom.propertyMap[property] || property; propertyShort = `__${propertyShort}`; - let cls = `${propertyShort}-${JSON.stringify(value)}`; + let cls = `${propertyShort}-${JSON.stringify(fastn_dom.class_count)}`; if (!!className) { cls = className; } else { @@ -1234,7 +1236,7 @@ class Node2 { this.attachCss("align-items", "end"); break; case 'bottom-center': - this.attachCss("justify-content", "start"); + this.attachCss("justify-content", "center"); this.attachCss("align-items", "end"); break; case 'bottom-right': @@ -1244,6 +1246,58 @@ class Node2 { } } } + attachLinkColor(value) { + ftd.dark_mode.addClosure(fastn.closure(() => { + if (!ssr) { + const anchors = this.#node.tagName.toLowerCase() === 'a' + ? [this.#node] + : Array.from(this.#node.querySelectorAll("a")); + let propertyShort = `__${fastn_dom.propertyMap["link-color"]}`; + + if(fastn_utils.isNull(value)) { + anchors.forEach(a => { + a.classList.values().forEach(className => { + if(className.startsWith(`${propertyShort}-`)) { + a.classList.remove(className); + } + }); + }); + } else { + const lightValue = fastn_utils.getStaticValue(value.get("light")); + const darkValue = fastn_utils.getStaticValue(value.get("dark")); + let cls = `${propertyShort}-${JSON.stringify(lightValue)}`; + + if (!fastn_dom.unsanitised_classes[cls]) { + fastn_dom.unsanitised_classes[cls] = ++fastn_dom.class_count; + } + + cls = `${propertyShort}-${fastn_dom.unsanitised_classes[cls]}`; + + const cssClass = `.${cls}`; + + if (!fastn_dom.classes[cssClass]) { + const obj = { property: "color", value: lightValue }; + fastn_dom.classes[cssClass] = fastn_dom.classes[cssClass] || obj; + let styles = document.getElementById('styles'); + styles.innerHTML = `${styles.innerHTML}${getClassAsString(cssClass, obj)}\n`; + } + + if(lightValue !== darkValue) { + const obj = { property: "color", value: darkValue }; + let darkCls = `body.dark ${cssClass}`; + if (!fastn_dom.classes[darkCls]) { + fastn_dom.classes[darkCls] = fastn_dom.classes[darkCls] || obj; + let styles = document.getElementById('styles'); + styles.innerHTML = `${styles.innerHTML}${getClassAsString(darkCls, obj)}\n`; + } + } + + anchors.forEach(a => a.classList.add(cls)); + } + } + }).addNodeProperty(this, null, inherited)); + this.#mutables.push(ftd.dark_mode); + } setStaticProperty(kind, value, inherited) { // value can be either static or mutable let staticValue = fastn_utils.getStaticValue(value); @@ -1494,6 +1548,8 @@ class Node2 { this.attachColorCss("border-top-color", staticValue); } else if (kind === fastn_dom.PropertyKind.BorderBottomColor) { this.attachColorCss("border-bottom-color", staticValue); + } else if (kind === fastn_dom.PropertyKind.LinkColor) { + this.attachLinkColor(staticValue); } else if (kind === fastn_dom.PropertyKind.Color) { this.attachColorCss("color", staticValue, true); } else if (kind === fastn_dom.PropertyKind.Background) { diff --git a/fastn-js/js/fastn.js b/fastn-js/js/fastn.js index 07dbe6d19a..fdb07bc50f 100644 --- a/fastn-js/js/fastn.js +++ b/fastn-js/js/fastn.js @@ -266,7 +266,7 @@ class MutableList { let current_list = this.#list; let new_list = []; for (let idx in current_list) { - new_list.push( { item: fastn_utils.clone(current_list[idx].item), index: new Mutable(parseInt(idx)) }); + new_list.push(fastn_utils.clone(current_list[idx].item)); } return new MutableList(new_list); } diff --git a/fastn-js/src/property.rs b/fastn-js/src/property.rs index 78d1d940b5..0ce8c31966 100644 --- a/fastn-js/src/property.rs +++ b/fastn-js/src/property.rs @@ -318,6 +318,7 @@ pub enum PropertyKind { Region, OpenInNewTab, Link, + LinkColor, LinkRel, Anchor, Classes, @@ -432,6 +433,7 @@ impl PropertyKind { PropertyKind::Id => "fastn_dom.PropertyKind.Id", PropertyKind::Css => "fastn_dom.PropertyKind.Css", PropertyKind::Js => "fastn_dom.PropertyKind.Js", + PropertyKind::LinkColor => "fastn_dom.PropertyKind.LinkColor", PropertyKind::LinkRel => "fastn_dom.PropertyKind.LinkRel", PropertyKind::AlignSelf => "fastn_dom.PropertyKind.AlignSelf", PropertyKind::Anchor => "fastn_dom.PropertyKind.Anchor", diff --git a/fastn-js/src/to_js.rs b/fastn-js/src/to_js.rs index 4c48967c35..1d58968bba 100644 --- a/fastn-js/src/to_js.rs +++ b/fastn-js/src/to_js.rs @@ -943,6 +943,9 @@ impl ExpressionGenerator { return format!( indoc::indoc! { "let fastn_utils_val_{refined_var} = fastn_utils.clone({val}); + if (fastn_utils_val_{refined_var} instanceof fastn.mutableClass) {{ + fastn_utils_val_{refined_var} = fastn_utils_val_{refined_var}.get(); + }} if (!fastn_utils.setter({var}, fastn_utils_val_{refined_var})) {{ {var} = fastn_utils_val_{refined_var}; }}" diff --git a/fastn/Cargo.toml b/fastn/Cargo.toml index e71a05a61f..3f2ba5399f 100644 --- a/fastn/Cargo.toml +++ b/fastn/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fastn" -version = "0.3.42" +version = "0.3.49" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ftd/src/interpreter/things/default.rs b/ftd/src/interpreter/things/default.rs index fa3ad63ed3..f46de6eac2 100644 --- a/ftd/src/interpreter/things/default.rs +++ b/ftd/src/interpreter/things/default.rs @@ -10761,6 +10761,12 @@ fn text_arguments() -> Vec { .into_optional() .into_kind_data(), ), + ftd::interpreter::Argument::default( + "link-color", + ftd::interpreter::Kind::record(ftd::interpreter::FTD_COLOR) + .into_optional() + .into_kind_data(), + ), ] } diff --git a/ftd/src/js/element.rs b/ftd/src/js/element.rs index 9448ec1d87..af6987c1ce 100644 --- a/ftd/src/js/element.rs +++ b/ftd/src/js/element.rs @@ -1888,6 +1888,7 @@ pub struct TextCommon { pub line_clamp: Option, pub style: Option, pub display: Option, + pub link_color: Option, } impl TextCommon { @@ -1910,6 +1911,7 @@ impl TextCommon { line_clamp: ftd::js::value::get_optional_js_value("line-clamp", properties, arguments), style: ftd::js::value::get_optional_js_value("style", properties, arguments), display: ftd::js::value::get_optional_js_value("display", properties, arguments), + link_color: ftd::js::value::get_optional_js_value("link-color", properties, arguments), } } @@ -1960,6 +1962,16 @@ impl TextCommon { display.to_set_property(fastn_js::PropertyKind::Display, doc, element_name, rdata), )); } + if let Some(ref link_color) = self.link_color { + component_statements.push(fastn_js::ComponentStatement::SetProperty( + link_color.to_set_property( + fastn_js::PropertyKind::LinkColor, + doc, + element_name, + rdata, + ), + )); + } component_statements } } diff --git a/ftd/t/js/01-basic-module.html b/ftd/t/js/01-basic-module.html index c2f051f5af..9501b33469 100644 --- a/ftd/t/js/01-basic-module.html +++ b/ftd/t/js/01-basic-module.html @@ -16,20 +16,20 @@ -
fastn
Hello
Hello
+ + + + + + +
Rithik
Ritesh
Heulitig
+ + diff --git a/ftd/t/js/53-link-color.ftd b/ftd/t/js/53-link-color.ftd new file mode 100644 index 0000000000..e944c47d31 --- /dev/null +++ b/ftd/t/js/53-link-color.ftd @@ -0,0 +1,14 @@ +-- ftd.color link-color: +dark: red +light: blue + +-- ftd.text: Click me +link: https://google.com +link-color: $link-color + +-- ftd.text: +link-color: $link-color + +Hello world [Test](https://google.com) + +This is awesome [Test](https://google.com) diff --git a/ftd/t/js/53-link-color.html b/ftd/t/js/53-link-color.html new file mode 100644 index 0000000000..a2126a6d6f --- /dev/null +++ b/ftd/t/js/53-link-color.html @@ -0,0 +1,90 @@ + + + + + + + + + + + + + + +
Click me
Hello world [Test](https://google.com) + +This is awesome [Test](https://google.com)
+ + diff --git a/ftd/t/js/54-class-fix.ftd b/ftd/t/js/54-class-fix.ftd new file mode 100644 index 0000000000..c86a68bba9 --- /dev/null +++ b/ftd/t/js/54-class-fix.ftd @@ -0,0 +1,10 @@ +-- ftd.text: Hello +background.solid: $bg-yg + +-- ftd.text: hello +background.solid: yellow + + +-- ftd.color bg-yg: +light: yellow +dark: green diff --git a/ftd/t/js/54-class-fix.html b/ftd/t/js/54-class-fix.html new file mode 100644 index 0000000000..36b2d923e1 --- /dev/null +++ b/ftd/t/js/54-class-fix.html @@ -0,0 +1,93 @@ + + + + + + + + + + + + + + +
Hello
hello
+ + From b83551fc31ee1830b572a1b0584f2f8b53afea54 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 14 Sep 2023 14:01:28 +0530 Subject: [PATCH 57/63] id to cache key fix --- fastn-core/src/utils.rs | 2 +- ftd/t/js/test.html | 4912 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 4913 insertions(+), 1 deletion(-) create mode 100644 ftd/t/js/test.html diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index efd7d3b76e..3f68a69be8 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -30,7 +30,7 @@ macro_rules! warning { } fn id_to_cache_key(id: &str) -> String { - id.replace('/', "_") + id.replace(std::path::MAIN_SEPARATOR, "_") } pub fn get_ftd_hash(path: &str) -> fastn_core::Result { diff --git a/ftd/t/js/test.html b/ftd/t/js/test.html new file mode 100644 index 0000000000..9660a90891 --- /dev/null +++ b/ftd/t/js/test.html @@ -0,0 +1,4912 @@ + + + + From e139553fc44d132e9680b158ce767072695cc18e Mon Sep 17 00:00:00 2001 From: Harsh Singh <64768386+harshdoesdev@users.noreply.github.com> Date: Thu, 14 Sep 2023 14:04:01 +0530 Subject: [PATCH 58/63] Delete ftd/t/js/test.html Signed-off-by: Harsh Singh <64768386+harshdoesdev@users.noreply.github.com> --- ftd/t/js/test.html | 4912 -------------------------------------------- 1 file changed, 4912 deletions(-) delete mode 100644 ftd/t/js/test.html diff --git a/ftd/t/js/test.html b/ftd/t/js/test.html deleted file mode 100644 index 9660a90891..0000000000 --- a/ftd/t/js/test.html +++ /dev/null @@ -1,4912 +0,0 @@ - - - - From f39f2de77cf3cf9f4916a35f5743a0430299f83f Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 14 Sep 2023 15:11:54 +0530 Subject: [PATCH 59/63] fixed windows bugs --- fastn-core/src/utils.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index 3f68a69be8..89bb2a6ae7 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -30,7 +30,8 @@ macro_rules! warning { } fn id_to_cache_key(id: &str) -> String { - id.replace(std::path::MAIN_SEPARATOR, "_") + // TODO: use MAIN_SEPARATOR here + id.replace("/", "_").replace("\\", "_") } pub fn get_ftd_hash(path: &str) -> fastn_core::Result { From 0adcf66c87749dfac987270bef6d9090b20d5968 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 14 Sep 2023 15:14:33 +0530 Subject: [PATCH 60/63] fixed conflicts --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f516e6878c..b1af5035d6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -75,7 +75,7 @@ jobs: - name: Download EnVar plugin for NSIS uses: carlosperate/download-file-action@v1.0.3 with: - file-url: https://nsis.sourceforge.io/mediawiki/images/7/7f/EnVar_plugin.zip + file-url: https://github.com/GsNSIS/EnVar/releases/download/v0.3.1/EnVar-Plugin.zip file-name: envar_plugin.zip location: ${{ github.workspace }} - name: Extract EnVar plugin From 6f940f90bba39a5f1e3042e644989a898c82b8b5 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 14 Sep 2023 15:25:35 +0530 Subject: [PATCH 61/63] updated files --- fastn-core/src/commands/build.rs | 3 +-- fastn-js/js/dom.js | 16 ++++++++++++++-- ftd/ftd-js.css | 14 +++++++++++++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 7c29712cfe..17fd0fe5fd 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -1,5 +1,3 @@ -use itertools::Itertools; - // #[tracing::instrument(skip(config))] pub async fn build( config: &mut fastn_core::Config, @@ -237,6 +235,7 @@ async fn incremental_build( test: bool, ) -> fastn_core::Result<()> { // https://fastn.com/rfc/incremental-build/ + use itertools::Itertools; let (cache_hit, mut c) = cache::get()?; diff --git a/fastn-js/js/dom.js b/fastn-js/js/dom.js index 7ca6102235..f4cd033ef8 100644 --- a/fastn-js/js/dom.js +++ b/fastn-js/js/dom.js @@ -1662,8 +1662,20 @@ class Node2 { } const is_dark_mode = ftd.dark_mode.get(); const src = staticValue.get(is_dark_mode ? 'dark' : 'light'); - - this.attachAttribute("src", fastn_utils.getStaticValue(src)); + if (!ssr) { + let image_node = this.#node; + if( image_node.nodeName.toLowerCase() === "a" ) { + let childNodes = image_node.childNodes; + childNodes.forEach(function(child) { + if (child.nodeName.toLowerCase() === "img") + image_node = child; + }); + } + image_node.setAttribute("src", fastn_utils.getStaticValue(src)); + } + else { + this.attachAttribute("src", fastn_utils.getStaticValue(src)); + } }).addNodeProperty(this, null, inherited)); this.#mutables.push(ftd.dark_mode); } else if (kind === fastn_dom.PropertyKind.Alt) { diff --git a/ftd/ftd-js.css b/ftd/ftd-js.css index e91fecf247..5acf30d3d7 100644 --- a/ftd/ftd-js.css +++ b/ftd/ftd-js.css @@ -87,7 +87,6 @@ body { input, code { vertical-align: middle; - font: inherit; } pre { white-space: break-spaces; @@ -222,6 +221,11 @@ pre code { background: #c1c8ce; } +ul { + /* Added padding to the left to move the bullet marker to the right */ + padding-left: 20px; +} + a { color: #2952a3; } @@ -290,3 +294,11 @@ h1:only-child { table, td, th { border: 1px solid; } + +th { + padding: 6px; +} + +td { + padding-left: 6px; +} From 4a879f3c1cc4b0d3cdde2ac65baf9937742ee640 Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 14 Sep 2023 15:28:42 +0530 Subject: [PATCH 62/63] clippy fixes --- fastn-core/src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastn-core/src/utils.rs b/fastn-core/src/utils.rs index 89bb2a6ae7..2a237d35dc 100644 --- a/fastn-core/src/utils.rs +++ b/fastn-core/src/utils.rs @@ -31,7 +31,7 @@ macro_rules! warning { fn id_to_cache_key(id: &str) -> String { // TODO: use MAIN_SEPARATOR here - id.replace("/", "_").replace("\\", "_") + id.replace(['/', '\\'], "_") } pub fn get_ftd_hash(path: &str) -> fastn_core::Result { From feaddfd10c12b699fff86b4d93d956ea0a99298a Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 14 Sep 2023 15:40:20 +0530 Subject: [PATCH 63/63] refactored code --- fastn-core/src/commands/build.rs | 89 ++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/fastn-core/src/commands/build.rs b/fastn-core/src/commands/build.rs index 17fd0fe5fd..912d5cb7b3 100644 --- a/fastn-core/src/commands/build.rs +++ b/fastn-core/src/commands/build.rs @@ -226,6 +226,56 @@ async fn handle_dependency_file( Ok(()) } +// removes deleted documents from cache and build folder +fn remove_deleted_documents( + config: &mut fastn_core::Config, + c: &mut cache::Cache, + documents: &std::collections::BTreeMap, +) -> fastn_core::Result<()> { + use itertools::Itertools; + + let removed_documents = c + .documents + .keys() + .filter(|cached_document_id| { + for document in documents.values() { + if remove_extension(document.get_id()).eq(cached_document_id.as_str()) + || remove_extension(&document.get_id_with_package()) + .eq(cached_document_id.as_str()) + { + return false; + } + } + + true + }) + .map(|id| id.to_string()) + .collect_vec(); + + for removed_doc_id in &removed_documents { + let folder_path = config.build_dir().join(removed_doc_id); + let folder_parent = folder_path.parent(); + let file_path = &folder_path.with_extension("ftd"); + + if file_path.exists() { + std::fs::remove_file(file_path)?; + } + + std::fs::remove_dir_all(&folder_path)?; + + // If the parent folder of the file's output folder is also empty, delete it as well. + if let Some(folder_parent) = folder_parent { + if folder_parent.read_dir()?.count().eq(&0) { + std::fs::remove_dir_all(folder_parent)?; + } + } + + c.documents.remove(removed_doc_id); + } + + Ok(()) +} + #[tracing::instrument(skip(config, documents))] async fn incremental_build( config: &mut fastn_core::Config, @@ -337,44 +387,7 @@ async fn incremental_build( } } - let removed_documents = c - .documents - .keys() - .filter(|cached_document_id| { - for document in documents.values() { - if remove_extension(document.get_id()).eq(cached_document_id.as_str()) - || remove_extension(&document.get_id_with_package()) - .eq(cached_document_id.as_str()) - { - return false; - } - } - - true - }) - .map(|id| id.to_string()) - .collect_vec(); - - for removed_doc_id in &removed_documents { - let folder_path = config.build_dir().join(removed_doc_id); - let folder_parent = folder_path.parent(); - let file_path = &folder_path.with_extension("ftd"); - - if file_path.exists() { - std::fs::remove_file(file_path)?; - } - - std::fs::remove_dir_all(&folder_path)?; - - // If the parent folder of the file's output folder is also empty, delete it as well. - if let Some(folder_parent) = folder_parent { - if folder_parent.read_dir()?.count().eq(&0) { - std::fs::remove_dir_all(folder_parent)?; - } - } - - c.documents.remove(removed_doc_id); - } + remove_deleted_documents(config, &mut c, documents)?; } else { for document in documents.values() { handle_file(