continuation based compiler, buggy yet

v0.5/fastn-compiler/Cargo.toml

@@ -8,9 +8,6 @@ license.workspace = true
 repository.workspace = true
 homepage.workspace = true
 
-[features]
-owned-tdoc = ["fastn-resolved/owned-tdoc"]
-
 [dependencies]
 fastn-section.workspace = true
 fastn-resolved.workspace = true

v0.5/fastn-compiler/src/compiler.rs

@@ -15,16 +15,16 @@ pub enum CompilerState {
 // exposing: y as x
 pub struct Compiler {
     pub(crate) definitions_used: std::collections::HashSet<fastn_unresolved::Symbol>,
-    pub(crate) arena: fastn_unresolved::Arena,
+    pub arena: fastn_unresolved::Arena,
     pub(crate) definitions: std::collections::HashMap<String, fastn_unresolved::URD>,
     /// we keep track of every module found (or not found), if not in dict we don't know
     /// if module exists, if in dict bool tells if it exists.
     pub(crate) modules: std::collections::HashMap<fastn_unresolved::Module, bool>,
     /// checkout resolve_document for why this is an Option
     pub(crate) content: Option<Vec<fastn_unresolved::URCI>>,
     pub(crate) document: fastn_unresolved::Document,
-    #[expect(unused)]
-    pub(crate) global_aliases: fastn_unresolved::AliasesSimple,
+    pub global_aliases: fastn_unresolved::AliasesSimple,
+    iterations: usize,
 }
 
 impl Compiler {
@@ -52,36 +52,10 @@ impl Compiler {
             document,
             global_aliases,
             definitions_used: Default::default(),
+            iterations: 0,
         }
     }
 
-    async fn fetch_unresolved_symbols(
-        &mut self,
-        _symbols_to_fetch: &std::collections::HashSet<fastn_unresolved::Symbol>,
-    ) {
-        todo!()
-        // self.definitions_used
-        //     .extend(symbols_to_fetch.iter().cloned());
-        // let definitions = self
-        //     .symbols
-        //     .lookup(&mut self.arena, &self.global_aliases, symbols_to_fetch)
-        //     .await;
-        // for definition in definitions {
-        //     // the following is only okay if our symbol store only returns unresolved definitions,
-        //     // some other store might return resolved definitions, and we need to handle that.
-        //     self.definitions.insert(
-        //         definition
-        //             .unresolved()
-        //             .unwrap()
-        //             .symbol
-        //             .clone()
-        //             .unwrap()
-        //             .string(&self.arena),
-        //         definition,
-        //     );
-        // }
-    }
-
     /// try to resolve as many symbols as possible, and return the ones that we made any progress on.
     ///
     /// this function should be called in a loop, until the list of symbols is empty.
@@ -177,48 +151,57 @@ impl Compiler {
         stuck_on_symbols
     }
 
-    async fn compile(mut self) -> CompilerState {
-        // we only make 10 attempts to resolve the document: we need a warning if we are not able to
-        // resolve the document in 10 attempts.
-        let mut unresolvable = std::collections::HashSet::new();
-        // let mut ever_used = std::collections::HashSet::new();
-        let mut iterations = 0;
-        while iterations < ITERATION_THRESHOLD {
-            // resolve_document can internally run in parallel.
-            // TODO: pass unresolvable to self.resolve_document() and make sure they don't come back
-            let unresolved_symbols = self.resolve_document();
-            if unresolved_symbols.is_empty() {
-                break;
-            }
-            // ever_used.extend(&unresolved_symbols);
-            self.fetch_unresolved_symbols(&unresolved_symbols).await;
-            // this itself has to happen in a loop. we need a warning if we are not able to resolve all
-            // symbols in 10 attempts.
-            let mut r = ResolveSymbolsResult::default();
-            r.need_more_symbols.extend(unresolved_symbols);
+    pub fn continue_with_definitions(
+        mut self,
+        definitions: Vec<fastn_unresolved::URD>,
+    ) -> CompilerState {
+        self.iterations += 1;
+        if self.iterations > ITERATION_THRESHOLD {
+            panic!("iterations too high");
+        }
 
-            while iterations < ITERATION_THRESHOLD {
-                // resolve_document can internally run in parallel.
-                // TODO: pass unresolvable to self.resolve_symbols() and make sure they don't come back
-                r = self.resolve_symbols(r.need_more_symbols);
-                unresolvable.extend(r.unresolvable);
-                if r.need_more_symbols.is_empty() {
-                    break;
-                }
-                // ever_used.extend(r.need_more_symbols);
-                self.fetch_unresolved_symbols(&r.need_more_symbols).await;
-                iterations += 1;
-            }
+        for definition in definitions {
+            // the following is only okay if our symbol store only returns unresolved definitions,
+            // some other store might return resolved definitions, and we need to handle that.
+            self.definitions.insert(
+                definition
+                    .unresolved()
+                    .unwrap()
+                    .symbol
+                    .clone()
+                    .unwrap()
+                    .string(&self.arena),
+                definition,
+            );
+        }
+
+        let unresolved_symbols = self.resolve_document();
+        if unresolved_symbols.is_empty() {
+            return CompilerState::Done(self.finalise(false));
+        }
 
-            iterations += 1;
+        // this itself has to happen in a loop. we need a warning if we are not able to resolve all
+        // symbols in 10 attempts.
+        let mut r = ResolveSymbolsResult {
+            need_more_symbols: unresolved_symbols,
+            unresolvable: Default::default(),
+        };
+        r = self.resolve_symbols(r.need_more_symbols);
+
+        if r.need_more_symbols.is_empty() {
+            return CompilerState::Done(self.finalise(true));
         }
 
+        CompilerState::StuckOnSymbols(Box::new(self), r.need_more_symbols)
+    }
+
+    fn finalise(
+        self,
+        some_symbols_are_unresolved: bool,
+    ) -> Result<fastn_resolved::CompiledDocument, fastn_compiler::Error> {
         // we are here means ideally we are done.
         // we could have some unresolvable symbols or self.document.errors may not be empty.
-        if !unresolvable.is_empty()
-            || !self.document.errors.is_empty()
-            || iterations == ITERATION_THRESHOLD
-        {
+        if some_symbols_are_unresolved || !self.document.errors.is_empty() {
             // we were not able to resolve all symbols or there were errors
             // return Err(fastn_compiler::Error {
             //     messages: todo!(),
@@ -229,14 +212,14 @@ impl Compiler {
         }
 
         // there were no errors, etc.
-        CompilerState::Done(Ok(fastn_resolved::CompiledDocument {
+        Ok(fastn_resolved::CompiledDocument {
             content: fastn_compiler::utils::resolved_content(self.content.unwrap()),
             definitions: fastn_compiler::utils::used_definitions(
                 self.definitions,
                 self.definitions_used,
                 self.arena,
             ),
-        }))
+        })
     }
 }
 
@@ -249,15 +232,13 @@ impl Compiler {
 ///
 /// earlier we had strict mode here, but to simplify things, now we let the caller convert non-empty
 /// warnings from OK part as error, and discard the generated JS.
-pub async fn compile(
+pub fn compile(
     source: &str,
     package: &str,
     module: Option<&str>,
     global_aliases: fastn_unresolved::AliasesSimple,
 ) -> CompilerState {
-    Compiler::new(source, package, module, global_aliases)
-        .compile()
-        .await
+    Compiler::new(source, package, module, global_aliases).continue_with_definitions(vec![])
 }
 
 #[derive(Default)]

v0.5/fastn-compiler/src/lib.rs

@@ -5,11 +5,9 @@
 extern crate self as fastn_compiler;
 
 mod compiler;
-mod tdoc;
 mod utils;
-pub use tdoc::CompiledDocument;
 
-pub use compiler::compile;
+pub use compiler::{compile, Compiler, CompilerState};
 pub use fastn_section::Result;
 
 #[derive(Debug)]

v0.5/fastn/Cargo.toml

@@ -10,8 +10,8 @@ homepage.workspace = true
 
 [dependencies]
 fastn-core.workspace = true
-#fastn-compiler.workspace = true
-#fastn-runtime.workspace = true
+fastn-compiler.workspace = true
+fastn-runtime.workspace = true
 fastn-unresolved.workspace = true
 hyper.workspace = true
 http-body-util.workspace = true

v0.5/fastn/src/commands/render.rs

@@ -18,19 +18,29 @@ impl fastn::commands::Render {
 }
 
 pub async fn render_document(
-    _global_aliases: fastn_unresolved::AliasesSimple,
-    _path: &str,
+    global_aliases: fastn_unresolved::AliasesSimple,
+    path: &str,
     _data: serde_json::Value,
     _strict: bool,
 ) -> String {
-    todo!()
-    // let source = std::fs::File::open(path)
-    //     .and_then(std::io::read_to_string)
-    //     .unwrap();
-    // let o = fastn_compiler::compile(&source, "main", None, global_aliases)
-    //     .await
-    //     .unwrap();
-    //
-    // let h = fastn_runtime::HtmlData::from_cd(o);
-    // h.to_test_html()
+    let source = std::fs::File::open(path)
+        .and_then(std::io::read_to_string)
+        .unwrap();
+    let mut cs = fastn_compiler::compile(&source, "main", None, global_aliases);
+    let mut symbol_store = fastn::Symbols {};
+
+    let o = loop {
+        match cs {
+            fastn_compiler::CompilerState::StuckOnSymbols(mut c, symbols) => {
+                let o = symbol_store.lookup(&mut c, &symbols).await;
+                cs = c.continue_with_definitions(o);
+            }
+            fastn_compiler::CompilerState::Done(c) => {
+                break c;
+            }
+        }
+    };
+
+    let h = fastn_runtime::HtmlData::from_cd(o.unwrap());
+    h.to_test_html()
 }