word token regex was wrong

fastn-stack · Oct 23, 2024 · 081b237 · 081b237
1 parent 0786cd1
commit 081b237
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 3 deletions.
diff --git a/v0.5/fastn-p1/src/parser_v3/scanner.rs b/v0.5/fastn-p1/src/parser_v3/scanner.rs
@@ -11,7 +11,7 @@ impl Scanner {
         Scanner {
             tokens: fastn_p1::Token::lexer(source)
                 .spanned()
-                .map(|(r, span)| (r.unwrap(), span))
+                .map(|(r, span)| (dbg!(r.unwrap()), span))
                 .collect(),
             index: 0,
             output: fastn_p1::ParseOutput {

diff --git a/v0.5/fastn-p1/src/test/mod.rs b/v0.5/fastn-p1/src/test/mod.rs
@@ -21,6 +21,7 @@ fn test_all() {
 fn p1(file: impl AsRef<std::path::Path> + std::fmt::Debug, fix: bool) {
     let json = file.as_ref().with_extension("json");
     let s = std::fs::read_to_string(&file).unwrap();
+    println!("testing {file:?}");
     let output = fastn_p1::ParseOutput::new("foo", &s);
     let expected_json =
         fastn_p1::test::sorted_json::to_json(&serde_json::to_value(&output).unwrap());

diff --git a/v0.5/fastn-p1/src/token.rs b/v0.5/fastn-p1/src/token.rs
@@ -15,7 +15,7 @@ pub enum Token {
     #[regex("--")]
     DashDash,
 
-    #[regex(r"[\w]+")]
+    #[regex(r"[^\s]+", priority = 0)]
     Word,
 
     #[regex("[\t ]+")]
@@ -81,7 +81,7 @@ mod test {
         let source = include_str!("../t/002-tutorial.ftd");
         assert_eq!(
             dbg!(super::Token::lexer(source).spanned().collect::<Vec<_>>()).len(),
-            585
+            434
         );
     }
 }