From d55aa763d06410a9aea738f7b8d80e0c2d8317d2 Mon Sep 17 00:00:00 2001 From: Tomas Tauber <2410580+tomtau@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:15:32 +0800 Subject: [PATCH 1/4] fix: adjust tag association in optional and repeat expressions fixes #984, fixes #982 --- derive/tests/implicit.rs | 10 ++++++ derive/tests/opt.pest | 8 +++++ derive/tests/opt.rs | 29 ++++++++++++++++ generator/src/generator.rs | 70 ++++++++++++++++++++++++++++++++------ 4 files changed, 107 insertions(+), 10 deletions(-) create mode 100644 derive/tests/opt.pest create mode 100644 derive/tests/opt.rs diff --git a/derive/tests/implicit.rs b/derive/tests/implicit.rs index 48c88021..0ca7a43d 100644 --- a/derive/tests/implicit.rs +++ b/derive/tests/implicit.rs @@ -35,3 +35,13 @@ fn test_implicit_whitespace() { assert_eq!(pairs.clone().find_tagged("one_comp").count(), 2); assert_eq!(pairs.find_tagged("one_array").count(), 2); } + +#[test] +#[cfg(feature = "grammar-extras")] +fn test_implicit_whitespace_multitag() { + let successful_parse = TestImplicitParser::parse(Rule::program, "a a a"); + assert!(successful_parse.is_ok()); + dbg!(&successful_parse); + let pairs = successful_parse.unwrap(); + assert_eq!(pairs.clone().find_tagged("tail").count(), 2); +} diff --git a/derive/tests/opt.pest b/derive/tests/opt.pest new file mode 100644 index 00000000..b5ff4661 --- /dev/null +++ b/derive/tests/opt.pest @@ -0,0 +1,8 @@ +expr = { + SOI ~ + #prefix=(STAR)? ~ #suffix=DOT? + ~ EOI +} + +STAR={"*"} +DOT={"."} diff --git a/derive/tests/opt.rs b/derive/tests/opt.rs new file mode 100644 index 00000000..8ce316c8 --- /dev/null +++ b/derive/tests/opt.rs @@ -0,0 +1,29 @@ +// Licensed under the Apache License, Version 2.0 +// or the MIT +// license , at your +// option. All files in the project carrying such notice may not be copied, +// modified, or distributed except according to those terms. + +#![cfg_attr(not(feature = "std"), no_std)] +extern crate alloc; +extern crate pest; +extern crate pest_derive; + +#[cfg(feature = "grammar-extras")] +use pest::Parser; +use pest_derive::Parser; + +#[derive(Parser)] +#[grammar = "../tests/opt.pest"] +struct TestOptParser; + +#[test] +#[cfg(feature = "grammar-extras")] +fn test_opt_tag() { + let successful_parse = TestOptParser::parse(Rule::expr, "*"); + assert!(successful_parse.is_ok()); + dbg!(&successful_parse); + let pairs = successful_parse.unwrap(); + assert!(pairs.find_first_tagged("prefix").is_some()); + assert!(pairs.find_first_tagged("suffix").is_none()); +} diff --git a/generator/src/generator.rs b/generator/src/generator.rs index f5fa5a1f..042b2ff9 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -571,12 +571,42 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream { } } #[cfg(feature = "grammar-extras")] - OptimizedExpr::NodeTag(expr, tag) => { - let expr = generate_expr(*expr); - quote! { - #expr.and_then(|state| state.tag_node(#tag)) + OptimizedExpr::NodeTag(expr, tag) => match *expr { + OptimizedExpr::Opt(expr) => { + let expr = generate_expr(*expr); + quote! { + state.optional(|state| { + #expr.and_then(|state| state.tag_node(#tag)) + }) + } } - } + OptimizedExpr::Rep(expr) => { + let expr = generate_expr(*expr); + quote! { + state.sequence(|state| { + state.optional(|state| { + #expr.and_then(|state| { + state.repeat(|state| { + state.sequence(|state| { + super::hidden::skip( + state + ).and_then(|state| { + #expr.and_then(|state| state.tag_node(#tag)) + }) + }) + }) + }).and_then(|state| state.tag_node(#tag)) + }) + }) + } + } + expr => { + let expr = generate_expr(expr); + quote! { + #expr.and_then(|state| state.tag_node(#tag)) + } + } + }, } } @@ -726,12 +756,32 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream { } } #[cfg(feature = "grammar-extras")] - OptimizedExpr::NodeTag(expr, tag) => { - let expr = generate_expr_atomic(*expr); - quote! { - #expr.and_then(|state| state.tag_node(#tag)) + OptimizedExpr::NodeTag(expr, tag) => match *expr { + OptimizedExpr::Opt(expr) => { + let expr = generate_expr_atomic(*expr); + + quote! { + state.optional(|state| { + #expr.and_then(|state| state.tag_node(#tag)) + }) + } } - } + OptimizedExpr::Rep(expr) => { + let expr = generate_expr_atomic(*expr); + + quote! { + state.repeat(|state| { + #expr.and_then(|state| state.tag_node(#tag)) + }) + } + } + expr => { + let expr = generate_expr_atomic(expr); + quote! { + #expr.and_then(|state| state.tag_node(#tag)) + } + } + }, } } From 3f579321cfc2388e632c8a599a612b5065449034 Mon Sep 17 00:00:00 2001 From: Tomas Tauber <2410580+tomtau@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:34:51 +0800 Subject: [PATCH 2/4] remove the dbg! macro in tests --- derive/tests/implicit.rs | 1 - derive/tests/opt.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/derive/tests/implicit.rs b/derive/tests/implicit.rs index 0ca7a43d..20dda844 100644 --- a/derive/tests/implicit.rs +++ b/derive/tests/implicit.rs @@ -41,7 +41,6 @@ fn test_implicit_whitespace() { fn test_implicit_whitespace_multitag() { let successful_parse = TestImplicitParser::parse(Rule::program, "a a a"); assert!(successful_parse.is_ok()); - dbg!(&successful_parse); let pairs = successful_parse.unwrap(); assert_eq!(pairs.clone().find_tagged("tail").count(), 2); } diff --git a/derive/tests/opt.rs b/derive/tests/opt.rs index 8ce316c8..e6204482 100644 --- a/derive/tests/opt.rs +++ b/derive/tests/opt.rs @@ -22,7 +22,6 @@ struct TestOptParser; fn test_opt_tag() { let successful_parse = TestOptParser::parse(Rule::expr, "*"); assert!(successful_parse.is_ok()); - dbg!(&successful_parse); let pairs = successful_parse.unwrap(); assert!(pairs.find_first_tagged("prefix").is_some()); assert!(pairs.find_first_tagged("suffix").is_none()); From 1e36973b88da9af2179ea1910b2403f3ed74a9ae Mon Sep 17 00:00:00 2001 From: Tomas Tauber <2410580+tomtau@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:38:45 +0800 Subject: [PATCH 3/4] extra tests for optional expression tags --- derive/tests/opt.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/derive/tests/opt.rs b/derive/tests/opt.rs index e6204482..5c07f84d 100644 --- a/derive/tests/opt.rs +++ b/derive/tests/opt.rs @@ -25,4 +25,18 @@ fn test_opt_tag() { let pairs = successful_parse.unwrap(); assert!(pairs.find_first_tagged("prefix").is_some()); assert!(pairs.find_first_tagged("suffix").is_none()); + + // Test with no STAR or DOT + let parse_no_components = TestOptParser::parse(Rule::expr, ""); + assert!(parse_no_components.is_ok()); + let pairs_no_components = parse_no_components.unwrap(); + assert!(pairs_no_components.find_first_tagged("prefix").is_none()); + assert!(pairs_no_components.find_first_tagged("suffix").is_none()); + + // Test with only DOT + let parse_only_dot = TestOptParser::parse(Rule::expr, "."); + assert!(parse_only_dot.is_ok()); + let pairs_only_dot = parse_only_dot.unwrap(); + assert!(pairs_only_dot.find_first_tagged("prefix").is_none()); + assert!(pairs_only_dot.find_first_tagged("suffix").is_some()); } From c7c2ff0200d0e4640a54418cc440ff514440d1b0 Mon Sep 17 00:00:00 2001 From: Tomas Tauber <2410580+tomtau@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:14:35 +0800 Subject: [PATCH 4/4] bump versions --- debugger/Cargo.toml | 19 +++++++++++++------ derive/Cargo.toml | 6 +++--- generator/Cargo.toml | 6 +++--- grammars/Cargo.toml | 6 +++--- meta/Cargo.toml | 12 +++++++++--- pest/Cargo.toml | 2 +- vm/Cargo.toml | 6 +++--- 7 files changed, 35 insertions(+), 22 deletions(-) diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml index 4947d191..f22d7173 100644 --- a/debugger/Cargo.toml +++ b/debugger/Cargo.toml @@ -1,9 +1,12 @@ [package] name = "pest_debugger" description = "pest grammar debugger" -version = "2.7.7" +version = "2.7.8" edition = "2021" -authors = ["Dragoș Tiselice ", "Tomas Tauber "] +authors = [ + "Dragoș Tiselice ", + "Tomas Tauber ", +] homepage = "https://pest.rs/" repository = "https://github.com/pest-parser/pest" documentation = "https://docs.rs/pest" @@ -14,10 +17,14 @@ readme = "_README.md" rust-version = "1.61" [dependencies] -pest = { path = "../pest", version = "2.7.7" } -pest_meta = { path = "../meta", version = "2.7.7" } -pest_vm = { path = "../vm", version = "2.7.7" } -reqwest = { version = "= 0.11.13", default-features = false, features = ["blocking", "json", "default-tls"] } +pest = { path = "../pest", version = "2.7.8" } +pest_meta = { path = "../meta", version = "2.7.8" } +pest_vm = { path = "../vm", version = "2.7.8" } +reqwest = { version = "= 0.11.13", default-features = false, features = [ + "blocking", + "json", + "default-tls", +] } rustyline = "10" serde_json = "1" thiserror = "1" diff --git a/derive/Cargo.toml b/derive/Cargo.toml index eecdfde8..58d56c58 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_derive" description = "pest's derive macro" -version = "2.7.7" +version = "2.7.8" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -25,5 +25,5 @@ grammar-extras = ["pest_generator/grammar-extras"] [dependencies] # for tests, included transitively anyway -pest = { path = "../pest", version = "2.7.7", default-features = false } -pest_generator = { path = "../generator", version = "2.7.7", default-features = false } +pest = { path = "../pest", version = "2.7.8", default-features = false } +pest_generator = { path = "../generator", version = "2.7.8", default-features = false } diff --git a/generator/Cargo.toml b/generator/Cargo.toml index 166da150..eb80f92f 100644 --- a/generator/Cargo.toml +++ b/generator/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_generator" description = "pest code generator" -version = "2.7.7" +version = "2.7.8" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -22,8 +22,8 @@ grammar-extras = ["pest_meta/grammar-extras"] export-internal = [] [dependencies] -pest = { path = "../pest", version = "2.7.7", default-features = false } -pest_meta = { path = "../meta", version = "2.7.7" } +pest = { path = "../pest", version = "2.7.8", default-features = false } +pest_meta = { path = "../meta", version = "2.7.8" } proc-macro2 = "1.0" quote = "1.0" syn = "2.0" diff --git a/grammars/Cargo.toml b/grammars/Cargo.toml index 4f3f2df4..4f1a4e0a 100644 --- a/grammars/Cargo.toml +++ b/grammars/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_grammars" description = "pest popular grammar implementations" -version = "2.7.7" +version = "2.7.8" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -14,8 +14,8 @@ readme = "_README.md" rust-version = "1.61" [dependencies] -pest = { path = "../pest", version = "2.7.7" } -pest_derive = { path = "../derive", version = "2.7.7" } +pest = { path = "../pest", version = "2.7.8" } +pest_derive = { path = "../derive", version = "2.7.8" } [dev-dependencies] criterion = "0.5" diff --git a/meta/Cargo.toml b/meta/Cargo.toml index 920e6df2..71fdbd9a 100644 --- a/meta/Cargo.toml +++ b/meta/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_meta" description = "pest meta language parser and validator" -version = "2.7.7" +version = "2.7.8" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -12,11 +12,17 @@ categories = ["parsing"] license = "MIT OR Apache-2.0" readme = "_README.md" exclude = ["src/grammar.pest"] -include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"] +include = [ + "Cargo.toml", + "src/**/*", + "src/grammar.rs", + "_README.md", + "LICENSE-*", +] rust-version = "1.61" [dependencies] -pest = { path = "../pest", version = "2.7.7" } +pest = { path = "../pest", version = "2.7.8" } once_cell = "1.8.0" [build-dependencies] diff --git a/pest/Cargo.toml b/pest/Cargo.toml index a0f4b7df..3ffb3ebf 100644 --- a/pest/Cargo.toml +++ b/pest/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest" description = "The Elegant Parser" -version = "2.7.7" +version = "2.7.8" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index ba38f140..fc597f1f 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_vm" description = "pest grammar virtual machine" -version = "2.7.7" +version = "2.7.8" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -14,8 +14,8 @@ readme = "_README.md" rust-version = "1.61" [dependencies] -pest = { path = "../pest", version = "2.7.7" } -pest_meta = { path = "../meta", version = "2.7.7" } +pest = { path = "../pest", version = "2.7.8" } +pest_meta = { path = "../meta", version = "2.7.8" } [features] grammar-extras = ["pest_meta/grammar-extras"]