From d7b8c3c4aad1a794c6631063a580e48d5748a8fb Mon Sep 17 00:00:00 2001 From: Ross MacArthur Date: Fri, 10 Jan 2025 17:43:13 +0200 Subject: [PATCH] Support list and map literals --- src/compile/lex.rs | 84 ++++++++++++++++++++++++++++++++++++++++++-- src/compile/mod.rs | 16 ++++++++- src/compile/parse.rs | 53 ++++++++++++++++++++++++++-- src/render/core.rs | 33 ++++++++++++++++- src/render/value.rs | 10 ++++++ src/types/ast.rs | 16 +++++++++ src/types/program.rs | 12 +++++++ tests/lex.rs | 33 +++++++++++++++++ tests/render.rs | 71 +++++++++++++++++++++++++++++++++++++ 9 files changed, 322 insertions(+), 6 deletions(-) diff --git a/src/compile/lex.rs b/src/compile/lex.rs index b075b46..a27f428 100644 --- a/src/compile/lex.rs +++ b/src/compile/lex.rs @@ -28,6 +28,9 @@ pub struct Lexer<'engine, 'source> { /// A buffer to store the next token. next: Option<(Token, Span)>, + + /// The stack of delimiters within a block. + delimiters: Vec<(Span, Token)>, } /// The state of the lexer. @@ -89,6 +92,14 @@ pub enum Token { BeginComment, /// End block tag, e.g. `#}` EndComment, + /// `[` + OpenBracket, + /// `]` + CloseBracket, + /// `{` + OpenBrace, + /// `}` + CloseBrace, /// `.` Dot, /// `?.` @@ -127,6 +138,7 @@ impl<'engine, 'source> Lexer<'engine, 'source> { state: State::Template, left_trim: false, next: None, + delimiters: Vec::new(), } } @@ -244,6 +256,10 @@ impl<'engine, 'source> Lexer<'engine, 'source> { if tk != end { return Err(self.err_unexpected_token(tk, i..j)); } + // Check for unclosed delimiters. + if let Some((open, end)) = self.delimiters.pop() { + return Err(self.err_unclosed(open, end)); + } // A matching end tag! Update the state and // return the token. @@ -270,6 +286,36 @@ impl<'engine, 'source> Lexer<'engine, 'source> { '+' => (Token::Plus, i + 1), '-' => (Token::Minus, i + 1), + // Delimiters + '[' => { + let sp = Span::from(i..(i + c.len_utf8())); + self.delimiters.push((sp, Token::CloseBracket)); + (Token::OpenBracket, i + 1) + } + ']' => { + let (_, close) = self.delimiters.pop().unwrap(); + if close != Token::CloseBracket { + return Err(self + .err_unexpected_token(Token::CloseBracket, i..(i + c.len_utf8()))); + } + (Token::CloseBracket, i + 1) + } + + '{' => { + let sp = Span::from(i..(i + c.len_utf8())); + self.delimiters.push((sp, Token::CloseBrace)); + (Token::OpenBrace, i + 1) + } + '}' => { + let (_, close) = self.delimiters.pop().unwrap(); + if close != Token::CloseBrace { + return Err( + self.err_unexpected_token(Token::CloseBrace, i..(i + c.len_utf8())) + ); + } + (Token::CloseBrace, i + 1) + } + // Multi-character tokens with a distinct start character. '?' => self.lex_question_dot(iter, i)?, '"' => self.lex_string(iter, i)?, @@ -464,6 +510,10 @@ impl Token { Self::EndBlock => "end block", Self::BeginComment => "begin comment", Self::EndComment => "end comment", + Self::OpenBracket => "open bracket", + Self::CloseBracket => "close bracket", + Self::OpenBrace => "open brace", + Self::CloseBrace => "close brace", Self::Dot => "member access operator", Self::QuestionDot => "optional member access operator", Self::Pipe => "pipe", @@ -480,7 +530,7 @@ impl Token { } } - /// Returns the corresponding tag if this token is a tag. + /// Returns the corresponding tag if this token is a tag or delimiter. fn pair(&self) -> Self { match self { Self::BeginExpr => Self::EndExpr, @@ -489,7 +539,11 @@ impl Token { Self::EndBlock => Self::BeginBlock, Self::BeginComment => Self::EndComment, Self::EndComment => Self::BeginComment, - _ => panic!("not a tag"), + Self::OpenBracket => Self::CloseBracket, + Self::CloseBracket => Self::OpenBracket, + Self::OpenBrace => Self::CloseBrace, + Self::CloseBrace => Self::OpenBrace, + _ => panic!("not a tag or delimiter"), } } @@ -653,6 +707,32 @@ mod tests { ); } + #[test] + fn lex_expr_literals() { + let tokens = lex("lorem {{ [1, 3] {.} }} dolor").unwrap(); + assert_eq!( + tokens, + [ + (Token::Raw, "lorem "), + (Token::BeginExpr, "{{"), + (Token::Whitespace, " "), + (Token::OpenBracket, "["), + (Token::Number, "1"), + (Token::Comma, ","), + (Token::Whitespace, " "), + (Token::Number, "3"), + (Token::CloseBracket, "]"), + (Token::Whitespace, " "), + (Token::OpenBrace, "{"), + (Token::Dot, "."), + (Token::CloseBrace, "}"), + (Token::Whitespace, " "), + (Token::EndExpr, "}}"), + (Token::Raw, " dolor") + ] + ); + } + #[cfg(feature = "unicode")] #[test] fn lex_expr() { diff --git a/src/compile/mod.rs b/src/compile/mod.rs index e5f0acc..8ed1773 100644 --- a/src/compile/mod.rs +++ b/src/compile/mod.rs @@ -163,6 +163,20 @@ impl Compiler { ast::BaseExpr::Literal(literal) => { self.push(Instr::ExprStartLiteral(literal)); } + ast::BaseExpr::List(list) => { + self.push(Instr::ExprStartList(list.span)); + for item in list.items { + self.compile_base_expr(item); + self.push(Instr::ExprListPush); + } + } + ast::BaseExpr::Map(map) => { + self.push(Instr::ExprStartMap(map.span)); + for (key, value) in map.items { + self.compile_base_expr(value); + self.push(Instr::ExprMapInsert(key)); + } + } } } @@ -171,7 +185,7 @@ impl Compiler { Some(Instr::Apply(_, _, _)) => { let instr = self.instrs.pop().unwrap(); match instr { - Instr::Apply(ident, len, span) => Instr::EmitWith(ident, len, span), + Instr::Apply(ident, arity, span) => Instr::EmitWith(ident, arity, span), _ => unreachable!(), } } diff --git a/src/compile/parse.rs b/src/compile/parse.rs index 69a690a..2649a6f 100644 --- a/src/compile/parse.rs +++ b/src/compile/parse.rs @@ -448,7 +448,7 @@ impl<'engine, 'source> Parser<'engine, 'source> { let name = self.parse_ident()?; let (args, span) = if self.is_next(Token::Colon)? { let span = self.expect(Token::Colon)?; - let args = self.parse_args(span)?; + let args = self.parse_call_args(span)?; let span = expr.span().combine(args.span); (Some(args), span) } else { @@ -516,6 +516,17 @@ impl<'engine, 'source> Parser<'engine, 'source> { let var = self.parse_var(first)?; ast::BaseExpr::Var(var) } + + (Token::OpenBracket, span) => { + let list = self.parse_list(span)?; + ast::BaseExpr::List(list) + } + + (Token::OpenBrace, span) => { + let map = self.parse_map(span)?; + ast::BaseExpr::Map(map) + } + (tk, span) => { return Err(self.err_unexpected_token("expression", tk, span)); } @@ -599,7 +610,7 @@ impl<'engine, 'source> Parser<'engine, 'source> { /// /// user.name, "a string", true /// - fn parse_args(&mut self, span: Span) -> Result { + fn parse_call_args(&mut self, span: Span) -> Result { let mut values = Vec::new(); loop { values.push(self.parse_base_expr()?); @@ -763,6 +774,44 @@ impl<'engine, 'source> Parser<'engine, 'source> { Ok(string) } + /// Parses a list literal. + fn parse_list(&mut self, span: Span) -> Result { + let mut items = Vec::new(); + loop { + if self.is_next(Token::CloseBracket)? { + break; + } + let item = self.parse_base_expr()?; + items.push(item); + if !self.is_next(Token::Comma)? { + break; + } + self.expect(Token::Comma)?; + } + let span = span.combine(self.expect(Token::CloseBracket)?); + Ok(ast::List { items, span }) + } + + /// Parses a map literal. + fn parse_map(&mut self, span: Span) -> Result { + let mut items = Vec::new(); + loop { + if self.is_next(Token::CloseBrace)? { + break; + } + let key = self.parse_ident()?; + self.expect(Token::Colon)?; + let value = self.parse_base_expr()?; + items.push((key, value)); + if !self.is_next(Token::Comma)? { + break; + } + self.expect(Token::Comma)?; + } + let span = span.combine(self.expect(Token::CloseBrace)?); + Ok(ast::Map { items, span }) + } + /// Expects the given keyword. fn expect_keyword(&mut self, exp: Keyword) -> Result { let (kw, span) = self.parse_keyword()?; diff --git a/src/render/core.rs b/src/render/core.rs index e0067d4..cf21d22 100644 --- a/src/render/core.rs +++ b/src/render/core.rs @@ -8,7 +8,7 @@ use crate::types::ast; use crate::types::program::{Instr, Template}; use crate::types::span::Span; use crate::value::ValueCow; -use crate::{EngineBoxFn, Error, Result}; +use crate::{EngineBoxFn, Error, Result, Value}; #[cfg_attr(internal_debug, derive(Debug))] pub struct RendererImpl<'render, 'stack> { @@ -225,6 +225,37 @@ where exprs.push((value, literal.span)); } + Instr::ExprStartList(span) => { + let value = ValueCow::Owned(crate::Value::new_list()); + exprs.push((value, *span)); + } + + Instr::ExprStartMap(span) => { + let value = ValueCow::Owned(crate::Value::new_map()); + exprs.push((value, *span)); + } + + Instr::ExprListPush => { + let (item, _) = exprs.pop().unwrap(); + match exprs.last_mut().unwrap() { + (ValueCow::Owned(Value::List(l)), _) => { + l.push(item.to_owned()); + } + _ => panic!("expected owned list"), + } + } + + Instr::ExprMapInsert(key) => { + let key = t.source[key.span].to_owned(); + let (value, _) = exprs.pop().unwrap(); + match exprs.last_mut().unwrap() { + (ValueCow::Owned(Value::Map(m)), _) => { + m.insert(key, value.to_owned()); + } + _ => panic!("expected owned map"), + } + } + Instr::Apply(name, _arity, _span) => { let name_raw = &t.source[name.span]; match self.inner.engine.functions.get(name_raw) { diff --git a/src/render/value.rs b/src/render/value.rs index 716467c..0f7cb45 100644 --- a/src/render/value.rs +++ b/src/render/value.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use crate::types::ast; use crate::value::ValueCow; use crate::{Error, Result, Value}; @@ -27,6 +29,14 @@ impl Value { Value::Map(_) => "map", } } + + pub(crate) const fn new_map() -> Self { + Self::Map(BTreeMap::new()) + } + + pub(crate) const fn new_list() -> Self { + Self::List(Vec::new()) + } } /// Lookup the given path. diff --git a/src/types/ast.rs b/src/types/ast.rs index f51777f..09a07db 100644 --- a/src/types/ast.rs +++ b/src/types/ast.rs @@ -99,6 +99,8 @@ pub struct Args { pub enum BaseExpr { Var(Var), Literal(Literal), + List(List), + Map(Map), } #[cfg_attr(internal_debug, derive(Debug))] @@ -145,6 +147,18 @@ pub struct Literal { pub span: Span, } +#[cfg_attr(internal_debug, derive(Debug))] +pub struct List { + pub items: Vec, + pub span: Span, +} + +#[cfg_attr(internal_debug, derive(Debug))] +pub struct Map { + pub items: Vec<(Ident, BaseExpr)>, + pub span: Span, +} + impl Scope { pub const fn new() -> Self { Self { stmts: Vec::new() } @@ -171,6 +185,8 @@ impl BaseExpr { match self { BaseExpr::Var(var) => var.span(), BaseExpr::Literal(lit) => lit.span, + BaseExpr::List(list) => list.span, + BaseExpr::Map(map) => map.span, } } } diff --git a/src/types/program.rs b/src/types/program.rs index 78c1974..58b5575 100644 --- a/src/types/program.rs +++ b/src/types/program.rs @@ -61,6 +61,18 @@ pub enum Instr { /// Start building an expression using a literal ExprStartLiteral(ast::Literal), + /// Start building a list expression + ExprStartList(Span), + + /// Start building a map expression + ExprStartMap(Span), + + /// Append an item to the current list expression + ExprListPush, + + /// Insert an item to the current map expression + ExprMapInsert(ast::Ident), + /// Apply the filter using the value and args on the top of the stack. /// /// The second value is the number of arguments to pop from the stack diff --git a/tests/lex.rs b/tests/lex.rs index cc6d392..8138ee8 100644 --- a/tests/lex.rs +++ b/tests/lex.rs @@ -256,6 +256,39 @@ fn lex_err_undelimited_string_newline() { ); } +#[test] +fn lex_err_unclosed_open_brace() { + let err = Engine::new().compile("lorem {{ { }} dolor").unwrap_err(); + assert_err( + &err, + "unclosed open brace", + r#" + --> :1:10 + | + 1 | lorem {{ { }} dolor + | ^-- + | + = reason: REASON +"#, + ); +} +#[test] +fn lex_err_unclosed_open_bracket() { + let err = Engine::new().compile("lorem {{ [ }} dolor").unwrap_err(); + assert_err( + &err, + "unclosed open bracket", + r#" + --> :1:10 + | + 1 | lorem {{ [ }} dolor + | ^-- + | + = reason: REASON +"#, + ); +} + #[track_caller] fn assert_err(err: &Error, reason: &str, pretty: &str) { let display = format!("invalid syntax: {reason}"); diff --git a/tests/render.rs b/tests/render.rs index f9cbdcd..638c938 100644 --- a/tests/render.rs +++ b/tests/render.rs @@ -120,6 +120,77 @@ fn render_inline_expr_literal_with_filter() { assert_eq!(result, "lorem TEST"); } +#[test] +fn render_inline_expr_literal_list() { + let mut engine = Engine::new(); + engine.add_formatter("debug", debug); + let result = engine + .compile(r#"{{ [true, 123, -3.14, "test", lorem] | debug }}"#) + .unwrap() + .render(&engine, value! { lorem: "ipsum" }) + .to_string() + .unwrap(); + assert_eq!(result, "[true, 123, -3.14, test, ipsum]"); +} + +#[test] +fn render_inline_expr_literal_map() { + let mut engine = Engine::new(); + engine.add_formatter("debug", debug); + let result = engine + .compile(r#"{{ {a: true, b: 123, c: -3.14, d: "test", e: lorem} | debug }}"#) + .unwrap() + .render(&engine, value! { lorem: "ipsum" }) + .to_string() + .unwrap(); + assert_eq!(result, "{a: true, b: 123, c: -3.14, d: test, e: ipsum}"); +} + +#[test] +fn render_inline_expr_nested_lists_and_maps() { + let mut engine = Engine::new(); + engine.add_formatter("debug", debug); + let result = engine + .compile(r#"{{ [true, [123, -3.14], {a: "test", b: lorem, c: [1, 2]}] | debug }}"#) + .unwrap() + .render(&engine, value! { lorem: "ipsum" }) + .to_string() + .unwrap(); + assert_eq!( + result, + "[true, [123, -3.14], {a: test, b: ipsum, c: [1, 2]}]" + ); +} + +fn debug(f: &mut fmt::Formatter<'_>, v: &Value) -> fmt::Result { + match v { + Value::List(list) => { + f.write_char('[')?; + for (i, item) in list.iter().enumerate() { + if i != 0 { + f.write_str(", ")?; + } + debug(f, item)?; + } + f.write_char(']')?; + Ok(()) + } + Value::Map(map) => { + f.write_char('{')?; + for (i, (key, value)) in map.iter().enumerate() { + if i != 0 { + f.write_str(", ")?; + } + write!(f, "{}: ", key)?; + debug(f, value)?; + } + f.write_char('}')?; + Ok(()) + } + _ => fmt::default(f, v), + } +} + #[test] fn render_inline_expr_map_key() { let engine = Engine::new();