From a2413aef5f93ab684a3b7478047b2238781a2ef3 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Wed, 11 Dec 2024 11:56:21 -0500 Subject: [PATCH] Implements `make_struct` (#876) --- src/lazy/expanded/e_expression.rs | 6 +- src/lazy/expanded/macro_evaluator.rs | 63 +++++++++++++- src/lazy/expanded/macro_table.rs | 3 +- src/lazy/expanded/mod.rs | 6 ++ src/lazy/expanded/struct.rs | 119 +++++++++++++++++++++++---- src/lazy/expanded/template.rs | 5 +- src/lazy/struct.rs | 10 ++- 7 files changed, 190 insertions(+), 22 deletions(-) diff --git a/src/lazy/expanded/e_expression.rs b/src/lazy/expanded/e_expression.rs index 0aadd319..992e4a0b 100644 --- a/src/lazy/expanded/e_expression.rs +++ b/src/lazy/expanded/e_expression.rs @@ -11,7 +11,8 @@ use crate::lazy::expanded::compiler::{ExpansionAnalysis, ExpansionSingleton}; use crate::lazy::expanded::macro_evaluator::{ AnnotateExpansion, ConditionalExpansion, EExpressionArgGroup, ExprGroupExpansion, FlattenExpansion, IsExhaustedIterator, MacroExpansion, MacroExpansionKind, MacroExpr, - MacroExprArgsIterator, MakeTextExpansion, RawEExpression, TemplateExpansion, ValueExpr, + MacroExprArgsIterator, MakeStructExpansion, MakeTextExpansion, RawEExpression, + TemplateExpansion, ValueExpr, }; use crate::lazy::expanded::macro_table::{MacroKind, MacroRef}; use crate::lazy::expanded::template::TemplateMacroRef; @@ -123,6 +124,9 @@ impl<'top, D: Decoder> EExpression<'top, D> { MacroKind::MakeSymbol => { MacroExpansionKind::MakeSymbol(MakeTextExpansion::symbol_maker(arguments)) } + MacroKind::MakeStruct => { + MacroExpansionKind::MakeStruct(MakeStructExpansion::new(arguments)) + } MacroKind::Annotate => MacroExpansionKind::Annotate(AnnotateExpansion::new(arguments)), MacroKind::Flatten => MacroExpansionKind::Flatten(FlattenExpansion::new( self.context, diff --git a/src/lazy/expanded/macro_evaluator.rs b/src/lazy/expanded/macro_evaluator.rs index 27e6f5ca..4d209d19 100644 --- a/src/lazy/expanded/macro_evaluator.rs +++ b/src/lazy/expanded/macro_evaluator.rs @@ -35,8 +35,8 @@ use crate::lazy::text::raw::v1_1::arg_group::EExpArg; use crate::lazy::text::raw::v1_1::reader::MacroIdRef; use crate::result::IonFailure; use crate::{ - ExpandedValueRef, ExpandedValueSource, IonError, IonResult, LazyValue, Span, SymbolRef, - ValueRef, + ExpandedValueRef, ExpandedValueSource, IonError, IonResult, LazyExpandedStruct, LazyStruct, + LazyValue, Span, SymbolRef, ValueRef, }; pub trait IsExhaustedIterator<'top, D: Decoder>: @@ -459,6 +459,7 @@ pub enum MacroExpansionKind<'top, D: Decoder> { ExprGroup(ExprGroupExpansion<'top, D>), MakeString(MakeTextExpansion<'top, D>), MakeSymbol(MakeTextExpansion<'top, D>), + MakeStruct(MakeStructExpansion<'top, D>), Annotate(AnnotateExpansion<'top, D>), Flatten(FlattenExpansion<'top, D>), Template(TemplateExpansion<'top>), @@ -540,6 +541,7 @@ impl<'top, D: Decoder> MacroExpansion<'top, D> { ExprGroup(expr_group_expansion) => expr_group_expansion.next(context, environment), MakeString(make_string_expansion) => make_string_expansion.make_text_value(context), MakeSymbol(make_symbol_expansion) => make_symbol_expansion.make_text_value(context), + MakeStruct(make_struct_expansion) => make_struct_expansion.next(context, environment), Annotate(annotate_expansion) => annotate_expansion.next(context, environment), Flatten(flatten_expansion) => flatten_expansion.next(), Conditional(cardiality_test_expansion) => cardiality_test_expansion.next(environment), @@ -556,6 +558,7 @@ impl Debug for MacroExpansion<'_, D> { MacroExpansionKind::ExprGroup(_) => "[internal] expr_group", MacroExpansionKind::MakeString(_) => "make_string", MacroExpansionKind::MakeSymbol(_) => "make_symbol", + MacroExpansionKind::MakeStruct(_) => "make_struct", MacroExpansionKind::Annotate(_) => "annotate", MacroExpansionKind::Flatten(_) => "flatten", MacroExpansionKind::Conditional(test) => test.name(), @@ -1112,6 +1115,40 @@ impl<'top, D: Decoder> ConditionalExpansion<'top, D> { } } +// ===== Implementation of the `make_struct` macro ===== +#[derive(Copy, Clone, Debug)] +pub struct MakeStructExpansion<'top, D: Decoder> { + arguments: MacroExprArgsIterator<'top, D>, +} + +impl<'top, D: Decoder> MakeStructExpansion<'top, D> { + pub fn new(arguments: MacroExprArgsIterator<'top, D>) -> Self { + Self { arguments } + } + + fn next( + &mut self, + context: EncodingContextRef<'top>, + environment: Environment<'top, D>, + ) -> IonResult> { + // The `make_struct` macro always produces a single struct value. When `next()` is called + // to begin its evaluation, it immediately returns a lazy value representing the (not yet + // computed) struct. If/when the application tries to iterate over its fields, + // the iterator will evaluate the field expressions incrementally. + let lazy_expanded_struct = + LazyExpandedStruct::from_constructed(context, environment, self.arguments); + let lazy_struct = LazyStruct::new(lazy_expanded_struct); + // Store the `Struct` in the bump so it's guaranteed to be around as long as the reader is + // positioned on this top-level value. + let value_ref = context + .allocator() + .alloc_with(|| ValueRef::Struct(lazy_struct)); + let lazy_expanded_value = LazyExpandedValue::from_constructed(context, &[], value_ref); + Ok(MacroExpansionStep::FinalStep(Some( + ValueExpr::ValueLiteral(lazy_expanded_value), + ))) + } +} // ===== Implementation of the `make_string` macro ===== #[derive(Copy, Clone, Debug)] @@ -2788,6 +2825,28 @@ mod tests { ) } + #[test] + fn make_struct_eexp() -> IonResult<()> { + stream_eq( + r#" + (:make_struct) + (:make_struct {a: 1}) + (:make_struct {a: 1} {}) + (:make_struct {a: 1} {b: 2}) + (:make_struct {a: 1} {b: 2} {a: 3, d: 4}) + (:make_struct {a: 1} {b: 2} {a: 3, d: 4} (:make_struct {e: 5} {f: 6})) + "#, + r#" + {} + {a: 1} + {a: 1} + {a: 1, b: 2} + {a: 1, b: 2, a: 3, d: 4} + {a: 1, b: 2, a: 3, d: 4, e: 5, f: 6} + "#, + ) + } + #[test] fn make_string_tdl_macro_invocation() -> IonResult<()> { let invocation = r#" diff --git a/src/lazy/expanded/macro_table.rs b/src/lazy/expanded/macro_table.rs index 27d4c43e..de24f40e 100644 --- a/src/lazy/expanded/macro_table.rs +++ b/src/lazy/expanded/macro_table.rs @@ -114,6 +114,7 @@ pub enum MacroKind { ExprGroup, MakeString, MakeSymbol, + MakeStruct, Annotate, Flatten, Template(TemplateBody), @@ -317,7 +318,7 @@ impl MacroTable { builtin( "make_struct", "(fields*)", - MacroKind::ToDo, + MacroKind::MakeStruct, ExpansionAnalysis::single_application_value(IonType::Struct), ), template( diff --git a/src/lazy/expanded/mod.rs b/src/lazy/expanded/mod.rs index 9c1f1a1d..b1df5c22 100644 --- a/src/lazy/expanded/mod.rs +++ b/src/lazy/expanded/mod.rs @@ -1060,6 +1060,12 @@ pub enum ExpandedAnnotationsSource<'top, Encoding: Decoder> { Constructed(std::slice::Iter<'top, SymbolRef<'top>>), } +impl ExpandedAnnotationsSource<'_, Encoding> { + pub fn empty() -> Self { + Self::Constructed(std::slice::Iter::default()) + } +} + pub struct ExpandedAnnotationsIterator<'top, Encoding: Decoder> { source: ExpandedAnnotationsSource<'top, Encoding>, } diff --git a/src/lazy/expanded/struct.rs b/src/lazy/expanded/struct.rs index 1083654d..ab79aa11 100644 --- a/src/lazy/expanded/struct.rs +++ b/src/lazy/expanded/struct.rs @@ -2,7 +2,7 @@ use crate::element::iterators::SymbolsIterator; use crate::lazy::decoder::private::{LazyRawStructPrivate, RawStructUnexpandedFieldsIterator}; use crate::lazy::decoder::{Decoder, LazyRawFieldName, LazyRawStruct}; use crate::lazy::expanded::macro_evaluator::{ - MacroEvaluator, MacroExpansion, MacroExpr, ValueExpr, + MacroEvaluator, MacroExpansion, MacroExpr, MacroExprArgsIterator, ValueExpr, }; use crate::lazy::expanded::sequence::Environment; use crate::lazy::expanded::template::{ @@ -13,7 +13,7 @@ use crate::lazy::expanded::{ LazyExpandedValue, }; use crate::result::IonFailure; -use crate::{try_or_some_err, IonResult, RawSymbolRef, SymbolRef}; +use crate::{try_next, try_or_some_err, IonResult, RawSymbolRef, SymbolRef}; /// A unified type embodying all possible field representations coming from both input data /// (i.e. raw structs of some encoding) and template bodies. @@ -48,6 +48,10 @@ impl<'top, D: Decoder> LazyExpandedField<'top, D> { pub fn name(&self) -> LazyExpandedFieldName<'top, D> { self.name } + + pub fn unexpanded(&self) -> UnexpandedField<'top, D> { + UnexpandedField::NameValue(self.name(), self.value()) + } } impl<'top, D: Decoder> LazyExpandedField<'top, D> { @@ -107,7 +111,8 @@ pub enum ExpandedStructSource<'top, D: Decoder> { TemplateElement<'top>, &'top TemplateStructIndex, ), - // TODO: Constructed + // The struct was produced by the `make_struct` macro. + Constructed(Environment<'top, D>, MacroExprArgsIterator<'top, D>), } impl<'top, D: Decoder> ExpandedStructSource<'top, D> { @@ -115,6 +120,7 @@ impl<'top, D: Decoder> ExpandedStructSource<'top, D> { match self { ExpandedStructSource::ValueLiteral(_) => Environment::empty(), ExpandedStructSource::Template(environment, _, _) => *environment, + ExpandedStructSource::Constructed(environment, _) => *environment, } } } @@ -154,18 +160,27 @@ impl<'top, D: Decoder> LazyExpandedStruct<'top, D> { Self { source, context } } + pub fn from_constructed( + context: EncodingContextRef<'top>, + environment: Environment<'top, D>, + arguments: MacroExprArgsIterator<'top, D>, + ) -> LazyExpandedStruct<'top, D> { + let source = ExpandedStructSource::Constructed(environment, arguments); + Self { source, context } + } + pub fn annotations(&self) -> ExpandedAnnotationsIterator<'top, D> { - match &self.source { - ExpandedStructSource::ValueLiteral(value) => ExpandedAnnotationsIterator { - source: ExpandedAnnotationsSource::ValueLiteral(value.annotations()), - }, + let iter_source = match &self.source { + ExpandedStructSource::ValueLiteral(value) => { + ExpandedAnnotationsSource::ValueLiteral(value.annotations()) + } ExpandedStructSource::Template(_environment, element, _index) => { let annotations = element.annotations(); - ExpandedAnnotationsIterator { - source: ExpandedAnnotationsSource::Template(SymbolsIterator::new(annotations)), - } + ExpandedAnnotationsSource::Template(SymbolsIterator::new(annotations)) } - } + ExpandedStructSource::Constructed(_, _) => ExpandedAnnotationsSource::empty(), + }; + ExpandedAnnotationsIterator::new(iter_source) } pub fn iter(&self) -> ExpandedStructIterator<'top, D> { @@ -197,6 +212,18 @@ impl<'top, D: Decoder> LazyExpandedStruct<'top, D> { ), ) } + ExpandedStructSource::Constructed(environment, arguments) => { + let evaluator = self + .context + .allocator() + .alloc_with(|| MacroEvaluator::new_with_environment(*environment)); + let current_struct_iter = self.context.allocator().alloc_with(|| None); + ExpandedStructIteratorSource::Constructed( + evaluator, + current_struct_iter, + *arguments, + ) + } }; ExpandedStructIterator { source, @@ -214,9 +241,9 @@ impl<'top, D: Decoder> LazyExpandedStruct<'top, D> { pub fn find(&self, name: &str) -> IonResult>> { match &self.source { - // If we're reading from a struct literal, do a linear scan over its fields until we - // encounter one with the requested name. - ExpandedStructSource::ValueLiteral(_) => { + // If we're reading from a struct literal or `make_struct` call, do a linear scan over + // its fields until we encounter one with the requested name. + ExpandedStructSource::ValueLiteral(_) | ExpandedStructSource::Constructed(_, _) => { for field_result in self.iter() { let field = field_result?; if field.name().read()?.text() == Some(name) { @@ -287,7 +314,14 @@ pub enum ExpandedStructIteratorSource<'top, D: Decoder> { &'top mut MacroEvaluator<'top, D>, TemplateStructUnexpandedFieldsIterator<'top, D>, ), - // TODO: Constructed + Constructed( + &'top mut MacroEvaluator<'top, D>, + // This is `&mut Option<_>` instead of `Option<&mut _>` so we can re-use the allocated space + // for each iterator we traverse. + &'top mut Option>, + // Remaining argument expressions + MacroExprArgsIterator<'top, D>, + ), } impl<'top, D: Decoder> ExpandedStructIteratorSource<'top, D> { @@ -300,13 +334,68 @@ impl<'top, D: Decoder> ExpandedStructIteratorSource<'top, D> { ExpandedStructIteratorSource::ValueLiteral(_, raw_struct_iter) => { raw_struct_iter.next() } + ExpandedStructIteratorSource::Constructed( + evaluator, + maybe_current_struct, + arguments, + ) => { + loop { + // If we're already traversing a struct, see if it has any fields remaining. + if let Some(current_struct) = maybe_current_struct { + match current_struct.next() { + // If we get a field, we're done. + Some(Ok(field)) => return Some(Ok(field.unexpanded())), + Some(Err(e)) => return Some(Err(e)), + // If we get `None`, the iterator is exhausted and we should continue on to the next struct. + None => **maybe_current_struct = None, + } + } + + // If we reach this point, we don't have a current struct. + // We've either just started evaluation and haven't set one yet or + // we just finished inlining a struct and need to set a new one. + + // See if the evaluator has an expansion in progress. + let mut next_struct = try_or_some_err!(evaluator.next()); + if next_struct.is_none() { + // If we don't get anything from the evaluator, we'll get our struct from the + // next argument expression. If there isn't a next argument expression, + // then evaluation is complete. + next_struct = match try_next!(arguments.next()) { + // If the expression is a value literal, that's our new sequence. + ValueExpr::ValueLiteral(value) => Some(value), + // If the expression is a macro invocation, we'll start evaluating it + // and return to the top of the loop. + ValueExpr::MacroInvocation(invocation) => { + evaluator.push(try_or_some_err!(invocation.expand())); + continue; + } + } + } + + // At this point, `next_struct` is definitely populated, so we can safely unwrap it. + let next_struct = next_struct.unwrap(); + // Set it as our new current struct. + let ExpandedValueRef::Struct(next_struct) = + try_or_some_err!(next_struct.read()) + else { + return Some(IonResult::decoding_error(format!( + "`make_struct` only accepts structs, received {next_struct:?}" + ))); + }; + **maybe_current_struct = Some(next_struct.iter()); + } + } } } fn evaluator(&mut self) -> &mut MacroEvaluator<'top, D> { + // TODO: Should the evaluator be lifted out of the iterator source? + // It is common to all of the variants. match self { ExpandedStructIteratorSource::Template(evaluator, _) => evaluator, ExpandedStructIteratorSource::ValueLiteral(evaluator, _) => evaluator, + ExpandedStructIteratorSource::Constructed(evaluator, _, _) => evaluator, } } } diff --git a/src/lazy/expanded/template.rs b/src/lazy/expanded/template.rs index d4233159..8c70ac0c 100644 --- a/src/lazy/expanded/template.rs +++ b/src/lazy/expanded/template.rs @@ -8,7 +8,7 @@ use compact_str::CompactString; use crate::lazy::binary::raw::v1_1::immutable_buffer::ArgGroupingBitmap; use crate::lazy::decoder::Decoder; use crate::lazy::expanded::compiler::ExpansionAnalysis; -use crate::lazy::expanded::macro_evaluator::{AnnotateExpansion, MacroEvaluator, MacroExpansion, MacroExpansionKind, MacroExpr, MacroExprArgsIterator, TemplateExpansion, ValueExpr, ExprGroupExpansion, MakeTextExpansion, FlattenExpansion, ConditionalExpansion}; +use crate::lazy::expanded::macro_evaluator::{AnnotateExpansion, MacroEvaluator, MacroExpansion, MacroExpansionKind, MacroExpr, MacroExprArgsIterator, TemplateExpansion, ValueExpr, ExprGroupExpansion, MakeTextExpansion, FlattenExpansion, ConditionalExpansion, MakeStructExpansion}; use crate::lazy::expanded::macro_table::{Macro, MacroKind}; use crate::lazy::expanded::r#struct::UnexpandedField; use crate::lazy::expanded::sequence::Environment; @@ -1221,6 +1221,9 @@ impl<'top, D: Decoder> TemplateMacroInvocation<'top, D> { MacroKind::MakeSymbol => { MacroExpansionKind::MakeSymbol(MakeTextExpansion::symbol_maker(arguments)) } + MacroKind::MakeStruct => { + MacroExpansionKind::MakeStruct(MakeStructExpansion::new(arguments)) + } MacroKind::Annotate => MacroExpansionKind::Annotate(AnnotateExpansion::new(arguments)), MacroKind::Flatten => MacroExpansionKind::Flatten(FlattenExpansion::new(self.context(), self.environment, arguments)), MacroKind::Template(template_body) => { diff --git a/src/lazy/struct.rs b/src/lazy/struct.rs index d6906e54..79bf187d 100644 --- a/src/lazy/struct.rs +++ b/src/lazy/struct.rs @@ -93,12 +93,18 @@ impl<'top, D: Decoder> LazyStruct<'top, D> { } pub fn as_value(&self) -> LazyValue<'top, D> { + let context = self.expanded_struct.context; let expanded_value = match self.expanded_struct.source { ExpandedStructSource::ValueLiteral(v) => { - LazyExpandedValue::from_literal(self.expanded_struct.context, v.as_value()) + LazyExpandedValue::from_literal(context, v.as_value()) } ExpandedStructSource::Template(env, element, _) => { - LazyExpandedValue::from_template(self.expanded_struct.context, env, element) + LazyExpandedValue::from_template(context, env, element) + } + ExpandedStructSource::Constructed(_env, _args) => { + let value_ref = context.allocator().alloc_with(|| ValueRef::Struct(*self)); + let annotations = &[]; + LazyExpandedValue::from_constructed(context, annotations, value_ref) } }; LazyValue::new(expanded_value)