diff --git a/src/lazy/any_encoding.rs b/src/lazy/any_encoding.rs index 26f7cf87..32fbb587 100644 --- a/src/lazy/any_encoding.rs +++ b/src/lazy/any_encoding.rs @@ -79,6 +79,9 @@ impl<'data> MacroInvocation<'data, AnyEncoding> for LazyRawAnyMacroInvocation<'d type ArgumentExpr = LazyRawValueExpr<'data, AnyEncoding>; type ArgumentsIterator = LazyRawAnyMacroArgsIterator<'data>; + type TransientEvaluator<'context> = + TransientEExpEvaluator<'context, 'data, AnyEncoding> where Self: 'context, 'data: 'context; + fn id(&self) -> MacroIdRef<'_> { match self.encoding { LazyRawAnyMacroInvocationKind::Text_1_0(_) => unreachable!("macro in text Ion 1.0"), @@ -97,16 +100,14 @@ impl<'data> MacroInvocation<'data, AnyEncoding> for LazyRawAnyMacroInvocation<'d } } - type TransientEvaluator<'top> = TransientEExpEvaluator<'top, 'data, AnyEncoding> where 'data: 'top, Self: 'top; - - fn make_transient_evaluator<'top>( - context: EncodingContext<'top>, - ) -> Self::TransientEvaluator<'top> + fn make_transient_evaluator<'context>( + context: EncodingContext<'context>, + ) -> Self::TransientEvaluator<'context> where - Self: 'top, - 'data: 'top, + 'data: 'context, + Self: 'context, { - TransientEExpEvaluator::new(context) + Self::TransientEvaluator::new(context) } } diff --git a/src/lazy/decoder.rs b/src/lazy/decoder.rs index e7288259..7d6d2906 100644 --- a/src/lazy/decoder.rs +++ b/src/lazy/decoder.rs @@ -183,6 +183,9 @@ impl<'data> MacroInvocation<'data, TextEncoding_1_1> for RawTextMacroInvocation< type ArgumentExpr = LazyRawValueExpr<'data, TextEncoding_1_1>; type ArgumentsIterator = RawTextSExpIterator_1_1<'data>; + type TransientEvaluator<'context> = + TransientEExpEvaluator<'context, 'data, TextEncoding_1_1> where Self: 'context, 'data: 'context; + fn id(&self) -> MacroIdRef { self.id } @@ -191,16 +194,13 @@ impl<'data> MacroInvocation<'data, TextEncoding_1_1> for RawTextMacroInvocation< RawTextSExpIterator_1_1::new(self.arguments_bytes()) } - type TransientEvaluator<'top> = TransientEExpEvaluator<'top, 'data, TextEncoding_1_1> where 'data: 'top, Self: 'top; - - fn make_transient_evaluator<'top>( - context: EncodingContext<'top>, - ) -> Self::TransientEvaluator<'top> + fn make_transient_evaluator<'context>( + context: EncodingContext<'context>, + ) -> Self::TransientEvaluator<'context> where - Self: 'top, - 'data: 'top, + Self: 'context, { - TransientEExpEvaluator::new(context) + Self::TransientEvaluator::new(context) } } diff --git a/src/lazy/expanded/compiler.rs b/src/lazy/expanded/compiler.rs index ebb25fb9..d7b2c168 100644 --- a/src/lazy/expanded/compiler.rs +++ b/src/lazy/expanded/compiler.rs @@ -4,8 +4,8 @@ use std::ops::Range; use crate::lazy::decoder::LazyDecoder; use crate::lazy::expanded::template::{ - MacroSignature, Parameter, ParameterEncoding, TemplateBody, TemplateElement, - TemplateExpansionStep, TemplateMacro, TemplateValue, + ExprRange, MacroSignature, Parameter, ParameterEncoding, TemplateBody, TemplateBodyElement, + TemplateBodyMacroInvocation, TemplateBodyValueExpr, TemplateMacro, TemplateValue, }; use crate::lazy::expanded::EncodingContext; use crate::lazy::r#struct::LazyStruct; @@ -44,10 +44,11 @@ impl TemplateCompiler { /// ] /// ) /// ``` - /// The step corresponding to `(values 2 3)` would store a `2`, indicating that to skip the + /// + /// the step corresponding to `(values 2 3)` would store a `2`, indicating that to skip the /// macro invocation altogether one would need to skip the next two expansion steps. The outer /// list (`[1, (values 2 3)]`) would store a `4`, indicating that it contains the `1`, a macro - /// expansion `values`, and the arguments that belong to `values`. + /// expansion `values`, and the two arguments that belong to `values`. /// /// The compiler recognizes the `(quote expr1 expr2 [...] exprN)` form, adding each subexpression /// to the template without interpretation. @@ -91,7 +92,7 @@ impl TemplateCompiler { let signature = MacroSignature::new(compiled_params); let body = values.next().expect("template body")?; let mut compiled_body = TemplateBody { - expansion_steps: Vec::new(), + expressions: Vec::new(), annotations_storage: Vec::new(), }; Self::compile_value( @@ -109,7 +110,7 @@ impl TemplateCompiler { } /// Recursively visits all of the expressions in `lazy_value` and adds their corresponding - /// [`TemplateExpansionStep`] sequences to the `TemplateBody`. + /// [`TemplateBodyValueExpr`] sequences to the `TemplateBody`. /// /// If `is_quoted` is true, nested symbols and s-expressions will not be interpreted. fn compile_value<'top, 'data, D: LazyDecoder<'data>>( @@ -178,8 +179,9 @@ impl TemplateCompiler { ) } }; - definition - .push_element(TemplateElement::with_value(value).with_annotations(annotations_range)); + definition.push_element( + TemplateBodyElement::with_value(value).with_annotations(annotations_range), + ); Ok(()) } @@ -192,25 +194,22 @@ impl TemplateCompiler { annotations_range: Range, lazy_list: LazyList<'top, 'data, D>, ) -> IonResult<()> { - let list_element_index = definition.expansion_steps.len(); + let list_element_index = definition.expressions.len(); // Assume the list contains zero expressions to start, we'll update this at the end - let list_element = - TemplateElement::with_value(TemplateValue::List(0)).with_annotations(annotations_range); + let list_element = TemplateBodyElement::with_value(TemplateValue::List(ExprRange::empty())); definition.push_element(list_element); - let list_children_start = definition.expansion_steps.len(); + let list_children_start = definition.expressions.len(); for value_result in &lazy_list { let value = value_result?; Self::compile_value(context, signature, definition, is_quoted, value)?; } - let list_children_end = definition.expansion_steps.len(); - let number_of_children = list_children_end - list_children_start; + let list_children_end = definition.expressions.len(); // Update the list entry to reflect the number of child expressions it contains - match &mut definition.expansion_steps.get_mut(list_element_index) { - Some(TemplateExpansionStep::Element(e)) => { - let _ = std::mem::replace(&mut e.value, TemplateValue::List(number_of_children)); - } - _ => unreachable!("list element was unexpectedly overwritten"), - } + let list_element = TemplateBodyElement::with_value(TemplateValue::List(ExprRange::new( + list_children_start..list_children_end, + ))) + .with_annotations(annotations_range); + definition.expressions[list_element_index] = TemplateBodyValueExpr::Element(list_element); Ok(()) } @@ -257,27 +256,26 @@ impl TemplateCompiler { ) -> IonResult<()> { let mut expressions = lazy_sexp.iter(); let macro_address = Self::address_from_id_expr(context, expressions.next())?; - let macro_step_index = definition.expansion_steps.len(); + let macro_step_index = definition.expressions.len(); // Assume the macro contains zero argument expressions to start, we'll update // this at the end of the function. - definition.push_macro_invocation(macro_address, 0); - let arguments_start = definition.expansion_steps.len(); + definition.push_macro_invocation(macro_address, ExprRange::empty()); + let arguments_start = definition.expressions.len(); for argument_result in expressions { let argument = argument_result?; Self::compile_value( context, signature, definition, /*is_quoted=*/ false, argument, )?; } - let arguments_end = definition.expansion_steps.len(); - let number_of_arguments = arguments_end - arguments_start; + let arguments_end = definition.expressions.len(); // Update the macro step to reflect the macro's address and number of child expressions it // contains - match &mut definition.expansion_steps.get_mut(macro_step_index) { - Some(TemplateExpansionStep::MacroInvocation(_address, num_args)) => { - *num_args = number_of_arguments; - } - _ => unreachable!("sexp element was unexpectedly overwritten"), - } + let template_macro_invocation = TemplateBodyMacroInvocation::new( + macro_address, + ExprRange::new(arguments_start..arguments_end), + ); + definition.expressions[macro_step_index] = + TemplateBodyValueExpr::MacroInvocation(template_macro_invocation); Ok(()) } @@ -303,11 +301,9 @@ impl TemplateCompiler { ValueRef::Int(int) => usize::try_from(int.expect_i64()?).map_err(|_| { IonError::decoding_error(format!("found an invalid macro address: {int}")) }), - other => { - return IonResult::decoding_error(format!( - "expected a macro name (symbol) or address (int), but found: {other:?}" - )) - } + other => IonResult::decoding_error(format!( + "expected a macro name (symbol) or address (int), but found: {other:?}" + )), }, } } @@ -345,27 +341,24 @@ impl TemplateCompiler { annotations_range: Range, lazy_sexp: LazySExp<'top, 'data, D>, ) -> IonResult<()> { - let sexp_element_index = definition.expansion_steps.len(); + let sexp_element_index = definition.expressions.len(); // Assume the sexp contains zero expressions to start, we'll update this at the end - let sexp_element = - TemplateElement::with_value(TemplateValue::SExp(0)).with_annotations(annotations_range); + let sexp_element = TemplateBodyElement::with_value(TemplateValue::SExp(ExprRange::empty())); definition.push_element(sexp_element); - let sexp_children_start = definition.expansion_steps.len(); + let sexp_children_start = definition.expressions.len(); for value_result in &lazy_sexp { let value = value_result?; Self::compile_value( context, signature, definition, /*is_quoted=*/ true, value, )?; } - let sexp_children_end = definition.expansion_steps.len(); - let number_of_children = sexp_children_end - sexp_children_start; + let sexp_children_end = definition.expressions.len(); + let sexp_element = TemplateBodyElement::with_value(TemplateValue::SExp(ExprRange::new( + sexp_children_start..sexp_children_end, + ))) + .with_annotations(annotations_range); // Update the sexp entry to reflect the number of child expressions it contains - match &mut definition.expansion_steps.get_mut(sexp_element_index) { - Some(TemplateExpansionStep::Element(e)) => { - let _ = std::mem::replace(&mut e.value, TemplateValue::SExp(number_of_children)); - } - _ => unreachable!("sexp element was unexpectedly overwritten"), - } + definition.expressions[sexp_element_index] = TemplateBodyValueExpr::Element(sexp_element); Ok(()) } @@ -378,7 +371,7 @@ impl TemplateCompiler { match first_expr { // If the sexp is empty and we're not in a quoted context, that's an error. None => IonResult::decoding_error("found an empty s-expression in an unquoted context"), - Some(Err(e)) => return Err(e), + Some(Err(e)) => Err(e), Some(Ok(lazy_value)) => { let value = lazy_value.read()?; Ok(value == ValueRef::Symbol("quote".as_symbol_ref())) @@ -395,29 +388,27 @@ impl TemplateCompiler { annotations_range: Range, lazy_struct: LazyStruct<'top, 'data, D>, ) -> IonResult<()> { - let struct_element_index = definition.expansion_steps.len(); + let struct_element_index = definition.expressions.len(); // Assume the struct contains zero expressions to start, we'll update this at the end - let struct_element = TemplateElement::with_value(TemplateValue::Struct(0)) - .with_annotations(annotations_range); + let struct_element = + TemplateBodyElement::with_value(TemplateValue::Struct(ExprRange::empty())); definition.push_element(struct_element); - let struct_start = definition.expansion_steps.len(); + let struct_start = definition.expressions.len(); for field_result in &lazy_struct { let field = field_result?; let name = field.name()?.to_owned(); - let name_element = TemplateElement::with_value(TemplateValue::Symbol(name)); + let name_element = TemplateBodyElement::with_value(TemplateValue::Symbol(name)); definition.push_element(name_element); Self::compile_value(context, signature, definition, is_quoted, field.value())?; } - let struct_end = definition.expansion_steps.len(); - // NOTE: this is the number of *children*, not the number of *fields*. A field value - // may be a container with many children of its own. - let number_of_children = struct_end - struct_start; - match &mut definition.expansion_steps.get_mut(struct_element_index) { - Some(TemplateExpansionStep::Element(e)) => { - let _ = std::mem::replace(&mut e.value, TemplateValue::Struct(number_of_children)); - } - _ => unreachable!("struct element was unexpectedly overwritten"), - } + let struct_end = definition.expressions.len(); + // Update the struct entry to reflect the range of expansion steps it contains. + let struct_element = TemplateBodyElement::with_value(TemplateValue::Struct( + ExprRange::new(struct_start..struct_end), + )) + .with_annotations(annotations_range); + definition.expressions[struct_element_index] = + TemplateBodyValueExpr::Element(struct_element); Ok(()) } @@ -445,7 +436,7 @@ impl TemplateCompiler { .ok_or_else(|| { IonError::decoding_error(format!("variable '{name}' is not recognized")) })?; - definition.push_variable(name, signature_index); + definition.push_variable(signature_index); Ok(()) } } @@ -454,7 +445,10 @@ impl TemplateCompiler { mod tests { use crate::lazy::expanded::compiler::TemplateCompiler; use crate::lazy::expanded::macro_table::MacroTable; - use crate::lazy::expanded::template::{TemplateExpansionStep, TemplateMacro, TemplateValue}; + use crate::lazy::expanded::template::{ + ExprRange, TemplateBodyMacroInvocation, TemplateBodyValueExpr, + TemplateBodyVariableReference, TemplateMacro, TemplateValue, + }; use crate::lazy::expanded::EncodingContext; use crate::{Int, IntoAnnotations, IonResult, Symbol, SymbolTable}; @@ -467,11 +461,11 @@ mod tests { ) -> IonResult<()> { let actual = definition .body() - .expansion_steps() + .expressions() .get(index) .expect("no such expansion step") .expect_element() - .expect(&format!("expected value {expected:?}")); + .unwrap_or_else(|_| panic!("expected value {expected:?}")); assert_eq!(actual.value(), &expected); Ok(()) } @@ -485,31 +479,36 @@ mod tests { expect_step( definition, index, - TemplateExpansionStep::MacroInvocation(expected_address, expected_num_arguments), + TemplateBodyValueExpr::MacroInvocation(TemplateBodyMacroInvocation::new( + expected_address, + // The arg range starts just after the macro invocation step and goes for `expected_num_arguments`. + ExprRange::new(index + 1..index + 1 + expected_num_arguments), + )), ) } fn expect_variable( definition: &TemplateMacro, index: usize, - expected_name: &str, expected_signature_index: usize, ) -> IonResult<()> { expect_step( definition, index, - TemplateExpansionStep::Variable(expected_name.to_owned(), expected_signature_index), + TemplateBodyValueExpr::Variable(TemplateBodyVariableReference::new( + expected_signature_index, + )), ) } fn expect_step( definition: &TemplateMacro, index: usize, - expected: TemplateExpansionStep, + expected: TemplateBodyValueExpr, ) -> IonResult<()> { let step = definition .body() - .expansion_steps() + .expressions() .get(index) .expect("no such expansion step"); assert_eq!(step, &expected); @@ -523,7 +522,7 @@ mod tests { ) { let element = definition .body - .expansion_steps() + .expressions() .get(index) .expect("requested index does not exist") .expect_element() @@ -531,7 +530,7 @@ mod tests { let actual_annotations = definition .body .annotations_storage() - .get(element.annotations_range.start as usize..element.annotations_range.end as usize) + .get(element.annotations_range().ops_range()) .expect("invalid annotations range") .into_annotations(); let expected_annotations = expected.into_annotations(); @@ -586,7 +585,7 @@ mod tests { let template = TemplateCompiler::compile_from_text(context, expression)?; assert_eq!(template.name(), "foo"); assert_eq!(template.signature().parameters().len(), 0); - expect_value(&template, 0, TemplateValue::List(3))?; + expect_value(&template, 0, TemplateValue::List(ExprRange::new(1..4)))?; expect_value(&template, 1, TemplateValue::Int(1.into()))?; expect_value(&template, 2, TemplateValue::Int(2.into()))?; expect_value(&template, 3, TemplateValue::Int(3.into()))?; @@ -623,19 +622,19 @@ mod tests { let expression = "(macro foo (x y z) [100, [200, a::b::300], x, {y: [true, false, z]}])"; let template = TemplateCompiler::compile_from_text(context, expression)?; - expect_value(&template, 0, TemplateValue::List(11))?; + expect_value(&template, 0, TemplateValue::List(ExprRange::new(1..12)))?; expect_value(&template, 1, TemplateValue::Int(Int::from(100)))?; - expect_value(&template, 2, TemplateValue::List(2))?; + expect_value(&template, 2, TemplateValue::List(ExprRange::new(3..5)))?; expect_value(&template, 3, TemplateValue::Int(Int::from(200)))?; expect_value(&template, 4, TemplateValue::Int(Int::from(300)))?; expect_annotations(&template, 4, ["a", "b"]); - expect_variable(&template, 5, "x", 0)?; - expect_value(&template, 6, TemplateValue::Struct(5))?; + expect_variable(&template, 5, 0)?; + expect_value(&template, 6, TemplateValue::Struct(ExprRange::new(7..12)))?; expect_value(&template, 7, TemplateValue::Symbol(Symbol::from("y")))?; - expect_value(&template, 8, TemplateValue::List(3))?; + expect_value(&template, 8, TemplateValue::List(ExprRange::new(9..12)))?; expect_value(&template, 9, TemplateValue::Bool(true))?; expect_value(&template, 10, TemplateValue::Bool(false))?; - expect_variable(&template, 11, "z", 2)?; + expect_variable(&template, 11, 2)?; Ok(()) } @@ -649,7 +648,7 @@ mod tests { let template = TemplateCompiler::compile_from_text(context, expression)?; assert_eq!(template.name(), "identity"); assert_eq!(template.signature().parameters().len(), 1); - expect_variable(&template, 0, "x", 0)?; + expect_variable(&template, 0, 0)?; Ok(()) } @@ -686,10 +685,10 @@ mod tests { context.macro_table.address_for_name("values").unwrap(), 1, )?; - expect_variable(&template, 2, "x", 0)?; + expect_variable(&template, 2, 0)?; // Second argument: `(quote (values x))` // Notice that the `quote` is not part of the compiled output, only its arguments - expect_value(&template, 3, TemplateValue::SExp(2))?; + expect_value(&template, 3, TemplateValue::SExp(ExprRange::new(4..6)))?; expect_value(&template, 4, TemplateValue::Symbol("values".into()))?; expect_value(&template, 5, TemplateValue::Symbol("x".into()))?; diff --git a/src/lazy/expanded/e_expression.rs b/src/lazy/expanded/e_expression.rs index b1d87a81..a80d937c 100644 --- a/src/lazy/expanded/e_expression.rs +++ b/src/lazy/expanded/e_expression.rs @@ -14,6 +14,7 @@ impl<'data, D: LazyDecoder<'data>> ToArgumentKind<'data, D, D::MacroInvocation> context: EncodingContext<'top>, ) -> ArgumentKind<'top, 'data, D, D::MacroInvocation> where + D::MacroInvocation: 'top, Self: 'top, { match self { diff --git a/src/lazy/expanded/macro_evaluator.rs b/src/lazy/expanded/macro_evaluator.rs index 3c04a8c0..c1b08885 100644 --- a/src/lazy/expanded/macro_evaluator.rs +++ b/src/lazy/expanded/macro_evaluator.rs @@ -19,34 +19,37 @@ use bumpalo::collections::{String as BumpString, Vec as BumpVec}; use crate::lazy::decoder::LazyDecoder; use crate::lazy::expanded::macro_table::MacroKind; use crate::lazy::expanded::stack::Stack; +use crate::lazy::expanded::template::{ + TemplateBodyVariableReference, TemplateMacroInvocationAddress, +}; use crate::lazy::expanded::EncodingContext; use crate::lazy::expanded::{ExpandedValueRef, ExpandedValueSource, LazyExpandedValue}; use crate::lazy::str_ref::StrRef; use crate::lazy::text::raw::v1_1::reader::MacroIdRef; use crate::result::IonFailure; -use crate::{IonError, IonResult, RawSymbolTokenRef, Sequence}; +use crate::{IonError, IonResult, RawSymbolTokenRef}; /// A syntactic entity that represents the invocation of a macro in some context. /// /// This entity may be an item from a binary stream, a text stream, or a template definition. /// Implementors must specify how their type can be mapped to a macro ID and a sequence of arguments. -pub trait MacroInvocation<'data, D: LazyDecoder<'data>>: Copy + Clone + Debug { +pub trait MacroInvocation<'data, D: LazyDecoder<'data>>: Debug + Copy + Clone { /// A syntax-specific type that represents an argument in this macro invocation. type ArgumentExpr: ToArgumentKind<'data, D, Self>; /// An iterator that yields the macro invocation's arguments in order. type ArgumentsIterator: Iterator>; - type TransientEvaluator<'top>: MacroEvaluator< - 'top, + type TransientEvaluator<'context>: MacroEvaluator< + 'context, 'data, D, MacroInvocation = Self, - Stack = BumpVec<'top, MacroExpansion<'data, D, Self>>, + Stack = BumpVec<'context, MacroExpansion<'data, D, Self>>, > where - 'data: 'top, - Self: 'top; + Self: 'context, + 'data: 'context; /// The macro name or address specified at the head of this macro invocation. fn id(&self) -> MacroIdRef; @@ -54,20 +57,20 @@ pub trait MacroInvocation<'data, D: LazyDecoder<'data>>: Copy + Clone + Debug { /// The arguments that follow the macro name or address in this macro invocation. fn arguments(&self) -> Self::ArgumentsIterator; - fn make_transient_evaluator<'top>( - context: EncodingContext<'top>, - ) -> Self::TransientEvaluator<'top> + fn make_transient_evaluator<'context>( + context: EncodingContext<'context>, + ) -> Self::TransientEvaluator<'context> where - 'data: 'top, - Self: 'top; + Self: 'context, + 'data: 'context; } /// A single expression appearing in argument position within a macro invocation. -pub enum ArgumentKind<'top, 'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { +pub enum ArgumentKind<'top, 'data: 'top, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { /// An Ion value that requires no further evaluation. ValueLiteral(LazyExpandedValue<'top, 'data, D>), /// A variable name that requires expansion. - Variable(RawSymbolTokenRef<'top>), + Variable(TemplateBodyVariableReference), /// A macro invocation that requires evaluation. MacroInvocation(M), } @@ -77,9 +80,55 @@ pub enum ArgumentKind<'top, 'data, D: LazyDecoder<'data>, M: MacroInvocation<'da pub trait ToArgumentKind<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { fn to_arg_expr<'top>(self, context: EncodingContext<'top>) -> ArgumentKind<'top, 'data, D, M> where + M: 'top, Self: 'top; } +pub struct VariableExpansion<'top, 'data, D: LazyDecoder<'data>> { + context: EncodingContext<'top>, + variable_name: &'top str, // useful? + args_iter: >::ArgumentsIterator, + evaluator: TransientEExpEvaluator<'top, 'data, D>, +} + +impl<'top, 'data, D: LazyDecoder<'data>> VariableExpansion<'top, 'data, D> { + fn next(&mut self) -> Option>> { + loop { + // If the evaluator's stack is not empty, it's still expanding a macro. + if self.evaluator.stack_depth() > 0 { + let value = self.evaluator.next(self.context, 0).transpose(); + if value.is_some() { + // The `Some` may contain a value or an error; either way, that's the next return value. + return value; + } + // It's possible for a macro to produce zero values. If that happens, we continue on to + // pull another expression from the arguments iterator. + } + + let next_arg_expr = match self.args_iter.next()? { + Ok(expr) => expr, + Err(e) => return Some(Err(e)), + }; + + let arg_kind: ArgumentKind = + next_arg_expr.to_arg_expr(self.context); + match arg_kind { + ArgumentKind::ValueLiteral(value) => return Some(Ok(value)), + ArgumentKind::MacroInvocation(invocation) => { + let begin_expansion_result = self.evaluator.push(self.context, invocation); + if let Err(e) = begin_expansion_result { + return Some(Err(e)); + } + continue; + } + ArgumentKind::Variable(_) => { + unreachable!("e-expressions cannot reference variables"); + } + } + } + } +} + /// Indicates which of the supported macros this represents and stores the state necessary to /// continue evaluating that macro. pub enum MacroExpansionKind<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { @@ -88,15 +137,16 @@ pub enum MacroExpansionKind<'data, D: LazyDecoder<'data>, M: MacroInvocation<'da MakeString(MakeStringExpansion<'data, D, M>), Template(String), // TODO: The others, including template macros. - // TODO: Treat variables as a special kind of macro invocation, similar to `values` but without - // an accessible entry in the macro table. } /// A macro in the process of being evaluated. Stores both the state of the evaluation and the /// syntactic element that represented the macro invocation. pub struct MacroExpansion<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { + // TODO: split this into two variants: + // a user variant (the one that exists) and a system variant (no `M`, secret kinds like variable expansion) kind: MacroExpansionKind<'data, D, M>, - invocation: M, + // Not every macro expansion is initiated by + invocation: Option, } impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> Debug @@ -142,6 +192,25 @@ pub enum MacroExpansionStep<'top, 'data, D: LazyDecoder<'data>, M: MacroInvocati Complete, } +// pub trait TransientMacroEvaluator< +// 'context, +// 'data: 'context, +// D: LazyDecoder<'data>, +// M: MacroInvocation<'data, D> + 'context, +// >: +// MacroEvaluator< +// 'context, +// 'data, +// D, +// MacroInvocation = M, +// Stack = BumpVec<'context, MacroExpansion<'data, D, M>>, +// > +// { +// fn new(context: EncodingContext<'context>) -> Self +// where +// Self: Sized; +// } + /// Evaluates macro invocations recursively, yielding a single expanded value at a time. /// /// This trait describes evaluation in a variety of contexts. It supports the cross product of three @@ -191,7 +260,7 @@ pub trait MacroEvaluator<'stack, 'data: 'stack, D: LazyDecoder<'data>> { }?; // Initialize a `MacroExpansionKind` with the state necessary to evaluate the requested // macro. - let expansion_kind = match macro_kind { + let expansion_kind = match macro_kind.kind() { MacroKind::Void => MacroExpansionKind::Void, MacroKind::Values => MacroExpansionKind::Values(ValuesExpansion { arguments: invocation_to_evaluate.arguments(), @@ -201,18 +270,19 @@ pub trait MacroEvaluator<'stack, 'data: 'stack, D: LazyDecoder<'data>> { invocation_to_evaluate.arguments(), )), MacroKind::Template(_template) => todo!("template expansion init"), + MacroKind::ExpandVariable => todo!("variable expansion"), }; Ok(MacroExpansion { kind: expansion_kind, - invocation: invocation_to_evaluate, + invocation: Some(invocation_to_evaluate), }) } /// Given a syntactic element representing a macro invocation, attempt to resolve it with the /// current encoding context and push the resulting `MacroExpansion` onto the stack. - fn push<'top>( + fn push( &mut self, - context: EncodingContext<'top>, + context: EncodingContext, invocation: Self::MacroInvocation, ) -> IonResult<()> { let expansion = self.resolve_invocation(context, invocation)?; @@ -346,6 +416,7 @@ impl<'iter, 'top, 'data: 'top, D: LazyDecoder<'data>, E: MacroEvaluator<'top, 'd } /// A [`MacroEvaluator`] for expanding e-expressions found in a data stream of the format `D`. +#[derive(Default)] pub struct EExpEvaluator<'data, D: LazyDecoder<'data>> { macro_stack: Vec>, } @@ -418,12 +489,12 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> MacroEvaluator<'top, 'data, D> /// A [`MacroEvaluator`] for expanding macro invocations found in a template, all in the context /// of a data stream in the format `D`. -pub struct TemplateEvaluator<'template, 'top, 'data: 'template, D: LazyDecoder<'data>> { +pub struct TemplateEvaluator<'top, 'data, D: LazyDecoder<'data>> { argument_stack: BumpVec<'top, LazyExpandedValue<'top, 'data, D>>, - macro_stack: BumpVec<'top, MacroExpansion<'data, D, &'template Sequence>>, + macro_stack: BumpVec<'top, MacroExpansion<'data, D, TemplateMacroInvocationAddress>>, } -impl<'template, 'top, 'data, D: LazyDecoder<'data>> TemplateEvaluator<'template, 'top, 'data, D> { +impl<'top, 'data, D: LazyDecoder<'data>> TemplateEvaluator<'top, 'data, D> { pub fn new(context: EncodingContext<'top>) -> Self { Self { argument_stack: BumpVec::new_in(context.allocator), @@ -432,11 +503,11 @@ impl<'template, 'top, 'data, D: LazyDecoder<'data>> TemplateEvaluator<'template, } } -impl<'template, 'top, 'data, D: LazyDecoder<'data>> MacroEvaluator<'top, 'data, D> - for TemplateEvaluator<'template, 'top, 'data, D> +impl<'top, 'data, D: LazyDecoder<'data>> MacroEvaluator<'top, 'data, D> + for TemplateEvaluator<'top, 'data, D> { type Stack = BumpVec<'top, MacroExpansion<'data, D, Self::MacroInvocation>>; - type MacroInvocation = &'template Sequence; + type MacroInvocation = TemplateMacroInvocationAddress; fn stack(&self) -> &Self::Stack { &self.macro_stack @@ -528,7 +599,9 @@ pub struct MakeStringExpansion<'data, D: LazyDecoder<'data>, M: MacroInvocation< spooky: PhantomData, } -impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> MakeStringExpansion<'data, D, M> { +impl<'top, 'data: 'top, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> + MakeStringExpansion<'data, D, M> +{ pub fn new(arguments: M::ArgumentsIterator) -> Self { Self { arguments, @@ -538,7 +611,7 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> MakeStringExpan } /// Yields the next [`MacroExpansionStep`] in this macro's evaluation. - pub fn next<'top>( + pub fn next( &mut self, context: EncodingContext<'top>, ) -> IonResult> @@ -589,7 +662,7 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> MakeStringExpan Ok(MacroExpansionStep::ExpandedValue(LazyExpandedValue { context, - source: ExpandedValueSource::Constructed((empty_annotations, expanded_value_ref)), + source: ExpandedValueSource::Constructed(empty_annotations, expanded_value_ref), })) } @@ -631,15 +704,8 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> MakeStringExpan #[cfg(test)] mod tests { - use bumpalo::Bump as BumpAllocator; - - use crate::lazy::encoding::TextEncoding_1_1; - use crate::lazy::expanded::macro_evaluator::{MacroEvaluator, TemplateEvaluator}; - use crate::lazy::expanded::macro_table::MacroTable; - use crate::lazy::expanded::EncodingContext; use crate::lazy::reader::LazyTextReader_1_1; - use crate::lazy::value::LazyValue; - use crate::{Element, ElementReader, IonResult, SymbolTable}; + use crate::{ElementReader, IonResult}; /// Reads `input` and `expected` using an expanding reader and asserts that their output /// is the same. @@ -666,24 +732,25 @@ mod tests { /// This test exists to demonstrate that macro evaluation within the TDL context works the /// same as evaluation in the data stream. fn eval_tdl_template_invocation(invocation: &str, expected: &str) -> IonResult<()> { - let macro_table = MacroTable::new(); - let symbol_table = SymbolTable::new(); - let allocator = BumpAllocator::new(); - let context = EncodingContext::new(¯o_table, &symbol_table, &allocator); - let invocation = Element::read_one(invocation)?; - let mut evaluator = TemplateEvaluator::::new(context); - let actuals = evaluator.evaluate(context, invocation.expect_sexp()?)?; - let mut expected_reader = LazyTextReader_1_1::new(expected.as_bytes())?; - for actual_result in actuals { - // Read the next expected value as a raw value, then wrap it in an `ExpandedRawValueRef` - // so it can be directly compared to the actual. - let expected: Element = expected_reader.next()?.unwrap().read()?.try_into()?; - let actual: Element = LazyValue::from(actual_result?).try_into()?; - assert_eq!(actual, expected); - } - assert!(matches!(expected_reader.next(), Ok(None))); - - Ok(()) + todo!() + // let macro_table = MacroTable::new(); + // let symbol_table = SymbolTable::new(); + // let allocator = BumpAllocator::new(); + // let context = EncodingContext::new(¯o_table, &symbol_table, &allocator); + // let invocation = Element::read_one(invocation)?; + // let mut evaluator = TemplateEvaluator::::new(context); + // let actuals = evaluator.evaluate(context, invocation.expect_sexp()?)?; + // let mut expected_reader = LazyTextReader_1_1::new(expected.as_bytes())?; + // for actual_result in actuals { + // // Read the next expected value as a raw value, then wrap it in an `ExpandedRawValueRef` + // // so it can be directly compared to the actual. + // let expected: Element = expected_reader.next()?.unwrap().read()?.try_into()?; + // let actual: Element = LazyValue::from(actual_result?).try_into()?; + // assert_eq!(actual, expected); + // } + // assert!(matches!(expected_reader.next(), Ok(None))); + // + // Ok(()) } #[test] diff --git a/src/lazy/expanded/macro_table.rs b/src/lazy/expanded/macro_table.rs index 3067e5b4..031c9727 100644 --- a/src/lazy/expanded/macro_table.rs +++ b/src/lazy/expanded/macro_table.rs @@ -1,7 +1,9 @@ +use std::collections::HashMap; + use crate::lazy::expanded::template::TemplateMacro; +use crate::lazy::text::raw::v1_1::reader::MacroAddress; use crate::result::IonFailure; use crate::IonResult; -use std::collections::HashMap; /// The kinds of macros supported by /// [`MacroEvaluator`](crate::lazy::expanded::macro_evaluator::MacroEvaluator). @@ -14,6 +16,9 @@ pub enum MacroKind { Values, MakeString, Template(TemplateMacro), + // The following macro types are implementation details for macro evaluation and are not + // directly available to users. They do not appear in the macro table. + ExpandVariable, } impl MacroKind { @@ -23,10 +28,32 @@ impl MacroKind { MacroKind::Values => "values", MacroKind::MakeString => "make_string", MacroKind::Template(template) => template.name(), + // The `#` prefix is an arbitrarily selected sigil that indicates this is macro kind is + // not directly available to users. If this name appears in (e.g.) a macro evaluator + // stack trace, users copy/pasting it into their own templates would get a syntax error. + MacroKind::ExpandVariable => "#expand_variable", } } } +#[derive(Debug, Clone)] +pub struct MacroKindRef<'top> { + address: MacroAddress, + kind: &'top MacroKind, +} + +impl<'top> MacroKindRef<'top> { + pub fn new(address: MacroAddress, kind: &'top MacroKind) -> Self { + Self { address, kind } + } + pub fn address(&self) -> MacroAddress { + self.address + } + pub fn kind(&self) -> &'top MacroKind { + self.kind + } +} + /// Allows callers to resolve a macro ID (that is: name or address) to a [`MacroKind`], confirming /// its validity and allowing evaluation to begin. #[derive(Debug)] @@ -55,17 +82,19 @@ impl MacroTable { } } - pub fn macro_at_address(&self, id: usize) -> Option<&MacroKind> { - self.macros_by_address.get(id) + pub fn macro_at_address(&self, address: usize) -> Option> { + let kind = self.macros_by_address.get(address)?; + Some(MacroKindRef { address, kind }) } pub fn address_for_name(&self, name: &str) -> Option { self.macros_by_name.get(name).copied() } - pub fn macro_with_name(&self, name: &str) -> Option<&MacroKind> { - let id = self.macros_by_name.get(name)?; - self.macros_by_address.get(*id) + pub fn macro_with_name(&self, name: &str) -> Option> { + let address = *self.macros_by_name.get(name)?; + let kind = self.macros_by_address.get(address)?; + Some(MacroKindRef { address, kind }) } pub fn add_macro(&mut self, template: TemplateMacro) -> IonResult { diff --git a/src/lazy/expanded/mod.rs b/src/lazy/expanded/mod.rs index 5f1d7b96..b25ded6e 100644 --- a/src/lazy/expanded/mod.rs +++ b/src/lazy/expanded/mod.rs @@ -47,6 +47,7 @@ use crate::lazy::encoding::RawValueLiteral; use crate::lazy::expanded::macro_evaluator::{EExpEvaluator, MacroEvaluator}; use crate::lazy::expanded::macro_table::MacroTable; use crate::lazy::expanded::r#struct::LazyExpandedStruct; +use crate::lazy::expanded::template::{TemplateElement, TemplateValue}; use crate::lazy::r#struct::LazyStruct; use crate::lazy::raw_stream_item::RawStreamItem; use crate::lazy::raw_value_ref::RawValueRef; @@ -55,9 +56,7 @@ use crate::lazy::str_ref::StrRef; use crate::lazy::value::LazyValue; use crate::raw_symbol_token_ref::AsRawSymbolTokenRef; use crate::result::IonFailure; -use crate::{ - Decimal, Element, Int, IonResult, IonType, RawSymbolTokenRef, SymbolTable, Timestamp, Value, -}; +use crate::{Decimal, Int, IonResult, IonType, RawSymbolTokenRef, SymbolTable, Timestamp}; // All of these modules (and most of their types) are currently `pub` as the lazy reader is gated // behind an experimental feature flag. We may constrain access to them in the future as the code @@ -197,16 +196,14 @@ pub enum ExpandedValueSource<'top, 'data, D: LazyDecoder<'data>> { /// This value was a literal in the input stream. ValueLiteral(D::Value), /// This value was part of a template definition. - Template(&'top Element), + Template(TemplateElement<'top>), /// This value was the computed result of a macro invocation like `(:make_string ...)`. Constructed( // TODO: Make this an associated type on the LazyDecoder trait so 1.0 types can set // it to `Never` and the compiler can eliminate this code path where applicable. - ( - // A collection of bump-allocated annotation strings - BumpVec<'top, &'top str>, - ExpandedValueRef<'top, 'data, D>, - ), + // A collection of bump-allocated annotation strings + BumpVec<'top, &'top str>, + ExpandedValueRef<'top, 'data, D>, ), } @@ -220,12 +217,11 @@ impl<'top, 'data, V: RawValueLiteral, D: LazyDecoder<'data, Value = V>> From } } -// Converts an Element from the body of a template into an ExpandedValueSource. -impl<'top, 'data, D: LazyDecoder<'data>> From<&'top Element> +impl<'top, 'data, D: LazyDecoder<'data>> From> for ExpandedValueSource<'top, 'data, D> { - fn from(element: &'top Element) -> Self { - ExpandedValueSource::Template(element) + fn from(template_element: TemplateElement<'top>) -> Self { + ExpandedValueSource::Template(template_element) } } @@ -250,7 +246,10 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedValue<'top, 'data, D> } } - pub(crate) fn from_template(context: EncodingContext<'top>, element: &'top Element) -> Self { + pub(crate) fn from_template( + context: EncodingContext<'top>, + element: TemplateElement<'top>, + ) -> Self { Self { context, source: ExpandedValueSource::Template(element), @@ -261,8 +260,8 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedValue<'top, 'data, D> use ExpandedValueSource::*; match &self.source { ValueLiteral(value) => value.ion_type(), - Template(element) => element.ion_type(), - Constructed((_annotations, value)) => value.ion_type(), + Template(element) => element.value().ion_type(), + Constructed(_annotations, value) => value.ion_type(), } } @@ -270,8 +269,8 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedValue<'top, 'data, D> use ExpandedValueSource::*; match &self.source { ValueLiteral(value) => value.is_null(), - Template(element) => element.is_null(), - Constructed((_annotations, value)) => { + Template(element) => element.value().is_null(), + Constructed(_, value) => { matches!(value, ExpandedValueRef::Null(_)) } } @@ -284,9 +283,9 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedValue<'top, 'data, D> ExpandedAnnotationsSource::ValueLiteral(value.annotations()), ), Template(element) => ExpandedAnnotationsIterator::new( - ExpandedAnnotationsSource::Template(element.annotations().iter()), + ExpandedAnnotationsSource::Template(SymbolsIterator::new(element.annotations())), ), - Constructed((_annotations, _value)) => { + Constructed(_annotations, _value) => { // TODO: iterate over constructed annotations // For now we return an empty iterator ExpandedAnnotationsIterator::new(ExpandedAnnotationsSource::Constructed(Box::new( @@ -300,8 +299,8 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedValue<'top, 'data, D> use ExpandedValueSource::*; match &self.source { ValueLiteral(value) => Ok(ExpandedValueRef::from_raw(self.context, value.read()?)), - Template(element) => Ok(ExpandedValueRef::from_template(element, self.context)), - Constructed((_annotations, value)) => Ok((*value).clone()), + Template(element) => Ok(ExpandedValueRef::from_template(self.context, element)), + Constructed(_annotations, value) => Ok((*value).clone()), } } @@ -585,8 +584,8 @@ impl<'top, 'data, D: LazyDecoder<'data>> Debug for ExpandedValueRef<'top, 'data, } impl<'top, 'data: 'top, D: LazyDecoder<'data>> ExpandedValueRef<'top, 'data, D> { - fn from_template(element: &'top Element, context: EncodingContext<'top>) -> Self { - use Value::*; + fn from_template(context: EncodingContext<'top>, element: &TemplateElement<'top>) -> Self { + use TemplateValue::*; match element.value() { Null(ion_type) => ExpandedValueRef::Null(*ion_type), Bool(b) => ExpandedValueRef::Bool(*b), @@ -600,18 +599,21 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> ExpandedValueRef<'top, 'data, D> Clob(c) => ExpandedValueRef::Clob(BytesRef::from(c.as_ref())), List(s) => ExpandedValueRef::List(LazyExpandedList::from_template( context, - element.annotations(), - s, + element.template(), + element.annotations_range(), + *s, )), SExp(s) => ExpandedValueRef::SExp(LazyExpandedSExp::from_template( context, - element.annotations(), - s, + element.template(), + element.annotations_range(), + *s, )), Struct(s) => ExpandedValueRef::Struct(LazyExpandedStruct::from_template( context, - element.annotations(), - s, + element.template(), + element.annotations_range(), + *s, )), } } diff --git a/src/lazy/expanded/sequence.rs b/src/lazy/expanded/sequence.rs index 39e8323e..db4891dc 100644 --- a/src/lazy/expanded/sequence.rs +++ b/src/lazy/expanded/sequence.rs @@ -1,18 +1,21 @@ +use crate::element::iterators::SymbolsIterator; use crate::lazy::decoder::{LazyDecoder, LazyRawSequence, LazyRawValueExpr, RawValueExpr}; use crate::lazy::expanded::macro_evaluator::{ MacroEvaluator, TemplateEvaluator, TransientEExpEvaluator, }; -use crate::lazy::expanded::template::TemplateSequenceIterator; +use crate::lazy::expanded::template::{ + AnnotationsRange, ExprRange, TemplateMacroRef, TemplateSequenceIterator, +}; use crate::lazy::expanded::{ EncodingContext, ExpandedAnnotationsIterator, ExpandedAnnotationsSource, ExpandedValueSource, LazyExpandedValue, }; -use crate::{Annotations, IonResult, IonType, Sequence}; +use crate::{IonResult, IonType}; #[derive(Clone)] pub enum ExpandedListSource<'top, 'data, D: LazyDecoder<'data>> { ValueLiteral(D::List), - Template(&'top Annotations, &'top Sequence), + Template(TemplateMacroRef<'top>, AnnotationsRange, ExprRange), // TODO: Constructed } @@ -33,10 +36,11 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedList<'top, 'data, D> { pub fn from_template( context: EncodingContext<'top>, - annotations: &'top Annotations, - sequence: &'top Sequence, + template: TemplateMacroRef<'top>, + annotations_range: AnnotationsRange, + step_range: ExprRange, ) -> LazyExpandedList<'top, 'data, D> { - let source = ExpandedListSource::Template(annotations, sequence); + let source = ExpandedListSource::Template(template, annotations_range, step_range); Self { source, context } } @@ -45,27 +49,36 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedList<'top, 'data, D> { } pub fn annotations(&self) -> ExpandedAnnotationsIterator<'top, 'data, D> { - match self.source { + match &self.source { ExpandedListSource::ValueLiteral(value) => ExpandedAnnotationsIterator { source: ExpandedAnnotationsSource::ValueLiteral(value.annotations()), }, - ExpandedListSource::Template(annotations, _sequence) => ExpandedAnnotationsIterator { - source: ExpandedAnnotationsSource::Template(annotations.iter()), - }, + ExpandedListSource::Template(template, annotations, _sequence) => { + let annotations = template + .body + .annotations_storage() + .get(annotations.ops_range()) + .unwrap(); + ExpandedAnnotationsIterator { + source: ExpandedAnnotationsSource::Template(SymbolsIterator::new(annotations)), + } + } } } pub fn iter(&self) -> ExpandedListIterator<'top, 'data, D> { - let source = match &self.source { + let source = match self.source { ExpandedListSource::ValueLiteral(list) => { let evaluator = TransientEExpEvaluator::new(self.context); ExpandedListIteratorSource::ValueLiteral(evaluator, list.iter()) } - ExpandedListSource::Template(_annotations, sequence) => { + ExpandedListSource::Template(template, _annotations, steps) => { + let steps = template.body.expressions().get(steps.ops_range()).unwrap(); ExpandedListIteratorSource::Template(TemplateSequenceIterator::new( self.context, + template, TemplateEvaluator::new(self.context), - sequence, + steps, )) } }; @@ -108,7 +121,7 @@ impl<'top, 'data, D: LazyDecoder<'data>> Iterator for ExpandedListIterator<'top, #[derive(Clone)] pub enum ExpandedSExpSource<'top, 'data, D: LazyDecoder<'data>> { ValueLiteral(D::SExp), - Template(&'top Annotations, &'top Sequence), + Template(TemplateMacroRef<'top>, AnnotationsRange, ExprRange), } #[derive(Clone)] @@ -123,27 +136,36 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedSExp<'top, 'data, D> { } pub fn annotations(&self) -> ExpandedAnnotationsIterator<'top, 'data, D> { - match self.source { + match &self.source { ExpandedSExpSource::ValueLiteral(value) => ExpandedAnnotationsIterator { source: ExpandedAnnotationsSource::ValueLiteral(value.annotations()), }, - ExpandedSExpSource::Template(annotations, _sequence) => ExpandedAnnotationsIterator { - source: ExpandedAnnotationsSource::Template(annotations.iter()), - }, + ExpandedSExpSource::Template(template, annotations, _sequence) => { + let annotations = template + .body + .annotations_storage() + .get(annotations.ops_range()) + .unwrap(); + ExpandedAnnotationsIterator { + source: ExpandedAnnotationsSource::Template(SymbolsIterator::new(annotations)), + } + } } } pub fn iter(&self) -> ExpandedSExpIterator<'top, 'data, D> { - let source = match &self.source { + let source = match self.source { ExpandedSExpSource::ValueLiteral(sexp) => { let evaluator = TransientEExpEvaluator::new(self.context); ExpandedSExpIteratorSource::ValueLiteral(evaluator, sexp.iter()) } - ExpandedSExpSource::Template(_annotations, sequence) => { + ExpandedSExpSource::Template(template, _annotations, steps) => { + let steps = template.body.expressions().get(steps.ops_range()).unwrap(); ExpandedSExpIteratorSource::Template(TemplateSequenceIterator::new( self.context, + template, TemplateEvaluator::new(self.context), - sequence, + steps, )) } }; @@ -163,10 +185,11 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedSExp<'top, 'data, D> { pub fn from_template( context: EncodingContext<'top>, - annotations: &'top Annotations, - sequence: &'top Sequence, + template: TemplateMacroRef<'top>, + annotations: AnnotationsRange, + expressions: ExprRange, ) -> LazyExpandedSExp<'top, 'data, D> { - let source = ExpandedSExpSource::Template(annotations, sequence); + let source = ExpandedSExpSource::Template(template, annotations, expressions); Self { source, context } } } diff --git a/src/lazy/expanded/struct.rs b/src/lazy/expanded/struct.rs index dd11c263..7eee1733 100644 --- a/src/lazy/expanded/struct.rs +++ b/src/lazy/expanded/struct.rs @@ -1,17 +1,20 @@ use std::ops::ControlFlow; +use crate::element::iterators::SymbolsIterator; use crate::lazy::decoder::{LazyDecoder, LazyRawStruct, RawFieldExpr, RawValueExpr}; use crate::lazy::expanded::macro_evaluator::{ MacroEvaluator, TemplateEvaluator, TransientEExpEvaluator, }; -use crate::lazy::expanded::template::TemplateStructRawFieldsIterator; +use crate::lazy::expanded::template::{ + AnnotationsRange, ExprRange, TemplateMacroRef, TemplateStructRawFieldsIterator, +}; use crate::lazy::expanded::{ EncodingContext, ExpandedAnnotationsIterator, ExpandedAnnotationsSource, ExpandedValueRef, ExpandedValueSource, LazyExpandedValue, }; use crate::raw_symbol_token_ref::AsRawSymbolTokenRef; use crate::result::IonFailure; -use crate::{Annotations, IonError, IonResult, RawSymbolTokenRef, Struct}; +use crate::{IonError, IonResult, RawSymbolTokenRef}; #[derive(Debug, Clone)] pub struct LazyExpandedField<'top, 'data, D: LazyDecoder<'data>> { @@ -36,7 +39,7 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedField<'top, 'data, D> { #[derive(Clone)] pub enum ExpandedStructSource<'top, 'data, D: LazyDecoder<'data>> { ValueLiteral(D::Struct), - Template(&'top Annotations, &'top Struct), + Template(TemplateMacroRef<'top>, AnnotationsRange, ExprRange), // TODO: Constructed } @@ -57,10 +60,11 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedStruct<'top, 'data, D pub fn from_template( context: EncodingContext<'top>, - annotations: &'top Annotations, - struct_: &'top Struct, + template: TemplateMacroRef<'top>, + annotations: AnnotationsRange, + expressions: ExprRange, ) -> LazyExpandedStruct<'top, 'data, D> { - let source = ExpandedStructSource::Template(annotations, struct_); + let source = ExpandedStructSource::Template(template, annotations, expressions); Self { source, context } } @@ -69,9 +73,16 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedStruct<'top, 'data, D ExpandedStructSource::ValueLiteral(value) => ExpandedAnnotationsIterator { source: ExpandedAnnotationsSource::ValueLiteral(value.annotations()), }, - ExpandedStructSource::Template(annotations, _struct) => ExpandedAnnotationsIterator { - source: ExpandedAnnotationsSource::Template(annotations.iter()), - }, + ExpandedStructSource::Template(template, annotations, _expressions) => { + let annotations = template + .body + .annotations_storage() + .get(annotations.ops_range()) + .unwrap(); + ExpandedAnnotationsIterator { + source: ExpandedAnnotationsSource::Template(SymbolsIterator::new(annotations)), + } + } } } @@ -83,10 +94,14 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedStruct<'top, 'data, D raw_struct.iter(), ) } - ExpandedStructSource::Template(_annotations, struct_) => { + ExpandedStructSource::Template(template, _annotations, expressions) => { ExpandedStructIteratorSource::Template( TemplateEvaluator::new(self.context), - TemplateStructRawFieldsIterator::new(struct_), + TemplateStructRawFieldsIterator::new( + self.context, + template, + &template.body.expressions[expressions.ops_range()], + ), ) } }; @@ -136,8 +151,8 @@ pub enum ExpandedStructIteratorSource<'top, 'data, D: LazyDecoder<'data>> { // The struct we're iterating over is a value in a TDL template. It may contain macro // invocations that need to be evaluated. Template( - TemplateEvaluator<'top, 'top, 'data, D>, - TemplateStructRawFieldsIterator<'top>, + TemplateEvaluator<'top, 'data, D>, + TemplateStructRawFieldsIterator<'top, 'data, D>, ), // TODO: Constructed } diff --git a/src/lazy/expanded/tdl_macro.rs b/src/lazy/expanded/tdl_macro.rs index aacac1da..89728ccb 100644 --- a/src/lazy/expanded/tdl_macro.rs +++ b/src/lazy/expanded/tdl_macro.rs @@ -1,116 +1,4 @@ //! Types and traits representing a macro invocation within a TDL (Template Definition //! Language) expression. -use crate::element::iterators::SequenceIterator; -use crate::lazy::decoder::LazyDecoder; -use crate::lazy::expanded::macro_evaluator::{ - ArgumentKind, MacroInvocation, TemplateEvaluator, ToArgumentKind, -}; -use crate::lazy::text::raw::v1_1::reader::MacroIdRef; - -use crate::lazy::expanded::{EncodingContext, ExpandedValueSource, LazyExpandedValue}; -use crate::raw_symbol_token_ref::AsRawSymbolTokenRef; -use crate::{Element, IonResult, Sequence, Value}; - -impl<'sequence, 'data: 'sequence, D: LazyDecoder<'data>> MacroInvocation<'data, D> - for &'sequence Sequence -where - Self: 'sequence, -{ - type ArgumentExpr = &'sequence Element; - type ArgumentsIterator = OkAdapter>; - - type TransientEvaluator<'top> = TemplateEvaluator<'sequence, 'top, 'data, D> where - 'data: 'top, - Self: 'top; - - // TODO: This dummy implementation using `&'top Sequence` will be replaced by a purpose-built - // type that validates the invocation before reaching this method. For now, this method can - // panic if the input is malformed. - fn id(&self) -> MacroIdRef { - match self.get(0).expect("TDL macro call missing ID").value() { - Value::Int(address) => MacroIdRef::LocalAddress( - usize::try_from(address.expect_i64().unwrap()) - .expect("macro address int out of bounds for usize"), - ), - Value::Symbol(name) => { - MacroIdRef::LocalName(name.text().expect("cannot use $0 as macro name")) - } - _ => panic!("macro IDs must be an int or symbol"), - } - } - - fn arguments(&self) -> Self::ArgumentsIterator { - let mut children = self.elements(); - let _id = children.next().unwrap(); - OkAdapter { iterator: children } - } - - fn make_transient_evaluator<'top>( - context: EncodingContext<'top>, - ) -> Self::TransientEvaluator<'top> - where - Self: 'top, - 'data: 'top, - { - TemplateEvaluator::new(context) - } -} - -/// Wraps an infallible iterator's output items in `Result::Ok`. -pub struct OkAdapter -where - I: Iterator, -{ - iterator: I, -} - -impl OkAdapter -where - I: Iterator, -{ - pub fn new(iterator: I) -> Self { - Self { iterator } - } -} - -impl Iterator for OkAdapter { - type Item = IonResult<::Item>; - - fn next(&mut self) -> Option { - self.iterator.next().map(Ok) - } -} - -// When an `&Element` appears in macro argument position within a template, this trait implementation -// recognizes whether the `&Element` represents a value, a variable, or another template invocation. -impl<'element, 'data: 'element, D: LazyDecoder<'data>> ToArgumentKind<'data, D, &'element Sequence> - for &'element Element -{ - fn to_arg_expr<'top>( - self, - context: EncodingContext<'top>, - ) -> ArgumentKind<'top, 'data, D, &'element Sequence> - where - 'element: 'top, - { - // In this implementation, we are reading the arguments to a template macro invocation. - // For example: - // - // (macro twice (a) - // // Inside a template definition, calling the `values` macro with two arguments - // (values a a) - // ) - // In this context, there are named variables to consider. If we encounter a symbol like `a` - // in argument position, we must flag it as a variable so the caller has the opportunity to - // resolve it to a value stream. - match self.value() { - Value::SExp(sequence) => ArgumentKind::MacroInvocation(sequence), - Value::Symbol(variable) => ArgumentKind::Variable(variable.as_raw_symbol_token_ref()), - _ => ArgumentKind::ValueLiteral(LazyExpandedValue { - context, - source: ExpandedValueSource::Template(self), - }), - } - } -} +// TODO: Remove this file diff --git a/src/lazy/expanded/template.rs b/src/lazy/expanded/template.rs index 8b5661b6..2db5a7a8 100644 --- a/src/lazy/expanded/template.rs +++ b/src/lazy/expanded/template.rs @@ -1,14 +1,17 @@ -use std::ops::Range; +use std::marker::PhantomData; +use std::ops::{Deref, Range}; use crate::lazy::decoder::{LazyDecoder, RawFieldExpr, RawValueExpr}; -use crate::lazy::expanded::macro_evaluator::{MacroEvaluator, TemplateEvaluator}; -use crate::lazy::expanded::{EncodingContext, ExpandedValueSource, LazyExpandedValue}; -use crate::raw_symbol_token_ref::AsRawSymbolTokenRef; -use crate::result::IonFailure; -use crate::{ - Bytes, Decimal, Element, Int, IonResult, IonType, Sequence, Str, Struct, Symbol, Timestamp, - Value, +use crate::lazy::expanded::macro_evaluator::{ + ArgumentKind, MacroEvaluator, MacroInvocation, TemplateEvaluator, ToArgumentKind, +}; +use crate::lazy::expanded::macro_table::MacroKind; +use crate::lazy::expanded::{ + EncodingContext, ExpandedValueRef, ExpandedValueSource, LazyExpandedValue, }; +use crate::lazy::text::raw::v1_1::reader::{MacroAddress, MacroIdRef, TemplateBodyExprAddress}; +use crate::result::IonFailure; +use crate::{Bytes, Decimal, Int, IonResult, IonType, Str, Symbol, Timestamp}; #[derive(Debug, Clone)] pub enum ParameterEncoding { @@ -55,6 +58,34 @@ impl MacroSignature { } } +/// A pairing of a template reference and the address in the macro table at which it was found. +/// This type implements Deref to allow easy access to the methods on the template. +#[derive(Debug, Clone, Copy)] +pub struct TemplateMacroRef<'top> { + template_address: MacroAddress, + template: &'top TemplateMacro, +} + +impl<'top> TemplateMacroRef<'top> { + pub fn new(template_address: MacroAddress, template: &'top TemplateMacro) -> Self { + Self { + template_address, + template, + } + } + pub fn address(&self) -> MacroAddress { + self.template_address + } +} + +impl<'top> Deref for TemplateMacroRef<'top> { + type Target = &'top TemplateMacro; + + fn deref(&self) -> &Self::Target { + &self.template + } +} + #[derive(Debug, Clone)] pub struct TemplateMacro { // TODO: Make the name optional @@ -77,23 +108,25 @@ impl TemplateMacro { pub struct TemplateSequenceIterator<'top, 'data, D: LazyDecoder<'data>> { context: EncodingContext<'top>, - evaluator: TemplateEvaluator<'top, 'top, 'data, D>, - // The list element over which we're iterating - sequence: &'top Sequence, + template: TemplateMacroRef<'top>, + evaluator: TemplateEvaluator<'top, 'data, D>, + value_expressions: &'top [TemplateBodyValueExpr], index: usize, } impl<'top, 'data, D: LazyDecoder<'data>> TemplateSequenceIterator<'top, 'data, D> { pub fn new( context: EncodingContext<'top>, - evaluator: TemplateEvaluator<'top, 'top, 'data, D>, - sequence: &'top Sequence, + template: TemplateMacroRef<'top>, + evaluator: TemplateEvaluator<'top, 'data, D>, + value_expressions: &'top [TemplateBodyValueExpr], ) -> Self { Self { - sequence, - index: 0, context, + template, evaluator, + value_expressions, + index: 0, } } } @@ -113,24 +146,36 @@ impl<'top, 'data, D: LazyDecoder<'data>> Iterator for TemplateSequenceIterator<' } } } - // We didn't get a value from the evaluator, so pull the next expression from the - // sequence. - let element = self.sequence.get(self.index)?; + let step_address = self.index; + // We didn't get a value from the evaluator, so pull the next expansion step. + let step = self.value_expressions.get(step_address)?; self.index += 1; - // If the expression is an s-expression... - if let Value::SExp(sequence) = element.value() { - // ...it's a TDL macro invocation. Push it onto the evaluator's stack and return - // to the top of the loop. - match self.evaluator.push(self.context, sequence) { - Ok(_) => continue, - Err(e) => return Some(Err(e)), + return match step { + TemplateBodyValueExpr::Element(element) => Some(Ok(LazyExpandedValue { + context: self.context, + source: ExpandedValueSource::Template(TemplateElement::new( + self.template, + element, + )), + })), + TemplateBodyValueExpr::MacroInvocation(body_invocation) => { + // ...it's a TDL macro invocation. Push it onto the evaluator's stack and return + // to the top of the loop. + let invocation_address = TemplateMacroInvocationAddress::new( + self.template.address(), + body_invocation.macro_address, + step_address, + body_invocation.arg_expr_range.len(), + ); + match self.evaluator.push(self.context, invocation_address) { + Ok(_) => continue, + Err(e) => return Some(Err(e)), + }; } - } - // Otherwise, it's our next value. - return Some(Ok(LazyExpandedValue { - context: self.context, - source: ExpandedValueSource::Template(element), - })); + TemplateBodyValueExpr::Variable(_) => { + todo!("TemplateSequenceIterator: variable expansion"); + } + }; } } } @@ -138,130 +183,504 @@ impl<'top, 'data, D: LazyDecoder<'data>> Iterator for TemplateSequenceIterator<' // An iterator that pulls values from a template body and wraps them in a `RawFieldExpr` to // mimic reading them from input. The LazyExpandedStruct handles evaluating any macros that this // yields. -pub struct TemplateStructRawFieldsIterator<'top> { - // The struct element over whose fields we're iterating - struct_: &'top Struct, +pub struct TemplateStructRawFieldsIterator<'top, 'data, D: LazyDecoder<'data>> { + context: EncodingContext<'top>, + template: TemplateMacroRef<'top>, + expressions: &'top [TemplateBodyValueExpr], index: usize, + spooky: PhantomData<&'data D>, } -impl<'top> TemplateStructRawFieldsIterator<'top> { - pub fn new(struct_: &'top Struct) -> Self { - Self { struct_, index: 0 } +impl<'top, 'data, D: LazyDecoder<'data>> TemplateStructRawFieldsIterator<'top, 'data, D> { + pub fn new( + context: EncodingContext<'top>, + template: TemplateMacroRef<'top>, + expressions: &'top [TemplateBodyValueExpr], + ) -> Self { + Self { + context, + template, + expressions, + index: 0, + spooky: PhantomData, + } } } -impl<'top> Iterator for TemplateStructRawFieldsIterator<'top> { - type Item = IonResult>; +impl<'top, 'data: 'top, D: LazyDecoder<'data>> Iterator + for TemplateStructRawFieldsIterator<'top, 'data, D> +{ + type Item = + IonResult, TemplateMacroInvocationAddress>>; fn next(&mut self) -> Option { - if let Some((name, element)) = self.struct_.get_index(self.index) { - self.index += 1; - let name = name.as_raw_symbol_token_ref(); - let value = if let Value::SExp(sequence) = element.value() { - RawValueExpr::MacroInvocation(sequence) - } else { - RawValueExpr::ValueLiteral(element) - }; - Some(Ok(RawFieldExpr::NameValuePair(name, value))) - } else { - None - } + let name_expr_address = self.index; + let name_element = self + .expressions + .get(name_expr_address)? + .expect_element() + .expect("field name must be a literal"); + let name_token = match LazyExpandedValue::::from_template( + self.context, + TemplateElement::new(self.template, name_element), + ) + .read() + { + Ok(ExpandedValueRef::Symbol(token)) => token, + Ok(ExpandedValueRef::String(str_ref)) => str_ref.into(), + Ok(_) => { + return Some(IonResult::decoding_error( + "template struct had a non-text field name", + )) + } + Err(e) => return Some(Err(e)), + }; + let value_expr_address = name_expr_address + 1; + let value = match self.expressions.get(value_expr_address) { + None => { + return Some(IonResult::decoding_error( + "template struct had field name with no value", + )) + } + Some(TemplateBodyValueExpr::Element(element)) => { + RawValueExpr::ValueLiteral(TemplateElement::new(self.template, element)) + } + Some(TemplateBodyValueExpr::MacroInvocation(invocation)) => { + RawValueExpr::MacroInvocation(TemplateMacroInvocationAddress::new( + self.template.address(), + invocation.macro_address(), + value_expr_address, + invocation.arg_expr_range().len(), + )) + } + Some(TemplateBodyValueExpr::Variable(_)) => { + todo!("variable expansion in template structs") + } + }; + self.index += 2; + Some(Ok(RawFieldExpr::NameValuePair(name_token, value))) } } /// Stores a sequence of expansion steps that need to be evaluated in turn. /// -/// See [`TemplateExpansionStep`] for details. +/// See [`TemplateBodyValueExpr`] for details. #[derive(Debug, Clone, PartialEq)] pub struct TemplateBody { - pub(crate) expansion_steps: Vec, + pub(crate) expressions: Vec, pub(crate) annotations_storage: Vec, } impl TemplateBody { - pub fn expansion_steps(&self) -> &Vec { - &self.expansion_steps + pub fn expressions(&self) -> &Vec { + &self.expressions } pub fn annotations_storage(&self) -> &Vec { &self.annotations_storage } - pub fn push_element(&mut self, element: TemplateElement) { - self.expansion_steps - .push(TemplateExpansionStep::Element(element)) + pub fn push_element(&mut self, element: TemplateBodyElement) { + self.expressions + .push(TemplateBodyValueExpr::Element(element)) } - pub fn push_variable(&mut self, name: impl Into, signature_index: usize) { - self.expansion_steps.push(TemplateExpansionStep::Variable( - name.into(), - signature_index, + pub fn push_variable(&mut self, signature_index: usize) { + self.expressions.push(TemplateBodyValueExpr::Variable( + TemplateBodyVariableReference::new(signature_index), )) } - pub fn push_macro_invocation(&mut self, address: usize, num_child_expressions: usize) { - self.expansion_steps - .push(TemplateExpansionStep::MacroInvocation( - address, - num_child_expressions, + pub fn push_macro_invocation(&mut self, address: usize, expr_range: ExprRange) { + self.expressions + .push(TemplateBodyValueExpr::MacroInvocation( + TemplateBodyMacroInvocation::new(address, expr_range), )) } } -/// A single step in the expansion (i.e. evaluation) of a template. +#[derive(Copy, Clone, Debug)] +pub struct TemplateValueExpr<'top> { + template: TemplateMacroRef<'top>, + value_expr: &'top TemplateBodyValueExpr, +} + +impl<'top> TemplateValueExpr<'top> { + pub fn new(template: TemplateMacroRef<'top>, value_expr: &'top TemplateBodyValueExpr) -> Self { + Self { + template, + value_expr, + } + } + pub fn template(&self) -> TemplateMacroRef<'top> { + self.template + } + pub fn value_expr(&self) -> &'top TemplateBodyValueExpr { + self.value_expr + } +} + +/// An expression appearing in value position in a template body. #[derive(Debug, Clone, PartialEq)] -pub enum TemplateExpansionStep { +pub enum TemplateBodyValueExpr { /// A potentially annotated value literal. - Element(TemplateElement), + Element(TemplateBodyElement), /// A reference to a variable that needs to be expanded. - /// Stores the variable's name and its index within the macro's signature. - Variable(String, usize), + Variable(TemplateBodyVariableReference), /// A macro invocation that needs to be expanded. - /// Stores the macro address and the number of argument expressions. - MacroInvocation(usize, usize), + MacroInvocation(TemplateBodyMacroInvocation), +} + +#[derive(Debug, Clone)] +pub struct TemplateMacroArgExpr { + // The address of the template macro in which this argument expression appears + host_template_address: MacroAddress, + value_expr_address: TemplateBodyExprAddress, +} + +impl TemplateMacroArgExpr { + pub fn new( + host_template_address: MacroAddress, + value_expr_address: TemplateBodyExprAddress, + ) -> Self { + Self { + host_template_address, + value_expr_address, + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct TemplateBodyMacroInvocation { + macro_address: MacroAddress, + arg_expr_range: ExprRange, +} + +impl TemplateBodyMacroInvocation { + pub fn new(macro_address: MacroAddress, arg_expr_range: ExprRange) -> Self { + Self { + macro_address, + arg_expr_range, + } + } + pub fn macro_address(&self) -> MacroAddress { + self.macro_address + } + pub fn arg_expr_range(&self) -> ExprRange { + self.arg_expr_range + } } -impl TemplateExpansionStep { - /// Returns `Ok(&element)` if this expansion step is an annotated value. Otherwise, returns +impl TemplateBodyValueExpr { + /// Returns `Ok(&element)` if this expression is an annotated value. Otherwise, returns /// `Err(IonError)`. - pub fn expect_element(&self) -> IonResult<&TemplateElement> { + pub fn expect_element(&self) -> IonResult<&TemplateBodyElement> { match self { - TemplateExpansionStep::Element(e) => Ok(e), - TemplateExpansionStep::Variable(name, _) => { - IonResult::decoding_error(format!("expected an element, found variable '{name}'")) + TemplateBodyValueExpr::Element(e) => Ok(e), + TemplateBodyValueExpr::Variable(variable_reference) => { + let index = variable_reference.signature_index(); + IonResult::decoding_error(format!( + "expected an element, found reference variable with signature index '{index}'" + )) + } + TemplateBodyValueExpr::MacroInvocation(invocation) => { + let address = invocation.macro_address(); + IonResult::decoding_error(format!( + "expected an element, found macro at address {address}" + )) + } + } + } +} + +impl<'data, D: LazyDecoder<'data>> ToArgumentKind<'data, D, TemplateMacroInvocationAddress> + for TemplateMacroArgExpr +{ + fn to_arg_expr<'top>( + self, + context: EncodingContext<'top>, + ) -> ArgumentKind<'top, 'data, D, TemplateMacroInvocationAddress> + where + TemplateBodyValueExpr: 'top, + Self: 'top, + { + let Some(macro_ref) = &context + .macro_table + .macro_at_address(self.host_template_address) + else { + unreachable!( + "template address {} was not valid", + self.host_template_address + ); + }; + + let MacroKind::Template(template) = macro_ref.kind() else { + unreachable!( + "macro at address {} was not a template: {:?}", + self.host_template_address, + macro_ref.kind() + ); + }; + + let template_ref = TemplateMacroRef::new(macro_ref.address(), template); + + let invocation_expr = template + .body() + .expressions() + .get(self.value_expr_address) + .unwrap(); + match invocation_expr { + TemplateBodyValueExpr::Element(element) => { + let template_element = TemplateElement::new(template_ref, element); + ArgumentKind::ValueLiteral(LazyExpandedValue::from_template( + context, + template_element, + )) } - TemplateExpansionStep::MacroInvocation(address, _) => IonResult::decoding_error( - format!("expected an element, found macro at address {address}"), - ), + TemplateBodyValueExpr::Variable(_variable) => { + todo!("variable expansion") + } + TemplateBodyValueExpr::MacroInvocation(invocation) => { + ArgumentKind::MacroInvocation(TemplateMacroInvocationAddress { + host_template_address: template_ref.address(), + invoked_macro_address: invocation.macro_address, + invocation_expression_address: self.value_expr_address, + num_arguments: invocation.arg_expr_range.len(), + }) + } + } + } +} + +#[derive(Copy, Clone, Debug)] +pub struct TemplateMacroInvocationAddress { + // The template where the invocation appeared + host_template_address: MacroAddress, + // The address of the macro being invoked + invoked_macro_address: MacroAddress, + // The address (Vec index) of the TemplateValueExpr::MacroInvocation, which enables an + // arguments iterator to visit each argument expression in the template. + invocation_expression_address: TemplateBodyExprAddress, + num_arguments: usize, +} + +impl TemplateMacroInvocationAddress { + pub fn new( + host_template_address: MacroAddress, + invoked_macro_address: MacroAddress, + invocation_expression_address: TemplateBodyExprAddress, + num_arguments: usize, + ) -> Self { + Self { + host_template_address, + invoked_macro_address, + invocation_expression_address, + num_arguments, + } + } + pub fn host_template_address(&self) -> MacroAddress { + self.host_template_address + } + pub fn invocation_expression_address(&self) -> usize { + self.invocation_expression_address + } + + pub fn arguments(&self) -> TemplateMacroInvocationArgsIterator { + TemplateMacroInvocationArgsIterator::new(*self) + } +} + +/// A macro invocation in a template body. +#[derive(Debug, Copy, Clone)] +pub struct TemplateMacroInvocation<'top> { + // The address of the template in which this macro invocation appears + host_template_address: MacroAddress, + // The definition of the template in which this macro invocation appears + host_template: &'top TemplateMacro, + // The address of the the macro invocation expression within the host template's body + invocation_expression_address: TemplateBodyExprAddress, + // The address of the macro being invoked + invoked_macro_address: MacroAddress, + // The range of value expressions in the host template's body that are arguments to the + // macro being invoked + arg_expressions: &'top [TemplateBodyValueExpr], +} + +impl<'top> TemplateMacroInvocation<'top> { + pub fn new( + host_template_address: MacroAddress, + host_template: &'top TemplateMacro, + invocation_expression_address: TemplateBodyExprAddress, + invoked_macro_address: MacroAddress, + arg_expressions: &'top [TemplateBodyValueExpr], + ) -> Self { + Self { + host_template_address, + host_template, + invocation_expression_address, + invoked_macro_address, + arg_expressions, } } + pub fn invocation_address(&self) -> TemplateMacroInvocationAddress { + TemplateMacroInvocationAddress { + host_template_address: self.host_template_address, + invoked_macro_address: self.invoked_macro_address, + invocation_expression_address: 0, + num_arguments: self.arg_expressions.len(), + } + } + + pub fn macro_address(&self) -> MacroAddress { + self.invoked_macro_address + } + pub fn args(&self) -> TemplateMacroInvocationArgsIterator { + TemplateMacroInvocationArgsIterator::new(self.invocation_address()) + } + + pub fn template(&self) -> &'top TemplateMacro { + self.host_template + } + pub fn arg_expressions(&self) -> &'top [TemplateBodyValueExpr] { + self.arg_expressions + } +} + +// TODO: note about why this can't hold a 'top lifetime +pub struct TemplateMacroInvocationArgsIterator { + invocation_address: TemplateMacroInvocationAddress, + arg_index: usize, +} + +impl TemplateMacroInvocationArgsIterator { + pub fn new(invocation_address: TemplateMacroInvocationAddress) -> Self { + Self { + invocation_address, + arg_index: 0, + } + } +} + +impl Iterator for TemplateMacroInvocationArgsIterator { + type Item = IonResult; + + fn next(&mut self) -> Option { + if self.arg_index >= self.invocation_address.num_arguments { + return None; + } + let arg_expr = TemplateMacroArgExpr::new( + self.invocation_address.host_template_address(), + self.arg_index, + ); + self.arg_index += 1; + Some(Ok(arg_expr)) + } +} + +// TODO: Explain why this is on the address and not the resolved invocation +impl<'data, D: LazyDecoder<'data>> MacroInvocation<'data, D> for TemplateMacroInvocationAddress { + type ArgumentExpr = TemplateMacroArgExpr; + type ArgumentsIterator = TemplateMacroInvocationArgsIterator; + type TransientEvaluator<'context> = TemplateEvaluator<'context, 'data, D> where Self: 'context, 'data: 'context; + + fn id(&self) -> MacroIdRef { + MacroIdRef::LocalAddress(self.invoked_macro_address) + } + + fn arguments(&self) -> Self::ArgumentsIterator { + self.arguments() + } + + fn make_transient_evaluator<'context>( + context: EncodingContext<'context>, + ) -> Self::TransientEvaluator<'context> + where + 'data: 'context, + Self: 'context, + { + Self::TransientEvaluator::new(context) + } +} + +/// A reference to a variable in a template body. +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct TemplateBodyVariableReference { + signature_index: usize, +} + +impl TemplateBodyVariableReference { + pub fn new(signature_index: usize) -> Self { + Self { signature_index } + } + pub fn signature_index(&self) -> usize { + self.signature_index + } +} + +#[derive(Clone, Copy, Debug)] +pub struct TemplateElement<'top> { + template: TemplateMacroRef<'top>, + element: &'top TemplateBodyElement, +} + +impl<'top> TemplateElement<'top> { + pub fn new(template: TemplateMacroRef<'top>, element: &'top TemplateBodyElement) -> Self { + Self { template, element } + } + pub fn annotations(&self) -> &'top [Symbol] { + self.template + .body() + .annotations_storage() + .get(self.element.annotations_range().ops_range()) + .unwrap() + } + + pub fn annotations_range(&self) -> AnnotationsRange { + self.element.annotations_range + } + + pub fn value(&self) -> &'top TemplateValue { + &self.element.value + } + pub fn template(&self) -> TemplateMacroRef<'top> { + self.template + } } /// An annotated value in a template body. #[derive(Debug, Clone, PartialEq)] -pub struct TemplateElement { +pub struct TemplateBodyElement { // To minimize allocations, all annotations live in a single `Vec` in the `TemplateBody`. // Each element holds a range pointing to its annotation sequence. - pub(crate) annotations_range: Range, + pub(crate) annotations_range: AnnotationsRange, pub(crate) value: TemplateValue, } -impl TemplateElement { +impl TemplateBodyElement { pub fn with_value(value: TemplateValue) -> Self { Self { - annotations_range: 0..0, + annotations_range: AnnotationsRange::empty(), value, } } pub fn with_annotations(mut self, range: Range) -> Self { - self.annotations_range = range.start as u32..range.end as u32; + self.annotations_range = AnnotationsRange::new(range); self } pub fn value(&self) -> &TemplateValue { &self.value } - pub fn annotations(&self) -> Range { - self.annotations_range.start as usize..self.annotations_range.end as usize + pub fn annotations_range(&self) -> AnnotationsRange { + self.annotations_range + } + + pub fn annotations<'a>(&self, template: &'a TemplateMacro) -> &'a [Symbol] { + template + .body() + .annotations_storage() + .get(self.annotations_range().ops_range()) + // If the annotations range is invalid, that's a bug; we cannot return an error. + .unwrap() } } @@ -278,9 +697,79 @@ pub enum TemplateValue { Clob(Bytes), Blob(Bytes), // The number of ensuing TemplateElements that are nested. - List(usize), - SExp(usize), - // The number of fields in the struct. Each field is made up of a pair of TemplateElements, + List(ExprRange), + SExp(ExprRange), + // The number of fields in the struct. Each field is made up of a pair of TemplateExpansionSteps, // a name (always a symbol, never has annotations), and a value (any element). - Struct(usize), + Struct(ExprRange), +} + +impl TemplateValue { + pub fn is_null(&self) -> bool { + matches!(self, TemplateValue::Null(_)) + } + + pub fn ion_type(&self) -> IonType { + // TODO: Implement this with a Rust macro instead. + // See: https://github.com/amazon-ion/ion-rust/issues/650 + use TemplateValue::*; + match self { + Null(ion_type) => *ion_type, + Bool(_) => IonType::Bool, + Int(_) => IonType::Int, + Float(_) => IonType::Float, + Decimal(_) => IonType::Decimal, + Timestamp(_) => IonType::Timestamp, + Symbol(_) => IonType::Symbol, + String(_) => IonType::String, + Clob(_) => IonType::Clob, + Blob(_) => IonType::Blob, + List(_) => IonType::List, + SExp(_) => IonType::SExp, + Struct(_) => IonType::Struct, + } + } +} + +/// A slice of a `TemplateBody`'s sequence of `TemplateExpansionStep`. This type can be used to +/// represent containers (list, sexp, struct) or macro invocations, all of which use an evaluator +/// to iteratively evaluate a series of `TemplateExpansionStep`s. This type does not hold a reference +/// to the template definition itself. +pub type ExprRange = SmallRange; +pub type AnnotationsRange = SmallRange; + +/// A range that takes 8 bytes instead of Range's 16. This is useful for cases like +/// annotations where a capacity of 4 billion+ is more than enough. It also implements `Copy`, +/// making it possible for enclosing types to also implement `Copy`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct SmallRange { + start: u32, + end: u32, +} + +impl SmallRange { + pub fn empty() -> Self { + Self { start: 0, end: 0 } + } + + pub fn new(range: Range) -> Self { + debug_assert!(u32::try_from(range.start).is_ok()); + debug_assert!(u32::try_from(range.end).is_ok()); + Self { + start: range.start as u32, + end: range.end as u32, + } + } + + pub fn ops_range(&self) -> Range { + self.start as usize..self.end as usize + } + + pub fn len(&self) -> usize { + (self.end - self.start) as usize + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } } diff --git a/src/lazy/never.rs b/src/lazy/never.rs index 019f972d..45a02e7e 100644 --- a/src/lazy/never.rs +++ b/src/lazy/never.rs @@ -21,9 +21,8 @@ impl<'data, D: LazyDecoder<'data>> MacroInvocation<'data, D> for Never { type ArgumentExpr = Never; // This uses a Box to avoid defining yet another placeholder type. type ArgumentsIterator = Box>>; - type TransientEvaluator<'top> = Never where - 'data: 'top, - Self: 'top; + + type TransientEvaluator<'context> = Never where Self: 'context, 'data: 'context; fn id(&self) -> MacroIdRef<'_> { unreachable!("macro in Ion 1.0 (method: id)") @@ -33,12 +32,12 @@ impl<'data, D: LazyDecoder<'data>> MacroInvocation<'data, D> for Never { unreachable!("macro in Ion 1.0 (method: arguments)") } - fn make_transient_evaluator<'top>( - _context: EncodingContext<'top>, - ) -> Self::TransientEvaluator<'top> + fn make_transient_evaluator<'context>( + _context: EncodingContext<'context>, + ) -> Self::TransientEvaluator<'context> where - 'data: 'top, - Self: 'top, + 'data: 'context, + Self: 'context, { unreachable!("macro in Ion 1.0 (method: make_transient_evaluator)") } @@ -48,8 +47,9 @@ impl<'data, D: LazyDecoder<'data>> ToArgumentKind<'data, D, Self> for Never { fn to_arg_expr<'top>( self, _context: EncodingContext<'top>, - ) -> ArgumentKind<'top, 'data, D, Self> + ) -> ArgumentKind<'top, 'data, D, Never> where + Never: 'top, Self: 'top, { unreachable!("macro in Ion 1.0 (method: to_arg_expr)") diff --git a/src/lazy/text/raw/v1_1/reader.rs b/src/lazy/text/raw/v1_1/reader.rs index 158ffddd..fbc098a4 100644 --- a/src/lazy/text/raw/v1_1/reader.rs +++ b/src/lazy/text/raw/v1_1/reader.rs @@ -26,6 +26,11 @@ pub struct LazyRawTextReader_1_1<'data> { bytes_to_skip: usize, } +/// The index at which this macro can be found in the macro table. +pub type MacroAddress = usize; + +/// The index at which a value expression can be found within a template's body. +pub type TemplateBodyExprAddress = usize; #[derive(Copy, Clone, Debug, PartialEq)] pub enum MacroIdRef<'data> { LocalName(&'data str),