From 2675d430ac119c15629a862cabbdcd71fa650bfc Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Fri, 13 Oct 2023 14:22:43 -0400 Subject: [PATCH] Working expansion w/o variables --- examples/lazy_read_all_values.rs | 2 +- src/ion_data/mod.rs | 2 +- src/lazy/any_encoding.rs | 131 ++--- src/lazy/decoder.rs | 71 +-- src/lazy/encoding.rs | 12 +- src/lazy/expanded/compiler.rs | 7 +- src/lazy/expanded/e_expression.rs | 126 ++++- src/lazy/expanded/macro_evaluator.rs | 807 +++++++++++++++------------ src/lazy/expanded/macro_table.rs | 43 +- src/lazy/expanded/mod.rs | 98 +--- src/lazy/expanded/sequence.rs | 65 +-- src/lazy/expanded/struct.rs | 56 +- src/lazy/expanded/template.rs | 550 +++++++++--------- src/lazy/never.rs | 76 +-- src/lazy/raw_stream_item.rs | 6 +- src/lazy/reader.rs | 11 + src/lazy/system_reader.rs | 18 +- src/lazy/text/buffer.rs | 8 +- src/lazy/text/raw/v1_1/reader.rs | 61 +- src/types/struct.rs | 2 +- 20 files changed, 1187 insertions(+), 965 deletions(-) diff --git a/examples/lazy_read_all_values.rs b/examples/lazy_read_all_values.rs index b42efe03..152ee9dd 100644 --- a/examples/lazy_read_all_values.rs +++ b/examples/lazy_read_all_values.rs @@ -61,7 +61,7 @@ mod lazy_reader_example { Ok(1 + child_count) } - fn count_sequence_children<'a, 'b>( + fn count_sequence_children<'a, 'b: 'a>( lazy_sequence: impl Iterator>>, ) -> IonResult { let mut count = 0; diff --git a/src/ion_data/mod.rs b/src/ion_data/mod.rs index 2ba2565b..3bae9b1e 100644 --- a/src/ion_data/mod.rs +++ b/src/ion_data/mod.rs @@ -78,7 +78,7 @@ impl Display for IonData { impl PartialOrd for IonData { fn partial_cmp(&self, other: &Self) -> Option { - Some(IonOrd::ion_cmp(&self.0, &other.0)) + Some(self.cmp(other)) } } diff --git a/src/lazy/any_encoding.rs b/src/lazy/any_encoding.rs index 5173476a..98423b16 100644 --- a/src/lazy/any_encoding.rs +++ b/src/lazy/any_encoding.rs @@ -1,7 +1,6 @@ #![allow(non_camel_case_types)] use std::fmt::Debug; -use std::marker::PhantomData; use crate::lazy::binary::raw::annotations_iterator::RawBinaryAnnotationsIterator; use crate::lazy::binary::raw::r#struct::{LazyRawBinaryStruct, RawBinaryStructIterator}; @@ -13,11 +12,11 @@ use crate::lazy::binary::raw::value::LazyRawBinaryValue; use crate::lazy::decoder::private::{LazyContainerPrivate, LazyRawValuePrivate}; use crate::lazy::decoder::{ LazyDecoder, LazyRawFieldExpr, LazyRawReader, LazyRawSequence, LazyRawStruct, LazyRawValue, - LazyRawValueExpr, RawArgumentExpr, RawFieldExpr, RawValueExpr, + LazyRawValueExpr, RawFieldExpr, RawValueExpr, }; use crate::lazy::encoding::{BinaryEncoding_1_0, TextEncoding_1_0, TextEncoding_1_1}; -use crate::lazy::expanded::macro_evaluator::{EExpEvaluator, MacroInvocation}; -use crate::lazy::expanded::sequence::Environment; +use crate::lazy::expanded::e_expression::EExpression; +use crate::lazy::expanded::macro_evaluator::RawMacroInvocation; use crate::lazy::expanded::EncodingContext; use crate::lazy::never::Never; use crate::lazy::raw_stream_item::RawStreamItem; @@ -29,13 +28,14 @@ use crate::lazy::text::raw::sequence::{ }; use crate::lazy::text::raw::v1_1::reader::{ LazyRawTextList_1_1, LazyRawTextSExp_1_1, LazyRawTextStruct_1_1, MacroIdRef, - RawTextListIterator_1_1, RawTextMacroInvocation, RawTextSExpIterator_1_1, + RawTextEExpression_1_1, RawTextListIterator_1_1, RawTextSExpIterator_1_1, RawTextStructIterator_1_1, }; use crate::lazy::text::value::{ LazyRawTextValue_1_0, LazyRawTextValue_1_1, RawTextAnnotationsIterator, }; -use crate::{IonResult, IonType, RawSymbolTokenRef}; +use crate::result::IonFailure; +use crate::{IonError, IonResult, IonType, RawSymbolTokenRef}; /// An implementation of the `LazyDecoder` trait that can read any encoding of Ion. #[derive(Debug, Clone, Copy)] @@ -51,97 +51,78 @@ impl<'data> LazyDecoder<'data> for AnyEncoding { type List = LazyRawAnyList<'data>; type Struct = LazyRawAnyStruct<'data>; type AnnotationsIterator = RawAnyAnnotationsIterator<'data>; - type MacroInvocation = LazyRawAnyMacroInvocation<'data>; + type RawMacroInvocation = LazyRawAnyEExpression<'data>; + type MacroInvocation<'top> = EExpression<'top, 'data, AnyEncoding> where 'data: 'top; } - #[derive(Debug, Copy, Clone)] -pub struct LazyRawAnyMacroInvocation<'data> { - encoding: LazyRawAnyMacroInvocationKind<'data>, +pub struct LazyRawAnyEExpression<'data> { + encoding: LazyRawAnyEExpressionKind<'data>, } #[derive(Debug, Copy, Clone)] -enum LazyRawAnyMacroInvocationKind<'data> { +enum LazyRawAnyEExpressionKind<'data> { // Ion 1.0 does not support macro invocations. Having these variants hold an instance of // `Never` (which cannot be instantiated) informs the compiler that it can eliminate these // branches in code paths exclusive to v1.0. Text_1_0(Never), Binary_1_0(Never), - Text_1_1(RawTextMacroInvocation<'data>), + Text_1_1(RawTextEExpression_1_1<'data>), } -impl<'data> From> for LazyRawAnyMacroInvocation<'data> { - fn from(text_invocation: RawTextMacroInvocation<'data>) -> Self { - LazyRawAnyMacroInvocation { - encoding: LazyRawAnyMacroInvocationKind::Text_1_1(text_invocation), +impl<'data> From> for LazyRawAnyEExpression<'data> { + fn from(text_invocation: RawTextEExpression_1_1<'data>) -> Self { + LazyRawAnyEExpression { + encoding: LazyRawAnyEExpressionKind::Text_1_1(text_invocation), } } } -impl<'data> MacroInvocation<'data, AnyEncoding> for LazyRawAnyMacroInvocation<'data> { - type ArgumentExpr = LazyRawValueExpr<'data, AnyEncoding>; - type ArgumentsIterator = LazyRawAnyMacroArgsIterator<'data>; - type RawArgumentsIterator = LazyRawAnyMacroRawArgsIterator<'data>; - - type TransientEvaluator<'context> = - EExpEvaluator<'context, 'data, AnyEncoding> where Self: 'context, 'data: 'context; +impl<'data> RawMacroInvocation<'data, AnyEncoding> for LazyRawAnyEExpression<'data> { + type RawArgumentsIterator<'a> = LazyRawAnyMacroArgsIterator<'data> where Self: 'a; + type MacroInvocation<'top> = EExpression<'top, 'data, AnyEncoding> where 'data: 'top; - fn id(&self) -> MacroIdRef<'_> { + fn id(&self) -> MacroIdRef<'data> { match self.encoding { - LazyRawAnyMacroInvocationKind::Text_1_0(_) => unreachable!("macro in text Ion 1.0"), - LazyRawAnyMacroInvocationKind::Binary_1_0(_) => unreachable!("macro in binary Ion 1.0"), - LazyRawAnyMacroInvocationKind::Text_1_1(ref m) => m.id(), + LazyRawAnyEExpressionKind::Text_1_0(_) => unreachable!("macro in text Ion 1.0"), + LazyRawAnyEExpressionKind::Binary_1_0(_) => unreachable!("macro in binary Ion 1.0"), + LazyRawAnyEExpressionKind::Text_1_1(ref m) => m.id(), } } - fn arguments(&self) -> Self::ArgumentsIterator { + fn raw_arguments(&self) -> Self::RawArgumentsIterator<'_> { match self.encoding { - LazyRawAnyMacroInvocationKind::Text_1_0(_) => unreachable!("macro in text Ion 1.0"), - LazyRawAnyMacroInvocationKind::Binary_1_0(_) => unreachable!("macro in binary Ion 1.0"), - LazyRawAnyMacroInvocationKind::Text_1_1(m) => LazyRawAnyMacroArgsIterator { - encoding: LazyRawAnyMacroArgsIteratorKind::Text_1_1(m.arguments()), + LazyRawAnyEExpressionKind::Text_1_0(_) => unreachable!("macro in text Ion 1.0"), + LazyRawAnyEExpressionKind::Binary_1_0(_) => unreachable!("macro in binary Ion 1.0"), + LazyRawAnyEExpressionKind::Text_1_1(m) => LazyRawAnyMacroArgsIterator { + encoding: LazyRawAnyMacroArgsIteratorKind::Text_1_1(m.raw_arguments()), }, } } - fn raw_arguments(&self) -> Self::RawArgumentsIterator { - todo!() - } - - fn make_transient_evaluator<'context>( - context: EncodingContext<'context>, - _environment: Environment<'context, 'data, AnyEncoding>, - ) -> Self::TransientEvaluator<'context> + fn resolve<'top>(self, context: EncodingContext<'top>) -> IonResult> where - 'data: 'context, - Self: 'context, + 'data: 'top, { - // E-expressions do not have an environment, so we ignore that parameter. - Self::TransientEvaluator::new(context) - } -} - -pub enum LazyRawAnyMacroRawArgsIteratorKind<'data> { - TODO(PhantomData<&'data ()>), -} - -pub struct LazyRawAnyMacroRawArgsIterator<'data> { - encoding: LazyRawAnyMacroRawArgsIteratorKind<'data>, -} - -impl<'data> Iterator for LazyRawAnyMacroRawArgsIterator<'data> { - type Item = RawArgumentExpr<'data, AnyEncoding>; - - fn next(&mut self) -> Option { - todo!() + let invoked_macro = context + .macro_table + .macro_with_id(self.id()) + .ok_or_else(|| { + IonError::decoding_error(format!("unrecognized macro ID {:?}", self.id())) + })?; + Ok(EExpression { + context, + raw_invocation: self, + invoked_macro, + }) } } pub enum LazyRawAnyMacroArgsIteratorKind<'data> { Text_1_1( - as MacroInvocation< + as RawMacroInvocation< 'data, TextEncoding_1_1, - >>::ArgumentsIterator, + >>::RawArgumentsIterator<'data>, ), } pub struct LazyRawAnyMacroArgsIterator<'data> { @@ -157,11 +138,11 @@ impl<'data> Iterator for LazyRawAnyMacroArgsIterator<'data> { Some(Ok(RawValueExpr::ValueLiteral(value))) => { Some(Ok(RawValueExpr::ValueLiteral(LazyRawAnyValue::from(value)))) } - Some(Ok(RawValueExpr::MacroInvocation(invocation))) => Some(Ok( - RawValueExpr::MacroInvocation(LazyRawAnyMacroInvocation { - encoding: LazyRawAnyMacroInvocationKind::Text_1_1(invocation), - }), - )), + Some(Ok(RawValueExpr::MacroInvocation(invocation))) => { + Some(Ok(RawValueExpr::MacroInvocation(LazyRawAnyEExpression { + encoding: LazyRawAnyEExpressionKind::Text_1_1(invocation), + }))) + } Some(Err(e)) => Some(Err(e)), None => None, }, @@ -259,8 +240,8 @@ impl<'data> From> match value { RawValueExpr::ValueLiteral(v) => RawValueExpr::ValueLiteral(v.into()), RawValueExpr::MacroInvocation(m) => { - let invocation = LazyRawAnyMacroInvocation { - encoding: LazyRawAnyMacroInvocationKind::Text_1_0(m), + let invocation = LazyRawAnyEExpression { + encoding: LazyRawAnyEExpressionKind::Text_1_0(m), }; RawValueExpr::MacroInvocation(invocation) } @@ -275,8 +256,8 @@ impl<'data> From> match value { RawValueExpr::ValueLiteral(v) => RawValueExpr::ValueLiteral(v.into()), RawValueExpr::MacroInvocation(m) => { - let invocation = LazyRawAnyMacroInvocation { - encoding: LazyRawAnyMacroInvocationKind::Binary_1_0(m), + let invocation = LazyRawAnyEExpression { + encoding: LazyRawAnyEExpressionKind::Binary_1_0(m), }; RawValueExpr::MacroInvocation(invocation) } @@ -291,8 +272,8 @@ impl<'data> From> match value { RawValueExpr::ValueLiteral(v) => RawValueExpr::ValueLiteral(v.into()), RawValueExpr::MacroInvocation(m) => { - let invocation = LazyRawAnyMacroInvocation { - encoding: LazyRawAnyMacroInvocationKind::Text_1_1(m), + let invocation = LazyRawAnyEExpression { + encoding: LazyRawAnyEExpressionKind::Text_1_1(m), }; RawValueExpr::MacroInvocation(invocation) } @@ -401,8 +382,8 @@ impl<'data> From> for RawStreamItem<'data } RawStreamItem::Value(value) => RawStreamItem::Value(value.into()), RawStreamItem::EExpression(invocation) => { - RawStreamItem::EExpression(LazyRawAnyMacroInvocation { - encoding: LazyRawAnyMacroInvocationKind::Text_1_1(invocation), + RawStreamItem::EExpression(LazyRawAnyEExpression { + encoding: LazyRawAnyEExpressionKind::Text_1_1(invocation), }) } RawStreamItem::EndOfStream => RawStreamItem::EndOfStream, diff --git a/src/lazy/decoder.rs b/src/lazy/decoder.rs index 727da804..395ccf63 100644 --- a/src/lazy/decoder.rs +++ b/src/lazy/decoder.rs @@ -1,17 +1,16 @@ use std::fmt::Debug; use crate::lazy::encoding::TextEncoding_1_1; -use crate::lazy::expanded::macro_evaluator::{EExpEvaluator, MacroInvocation}; -use crate::lazy::expanded::sequence::Environment; -use crate::lazy::expanded::template::TemplateMacroArgExpr; +use crate::lazy::expanded::e_expression::TextEExpression_1_1; +use crate::lazy::expanded::macro_evaluator::{MacroInvocation, RawMacroInvocation}; use crate::lazy::expanded::EncodingContext; use crate::lazy::raw_stream_item::RawStreamItem; use crate::lazy::raw_value_ref::RawValueRef; use crate::lazy::text::raw::v1_1::reader::{ - MacroIdRef, RawTextMacroInvocation, RawTextSExpIterator_1_1, + MacroIdRef, RawTextEExpression_1_1, RawTextSExpIterator_1_1, }; use crate::result::IonFailure; -use crate::{IonResult, IonType, RawSymbolTokenRef}; +use crate::{IonError, IonResult, IonType, RawSymbolTokenRef}; /// A family of types that collectively comprise the lazy reader API for an Ion serialization /// format. These types operate at the 'raw' level; they do not attempt to resolve symbols @@ -32,7 +31,10 @@ pub trait LazyDecoder<'data>: 'static + Sized + Debug + Clone + Copy { /// An iterator over the annotations on the input stream's values. type AnnotationsIterator: Iterator>>; /// An e-expression invoking a macro. (Ion 1.1+) - type MacroInvocation: MacroInvocation<'data, Self>; + type RawMacroInvocation: RawMacroInvocation<'data, Self>; + type MacroInvocation<'top>: MacroInvocation<'top, 'data, Self> + where + 'data: 'top; } /// An expression found in value position in either serialized Ion or a template. @@ -61,7 +63,7 @@ pub enum RawValueExpr { /// For a version of this type that is not constrained to a particular encoding, see /// [`RawValueExpr`]. pub type LazyRawValueExpr<'data, D> = - RawValueExpr<>::Value, >::MacroInvocation>; + RawValueExpr<>::Value, >::RawMacroInvocation>; impl RawValueExpr { pub fn expect_value(self) -> IonResult { @@ -84,24 +86,6 @@ impl RawValueExpr { } } -#[derive(Copy, Clone, Debug)] -pub enum RawArgumentExpr<'data, D: LazyDecoder<'data>> { - StreamExpr(LazyRawValueExpr<'data, D>), - TemplateExpr(TemplateMacroArgExpr), -} - -impl<'data, D: LazyDecoder<'data>> From> for RawArgumentExpr<'data, D> { - fn from(stream_value_expr: LazyRawValueExpr<'data, D>) -> Self { - RawArgumentExpr::StreamExpr(stream_value_expr) - } -} - -impl<'data, D: LazyDecoder<'data>> From for RawArgumentExpr<'data, D> { - fn from(template_value_expr: TemplateMacroArgExpr) -> Self { - RawArgumentExpr::TemplateExpr(template_value_expr) - } -} - /// An item found in field position within a struct. /// This item may be: /// * a name/value pair (as it is in Ion 1.0) @@ -122,7 +106,7 @@ pub enum RawFieldExpr<'name, V, M> { pub type LazyRawFieldExpr<'data, D> = RawFieldExpr< 'data, >::Value, - >::MacroInvocation, + >::RawMacroInvocation, >; impl<'name, V: Debug, M: Debug> RawFieldExpr<'name, V, M> { @@ -196,35 +180,32 @@ pub(crate) mod private { } } -impl<'data> MacroInvocation<'data, TextEncoding_1_1> for RawTextMacroInvocation<'data> { - type ArgumentExpr = LazyRawValueExpr<'data, TextEncoding_1_1>; - type ArgumentsIterator = RawTextSExpIterator_1_1<'data>; - type RawArgumentsIterator = Box>>; +// TODO: Everything should use LazyRawValueExpr in the RMI impl? - type TransientEvaluator<'context> = - EExpEvaluator<'context, 'data, TextEncoding_1_1> where Self: 'context, 'data: 'context; +impl<'data> RawMacroInvocation<'data, TextEncoding_1_1> for RawTextEExpression_1_1<'data> { + type RawArgumentsIterator<'a> = RawTextSExpIterator_1_1<'data> where Self: 'a; - fn id(&self) -> MacroIdRef { + type MacroInvocation<'top> = TextEExpression_1_1<'top, 'data> where 'data: 'top; + + fn id(&self) -> MacroIdRef<'data> { self.id } - fn arguments(&self) -> Self::ArgumentsIterator { + fn raw_arguments(&self) -> Self::RawArgumentsIterator<'_> { RawTextSExpIterator_1_1::new(self.arguments_bytes()) } - fn raw_arguments(&self) -> Self::RawArgumentsIterator { - todo!() - } - - fn make_transient_evaluator<'context>( - context: EncodingContext<'context>, - _environment: Environment<'context, 'data, TextEncoding_1_1>, - ) -> Self::TransientEvaluator<'context> + fn resolve<'top>(self, context: EncodingContext<'top>) -> IonResult> where - Self: 'context, + 'data: 'top, { - // E-expressions do not have an environment, so we ignore that parameter. - Self::TransientEvaluator::new(context) + let invoked_macro = context + .macro_table + .macro_with_id(self.id()) + .ok_or_else(|| { + IonError::decoding_error(format!("unrecognized macro ID {:?}", self.id())) + })?; + Ok(TextEExpression_1_1::new(context, self, invoked_macro)) } } diff --git a/src/lazy/encoding.rs b/src/lazy/encoding.rs index 5769827a..1e366afc 100644 --- a/src/lazy/encoding.rs +++ b/src/lazy/encoding.rs @@ -7,13 +7,14 @@ use crate::lazy::binary::raw::reader::LazyRawBinaryReader; use crate::lazy::binary::raw::sequence::{LazyRawBinaryList, LazyRawBinarySExp}; use crate::lazy::binary::raw::value::LazyRawBinaryValue; use crate::lazy::decoder::LazyDecoder; +use crate::lazy::expanded::e_expression::TextEExpression_1_1; use crate::lazy::never::Never; use crate::lazy::text::raw::r#struct::LazyRawTextStruct_1_0; use crate::lazy::text::raw::reader::LazyRawTextReader_1_0; use crate::lazy::text::raw::sequence::{LazyRawTextList_1_0, LazyRawTextSExp_1_0}; use crate::lazy::text::raw::v1_1::reader::{ LazyRawTextList_1_1, LazyRawTextReader_1_1, LazyRawTextSExp_1_1, LazyRawTextStruct_1_1, - RawTextMacroInvocation, + RawTextEExpression_1_1, }; use crate::lazy::text::value::{ LazyRawTextValue, LazyRawTextValue_1_0, LazyRawTextValue_1_1, MatchedRawTextValue, @@ -97,7 +98,8 @@ impl<'data> LazyDecoder<'data> for BinaryEncoding_1_0 { type Struct = LazyRawBinaryStruct<'data>; type AnnotationsIterator = RawBinaryAnnotationsIterator<'data>; // Macros are not supported in Ion 1.0 - type MacroInvocation = Never; + type RawMacroInvocation = Never; + type MacroInvocation<'top> = Never where 'data: 'top; } impl<'data> LazyDecoder<'data> for TextEncoding_1_0 { @@ -108,7 +110,8 @@ impl<'data> LazyDecoder<'data> for TextEncoding_1_0 { type Struct = LazyRawTextStruct_1_0<'data>; type AnnotationsIterator = RawTextAnnotationsIterator<'data>; // Macros are not supported in Ion 1.0 - type MacroInvocation = Never; + type RawMacroInvocation = Never; + type MacroInvocation<'top> = Never where 'data: 'top; } impl<'data> LazyDecoder<'data> for TextEncoding_1_1 { @@ -118,7 +121,8 @@ impl<'data> LazyDecoder<'data> for TextEncoding_1_1 { type List = LazyRawTextList_1_1<'data>; type Struct = LazyRawTextStruct_1_1<'data>; type AnnotationsIterator = RawTextAnnotationsIterator<'data>; - type MacroInvocation = RawTextMacroInvocation<'data>; + type RawMacroInvocation = RawTextEExpression_1_1<'data>; + type MacroInvocation<'top> = TextEExpression_1_1<'top, 'data> where 'data: 'top; } /// Marker trait for types that represent value literals in an Ion stream of some encoding. diff --git a/src/lazy/expanded/compiler.rs b/src/lazy/expanded/compiler.rs index d7b2c168..654558e7 100644 --- a/src/lazy/expanded/compiler.rs +++ b/src/lazy/expanded/compiler.rs @@ -52,7 +52,10 @@ impl TemplateCompiler { /// /// The compiler recognizes the `(quote expr1 expr2 [...] exprN)` form, adding each subexpression /// to the template without interpretation. - fn compile_from_text(context: EncodingContext, expression: &str) -> IonResult { + pub(crate) fn compile_from_text( + context: EncodingContext, + expression: &str, + ) -> IonResult { // TODO: This is a rudimentary implementation that panics instead of performing thorough // validation. Where it does surface errors, the messages are too terse. let mut reader = LazyTextReader_1_1::new(expression.as_bytes())?; @@ -271,6 +274,7 @@ impl TemplateCompiler { // Update the macro step to reflect the macro's address and number of child expressions it // contains let template_macro_invocation = TemplateBodyMacroInvocation::new( + macro_step_index, macro_address, ExprRange::new(arguments_start..arguments_end), ); @@ -480,6 +484,7 @@ mod tests { definition, index, TemplateBodyValueExpr::MacroInvocation(TemplateBodyMacroInvocation::new( + index, expected_address, // The arg range starts just after the macro invocation step and goes for `expected_num_arguments`. ExprRange::new(index + 1..index + 1 + expected_num_arguments), diff --git a/src/lazy/expanded/e_expression.rs b/src/lazy/expanded/e_expression.rs index f79a824e..66cde3de 100644 --- a/src/lazy/expanded/e_expression.rs +++ b/src/lazy/expanded/e_expression.rs @@ -1,34 +1,108 @@ //! Types and traits representing an e-expression in an Ion stream. +#![allow(non_camel_case_types)] -use crate::lazy::decoder::{LazyDecoder, LazyRawValueExpr, RawValueExpr}; -use crate::lazy::expanded::macro_evaluator::{ArgumentKind, ToArgumentKind}; +use crate::lazy::decoder::{LazyDecoder, LazyRawValueExpr}; +use crate::lazy::encoding::TextEncoding_1_1; +use crate::lazy::expanded::macro_evaluator::{ + ArgumentExpr, MacroExpr, MacroInvocation, RawMacroInvocation, +}; +use crate::lazy::expanded::macro_table::MacroRef; use crate::lazy::expanded::sequence::Environment; -use crate::lazy::expanded::{EncodingContext, ExpandedValueSource, LazyExpandedValue}; +use crate::lazy::expanded::{EncodingContext, LazyExpandedValue}; +use crate::lazy::text::raw::v1_1::reader::MacroIdRef; +use crate::IonResult; +use std::fmt::{Debug, Formatter}; -// When a `LazyRawValueExpr` appears in argument position within an e-expression, this trait -// implementation recognizes it as either a value or another macro invocation. -impl<'data, D: LazyDecoder<'data>> ToArgumentKind<'data, D, D::MacroInvocation> - for LazyRawValueExpr<'data, D> -{ - fn to_arg_expr<'top>( - self, +#[derive(Copy, Clone)] +pub struct EExpression<'top, 'data, D: LazyDecoder<'data>> { + pub(crate) context: EncodingContext<'top>, + pub(crate) raw_invocation: D::RawMacroInvocation, + pub(crate) invoked_macro: MacroRef<'top>, +} + +impl<'top, 'data, D: LazyDecoder<'data>> Debug for EExpression<'top, 'data, D> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "EExpression {:?}", self.raw_invocation) + } +} + +impl<'top, 'data, D: LazyDecoder<'data>> EExpression<'top, 'data, D> { + pub fn new( context: EncodingContext<'top>, - _environment: Environment<'top, 'data, D>, - ) -> ArgumentKind<'top, 'data, D, D::MacroInvocation> - where - D::MacroInvocation: 'top, - Self: 'top, - { - match self { - // In this implementation, we're reading arguments to an E-expression in the data stream. - // Because e-expressions appear in the data stream (and not in a template), there is no - // environment of named variables. We do not attempt to resolve symbols as though they - // were variable names and instead pass them along as value literals. - RawValueExpr::ValueLiteral(value) => ArgumentKind::ValueLiteral(LazyExpandedValue { - context, - source: ExpandedValueSource::ValueLiteral(value), - }), - RawValueExpr::MacroInvocation(invocation) => ArgumentKind::MacroInvocation(invocation), + raw_invocation: D::RawMacroInvocation, + invoked_macro: MacroRef<'top>, + ) -> Self { + Self { + context, + raw_invocation, + invoked_macro, + } + } +} + +impl< + 'top, + 'data: 'top, + D: LazyDecoder<'data, MacroInvocation<'top> = EExpression<'top, 'data, D>>, + > MacroInvocation<'top, 'data, D> for EExpression<'top, 'data, D> +{ + type ArgumentsIterator = EExpressionArgsIterator<'top, 'data, D>; + + fn id(&self) -> MacroIdRef<'top> { + self.raw_invocation.id() + } + + fn arguments(&self, _environment: Environment<'top, 'data, D>) -> Self::ArgumentsIterator { + // E-expressions do not have an environment, so we can ignore/discard that parameter. + // Callers pass `Environment::empty()` in E-expression contexts anyway. + EExpressionArgsIterator { + context: self.context, + raw_args: self.raw_invocation.raw_arguments(), } } } + +impl< + 'top, + 'data: 'top, + D: LazyDecoder<'data, MacroInvocation<'top> = EExpression<'top, 'data, D>>, + > From> for MacroExpr<'top, 'data, D> +{ + fn from(value: EExpression<'top, 'data, D>) -> Self { + MacroExpr::EExp(value) + } +} + +pub struct EExpressionArgsIterator<'top, 'data, D: LazyDecoder<'data>> { + context: EncodingContext<'top>, + raw_args: >::RawArgumentsIterator<'data>, +} + +impl<'top, 'data: 'top, D: LazyDecoder<'data>> Iterator + for EExpressionArgsIterator<'top, 'data, D> +{ + type Item = IonResult>; + + fn next(&mut self) -> Option { + let raw_arg: LazyRawValueExpr<'data, D> = match self.raw_args.next()? { + Ok(arg) => arg, + Err(e) => return Some(Err(e)), + }; + + let expr = match raw_arg { + LazyRawValueExpr::::ValueLiteral(value) => { + ArgumentExpr::ValueLiteral(LazyExpandedValue::from_value(self.context, value)) + } + LazyRawValueExpr::::MacroInvocation(raw_invocation) => { + let invocation = match raw_invocation.resolve(self.context) { + Ok(invocation) => invocation, + Err(e) => return Some(Err(e)), + }; + ArgumentExpr::MacroInvocation(invocation.into()) + } + }; + Some(Ok(expr)) + } +} + +pub type TextEExpression_1_1<'top, 'data> = EExpression<'top, 'data, TextEncoding_1_1>; diff --git a/src/lazy/expanded/macro_evaluator.rs b/src/lazy/expanded/macro_evaluator.rs index b67d1427..035f454c 100644 --- a/src/lazy/expanded/macro_evaluator.rs +++ b/src/lazy/expanded/macro_evaluator.rs @@ -10,175 +10,275 @@ //! The evaluation logic is the same for macros in both contexts, though there are differences in //! encoding and argument handling that must be considered. For more information, see the //! documentation for the types below. +#![allow(non_camel_case_types)] use std::fmt::{Debug, Formatter}; use std::marker::PhantomData; -use bumpalo::collections::{String as BumpString, Vec as BumpVec}; - -use crate::lazy::decoder::{LazyDecoder, RawArgumentExpr}; +use crate::lazy::decoder::{LazyDecoder, LazyRawValueExpr}; use crate::lazy::expanded::macro_table::MacroKind; use crate::lazy::expanded::sequence::Environment; -use crate::lazy::expanded::stack::{BumpVecStack, Stack, StackStorage, VecStack}; use crate::lazy::expanded::template::{ - TemplateBodyVariableReference, TemplateMacroInvocationAddress, + TemplateBodyValueExpr, TemplateBodyVariableReference, TemplateElement, TemplateMacroInvocation, + TemplateMacroInvocationArgsIterator, TemplateMacroRef, }; use crate::lazy::expanded::EncodingContext; use crate::lazy::expanded::{ExpandedValueRef, ExpandedValueSource, LazyExpandedValue}; use crate::lazy::str_ref::StrRef; -use crate::lazy::text::raw::v1_1::reader::{MacroAddress, MacroIdRef, TemplateBodyExprAddress}; +use crate::lazy::text::raw::v1_1::reader::MacroIdRef; use crate::result::IonFailure; use crate::{IonError, IonResult, RawSymbolTokenRef}; +use bumpalo::collections::{String as BumpString, Vec as BumpVec}; /// A syntactic entity that represents the invocation of a macro in some context. /// /// This entity may be an item from a binary stream, a text stream, or a template definition. /// Implementors must specify how their type can be mapped to a macro ID and a sequence of arguments. -pub trait MacroInvocation<'data, D: LazyDecoder<'data>>: Debug + Copy + Clone { - /// A syntax-specific type that represents an argument in this macro invocation. - type ArgumentExpr: ToArgumentKind<'data, D, Self>; - +pub trait RawMacroInvocation<'data, D: LazyDecoder<'data>>: Debug + Copy + Clone { /// An iterator that yields the macro invocation's arguments in order. - type ArgumentsIterator: Iterator>; - - // TODO: Item should be IonResult<_> - type RawArgumentsIterator: Iterator>; + type RawArgumentsIterator<'a>: Iterator>> + where + Self: 'a; - type TransientEvaluator<'context>: MacroEvaluator<'context, 'data, D, MacroInvocation = Self> + type MacroInvocation<'top>: MacroInvocation<'top, 'data, D> where - Self: 'context, - 'data: 'context; + 'data: 'top; /// The macro name or address specified at the head of this macro invocation. - fn id(&self) -> MacroIdRef; + fn id(&self) -> MacroIdRef<'data>; /// The arguments that follow the macro name or address in this macro invocation. - fn arguments(&self) -> Self::ArgumentsIterator; + fn raw_arguments(&self) -> Self::RawArgumentsIterator<'data>; - /// A version of the arguments that follow the macro name or address that does not depend - /// on the encoding context. - fn raw_arguments(&self) -> Self::RawArgumentsIterator; - - fn make_transient_evaluator<'context>( - context: EncodingContext<'context>, - environment: Environment<'context, 'data, D>, - ) -> Self::TransientEvaluator<'context> + fn resolve<'top>( + self, + context: EncodingContext<'top>, + ) -> IonResult> where - Self: 'context, - 'data: 'context; + 'data: 'top; +} + +pub trait MacroInvocation<'top, 'data: 'top, D: LazyDecoder<'data>>: + Debug + Copy + Clone + Into> +{ + type ArgumentsIterator: Iterator>>; + + fn id(&self) -> MacroIdRef<'top>; + fn arguments(&self, environment: Environment<'top, 'data, D>) -> Self::ArgumentsIterator; +} + +#[derive(Copy, Clone, Debug)] +pub enum MacroExpr<'top, 'data: 'top, D: LazyDecoder<'data>> { + TemplateMacro(TemplateMacroInvocation<'top>), + EExp(D::MacroInvocation<'top>), +} + +impl<'top, 'data: 'top, D: LazyDecoder<'data>> MacroExpr<'top, 'data, D> { + fn id(&self) -> MacroIdRef { + match &self { + MacroExpr::TemplateMacro(m) => { + as MacroInvocation<'top, 'data, D>>::id(m) + } + MacroExpr::EExp(e) => e.id(), + } + } + + fn arguments( + &self, + environment: Environment<'top, 'data, D>, + ) -> MacroExprArgsIterator<'top, 'data, D> { + let args_kind = match &self { + MacroExpr::TemplateMacro(m) => { + MacroExprArgsKind::<'top, 'data, D>::Macro(m.arguments(environment)) + } + MacroExpr::EExp(e) => { + MacroExprArgsKind::<'top, 'data, D>::EExp(e.arguments(Environment::empty())) + } + }; + MacroExprArgsIterator { source: args_kind } + } +} + +pub enum MacroExprArgsKind<'top, 'data: 'top, D: LazyDecoder<'data>> { + Macro(TemplateMacroInvocationArgsIterator<'top, 'data, D>), + EExp( as MacroInvocation<'top, 'data, D>>::ArgumentsIterator), +} + +pub struct MacroExprArgsIterator<'top, 'data: 'top, D: LazyDecoder<'data>> { + source: MacroExprArgsKind<'top, 'data, D>, +} + +impl<'top, 'data: 'top, D: LazyDecoder<'data>> Iterator for MacroExprArgsIterator<'top, 'data, D> { + type Item = IonResult>; + + fn next(&mut self) -> Option { + match &mut self.source { + MacroExprArgsKind::Macro(m) => m.next(), + MacroExprArgsKind::EExp(e) => e.next(), + } + } } /// A single expression appearing in argument position within a macro invocation. -pub enum ArgumentKind<'top, 'data: 'top, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { +#[derive(Debug, Clone)] +pub enum ArgumentExpr<'top, 'data: 'top, D: LazyDecoder<'data>> { /// An Ion value that requires no further evaluation. + // `LazyExpandedValue` can be backed by either a stream value or a template value, so it covers + // both contexts. ValueLiteral(LazyExpandedValue<'top, 'data, D>), /// A variable name that requires expansion. Variable(TemplateBodyVariableReference), - /// A macro invocation that requires evaluation. - MacroInvocation(M), + /// A template macro invocation that requires evaluation. + MacroInvocation(MacroExpr<'top, 'data, D>), } -/// Converts a syntactic element appearing in argument position into an [`ArgumentKind`] using the -/// provided [`EncodingContext`]. -pub trait ToArgumentKind<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { - fn to_arg_expr<'top>( - self, - context: EncodingContext<'top>, - environment: Environment<'top, 'data, D>, - ) -> ArgumentKind<'top, 'data, D, M> - where - M: 'top, - Self: 'top; -} - -pub struct VariableExpansion<'top, 'data, D: LazyDecoder<'data>> { +pub struct VariableExpansion< + 'top, + 'data: 'top, + D: LazyDecoder<'data>, + M: MacroInvocation<'top, 'data, D>, +> { context: EncodingContext<'top>, - variable_name: &'top str, // useful? - args_iter: >::ArgumentsIterator, - evaluator: EExpEvaluator<'top, 'data, D>, + spooky: PhantomData<(&'data D, &'top M)>, // variable_name: &'top str, // useful? + // args_iter: >::RawArgumentsIterator<'data>, + // evaluator: EExpEvaluator<'top, 'data, D>, } -impl<'top, 'data, D: LazyDecoder<'data>> VariableExpansion<'top, 'data, D> { +impl<'top, 'data, D: LazyDecoder<'data>, M: MacroInvocation<'top, 'data, D>> + VariableExpansion<'top, 'data, D, M> +{ fn next( &mut self, - environment: Environment<'top, 'data, D>, + _environment: Environment<'top, 'data, D>, ) -> Option>> { - loop { - // If the evaluator's stack is not empty, it's still expanding a macro. - if self.evaluator.stack_depth() > 0 { - let value = self.evaluator.next(self.context, 0).transpose(); - if value.is_some() { - // The `Some` may contain a value or an error; either way, that's the next return value. - return value; - } - // It's possible for a macro to produce zero values. If that happens, we continue on to - // pull another expression from the arguments iterator. - } - - let next_arg_expr = match self.args_iter.next()? { - Ok(expr) => expr, - Err(e) => return Some(Err(e)), - }; - - let arg_kind: ArgumentKind = - next_arg_expr.to_arg_expr(self.context, environment); - match arg_kind { - ArgumentKind::ValueLiteral(value) => return Some(Ok(value)), - ArgumentKind::MacroInvocation(invocation) => { - let begin_expansion_result = self.evaluator.push(self.context, invocation); - if let Err(e) = begin_expansion_result { - return Some(Err(e)); - } - continue; - } - ArgumentKind::Variable(_) => { - unreachable!("e-expressions cannot reference variables"); - } - } - } + // loop { + // // If the evaluator's stack is not empty, it's still expanding a macro. + // if self.evaluator.stack_depth() > 0 { + // let value = self.evaluator.next(self.context, 0).transpose(); + // if value.is_some() { + // // The `Some` may contain a value or an error; either way, that's the next return value. + // return value; + // } + // // It's possible for a macro to produce zero values. If that happens, we continue on to + // // pull another expression from the arguments iterator. + // } + // + // let next_arg_expr = match self.args_iter.next()? { + // Ok(expr) => expr, + // Err(e) => return Some(Err(e)), + // }; + // + // let arg_kind: ArgumentExpr> = + // next_arg_expr.to_arg_expr(self.context, environment); + // match arg_kind { + // ArgumentExpr::ValueLiteral(value) => return Some(Ok(value)), + // ArgumentExpr::MacroInvocation(invocation) => { + // let begin_expansion_result = self.evaluator.push(self.context, invocation); + // if let Err(e) = begin_expansion_result { + // return Some(Err(e)); + // } + // continue; + // } + // ArgumentExpr::Variable(_) => { + // unreachable!("e-expressions cannot reference variables"); + // } + // } + // } + todo!("variable expansion") } } /// Indicates which of the supported macros this represents and stores the state necessary to /// continue evaluating that macro. -pub enum MacroExpansionKind<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { +pub enum MacroExpansionKind<'top, 'data, D: LazyDecoder<'data>> { Void, - Values(ValuesExpansion<'data, D, M>), - MakeString(MakeStringExpansion<'data, D, M>), - Template(TemplateExpansion), + Values(ValuesExpansion<'top, 'data, D>), + MakeString(MakeStringExpansion<'top, 'data, D>), + Template(TemplateExpansion<'top>), // TODO: The others, including template macros. } +#[derive(Clone, Debug)] +pub struct TemplateExpansion<'top> { + template: TemplateMacroRef<'top>, + step_index: usize, +} + +impl<'top> TemplateExpansion<'top> { + pub fn new(template: TemplateMacroRef<'top>) -> Self { + Self { + template, + step_index: 0, + } + } + + fn next<'data: 'top, D: LazyDecoder<'data>>( + &mut self, + context: EncodingContext<'top>, + environment: Environment<'top, 'data, D>, + ) -> IonResult> { + let value_expr = match self.template.body().expressions().get(self.step_index) { + None => return Ok(MacroExpansionStep::Complete), + Some(expr) => expr, + }; + self.step_index += 1; + + let step = match value_expr { + TemplateBodyValueExpr::Element(e) => { + MacroExpansionStep::ExpandedValue(LazyExpandedValue::from_template( + context, + environment, + TemplateElement::new(self.template, e), + )) + } + TemplateBodyValueExpr::Variable(_) => { + todo!("variable expansion") + } + TemplateBodyValueExpr::MacroInvocation(raw_invocation) => { + let invocation = raw_invocation.resolve(self.template, context)?; + self.step_index += invocation.arg_expressions().len(); + MacroExpansionStep::MacroToEvaluate(invocation.into()) + } + }; + + Ok(step) + } +} + /// A macro in the process of being evaluated. Stores both the state of the evaluation and the /// syntactic element that represented the macro invocation. -pub struct MacroExpansion<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { - kind: MacroExpansionKind<'data, D, M>, - // Not every macro expansion is initiated by - invocation: Option, +pub struct MacroExpansion<'top, 'data, D: LazyDecoder<'data>> { + kind: MacroExpansionKind<'top, 'data, D>, + // Not every macro expansion is initiated by an expression in the input; the evaluator itself + // can add expansions to the stack as an implementation detail (e.g. for variable expansion). + // In such cases, the `kind` stores any information needed to perform the expansion. + invocation: Option>, } -impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> Debug - for MacroExpansion<'data, D, M> -{ +impl<'top, 'data, D: LazyDecoder<'data>> Debug for MacroExpansion<'top, 'data, D> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "", self.invocation) + let name = match &self.kind { + MacroExpansionKind::Void => "void", + MacroExpansionKind::Values(_) => "values", + MacroExpansionKind::MakeString(_) => "make_string", + MacroExpansionKind::Template(t) => { + return write!(f, "", t.template.name()) + } + }; + write!(f, "") } } -impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> MacroExpansion<'data, D, M> { +impl<'top, 'data: 'top, D: LazyDecoder<'data>> MacroExpansion<'top, 'data, D> { /// Continues evaluating this macro until it: /// * produces another value. /// * encounters another macro or variable that needs to be expanded. /// * is completed. - fn next<'top>( + fn next( &mut self, context: EncodingContext<'top>, environment: Environment<'top, 'data, D>, - ) -> IonResult> - where - 'data: 'top, - M: 'top, - { + ) -> IonResult> { use MacroExpansionKind::*; // Delegate the call to `next()` based on the macro kind. match &mut self.kind { @@ -186,22 +286,23 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> MacroExpansion< Values(values_expansion) => values_expansion.next(context, environment), // `void` is trivial and requires no delegation Void => Ok(MacroExpansionStep::Complete), - Template(_template) => todo!("support {_template:?}"), + Template(template_expansion) => template_expansion.next(context, environment), } } } /// Represents a single step in the process of evaluating a macro. -pub enum MacroExpansionStep<'top, 'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { +pub enum MacroExpansionStep<'top, 'data, D: LazyDecoder<'data>> { /// The next value produced by continuing the macro evaluation. ExpandedValue(LazyExpandedValue<'top, 'data, D>), /// Another macro that will need to be evaluated before an expanded value can be returned. - AnotherMacroToEvaluate(M), + MacroToEvaluate(MacroExpr<'top, 'data, D>), /// This macro will not produce any further values. Complete, } -pub type MacroStack<'top, 'data, D, M> = BumpVec<'top, MacroExpansion<'data, D, M>>; +pub type MacroStack<'top, 'data, D> = BumpVec<'top, MacroExpansion<'top, 'data, D>>; +pub type EnvironmentStack<'top, 'data, D> = BumpVec<'top, Environment<'top, 'data, D>>; /// Evaluates macro invocations recursively, yielding a single expanded value at a time. /// @@ -211,29 +312,46 @@ pub type MacroStack<'top, 'data, D, M> = BumpVec<'top, MacroExpansion<'data, D, /// {e-expression, template macro invocation} /// x {text, binary} /// x {eager, lazy} -pub trait MacroEvaluator<'top, 'data: 'top, D: LazyDecoder<'data>> -where - Self: 'top, -{ - type MacroInvocation: MacroInvocation<'data, D>; +pub struct MacroEvaluator<'top, 'data: 'top, D: LazyDecoder<'data>> { + // TODO: Describe how macro stack and environment stack do not grow at the same rate + macro_stack: MacroStack<'top, 'data, D>, + context: EncodingContext<'top>, + env_stack: EnvironmentStack<'top, 'data, D>, +} + +impl<'top, 'data: 'top, D: LazyDecoder<'data>> MacroEvaluator<'top, 'data, D> { + pub fn new(context: EncodingContext<'top>, environment: Environment<'top, 'data, D>) -> Self { + let macro_stack = BumpVec::new_in(context.allocator); + let mut env_stack = BumpVec::new_in(context.allocator); + env_stack.push(environment); + Self { + macro_stack, + env_stack, + context, + } + } +} - fn macro_stack(&self) -> &MacroStack<'top, 'data, D, Self::MacroInvocation>; - fn macro_stack_mut(&mut self) -> &mut MacroStack<'top, 'data, D, Self::MacroInvocation>; - fn stack_depth(&self) -> usize { - self.macro_stack().len() +impl<'top, 'data: 'top, D: LazyDecoder<'data>> MacroEvaluator<'top, 'data, D> { + pub fn macro_stack_depth(&self) -> usize { + self.macro_stack.len() } - fn environment(&self) -> Environment<'top, 'data, D>; - fn environment_mut(&mut self) -> &mut Environment<'top, 'data, D>; + pub fn environment(&self) -> Environment<'top, 'data, D> { + // The stack is never completely empty. + *self.env_stack.last().unwrap() + } fn make_eval_env( &mut self, context: EncodingContext<'top>, - invocation: Self::MacroInvocation, + invocation: MacroExpr<'top, 'data, D>, ) -> IonResult> { + // Using the current environment, make a new environment for the next invocation. + // TODO: Explain better let mut args = BumpVec::new_in(context.allocator); - for raw_arg in invocation.raw_arguments() { - args.push(raw_arg); + for arg in invocation.arguments(self.environment()) { + args.push(arg?); } let environment = Environment::new(args); Ok(environment) @@ -245,46 +363,40 @@ where fn resolve_invocation( &mut self, context: EncodingContext<'top>, - invocation_to_evaluate: Self::MacroInvocation, - ) -> IonResult> { - // Get the `MacroKind` corresponding to the given ID. It contains either a name (`&str`) or - // an address (`usize`). - let macro_kind = match invocation_to_evaluate.id() { - MacroIdRef::LocalName(name) => { - context.macro_table.macro_with_name(name).ok_or_else(|| { - IonError::decoding_error(format!( - "unrecognized macro name '{name}' in {:?}", - invocation_to_evaluate - )) - }) + invocation_to_evaluate: MacroExpr<'top, 'data, D>, + ) -> IonResult> { + // Get the `MacroKind` corresponding to the given ID. + let macro_kind = match self + .context + .macro_table + .macro_with_id(invocation_to_evaluate.id()) + { + Some(kind) => kind, + None => { + return IonResult::decoding_error(format!( + "found unrecognized macro id {:?}", + invocation_to_evaluate.id() + )) } - MacroIdRef::LocalAddress(address) => context - .macro_table - .macro_at_address(address) - .ok_or_else(|| { - IonError::decoding_error(format!( - "invalid macro address '{address}' in {:?}", - invocation_to_evaluate - )) - }), - }?; + }; // Initialize a `MacroExpansionKind` with the state necessary to evaluate the requested // macro. let expansion_kind = match macro_kind.kind() { MacroKind::Void => MacroExpansionKind::Void, MacroKind::Values => MacroExpansionKind::Values(ValuesExpansion { - arguments: invocation_to_evaluate.arguments(), - initial_eval_stack_depth: self.stack_depth(), + arguments: invocation_to_evaluate.arguments(self.environment()), + initial_eval_stack_depth: self.macro_stack_depth(), }), MacroKind::MakeString => MacroExpansionKind::MakeString(MakeStringExpansion::new( - invocation_to_evaluate.arguments(), + invocation_to_evaluate.arguments(self.environment()), )), - MacroKind::Template(_template) => { - let environment = self.make_eval_env(context, invocation_to_evaluate)?; - MacroExpansionKind::Template(TemplateExpansion::new( - macro_kind.address(), - environment.args().len(), - )) + MacroKind::Template(template) => { + let template_address = macro_kind.address(); + let template_ref = TemplateMacroRef::new(template_address, template); + let new_environment = self.make_eval_env(context, invocation_to_evaluate)?; + println!("push {new_environment:?}"); + self.env_stack.push(new_environment); + MacroExpansionKind::Template(TemplateExpansion::new(template_ref)) } MacroKind::ExpandVariable => todo!("variable expansion"), }; @@ -296,13 +408,15 @@ where /// Given a syntactic element representing a macro invocation, attempt to resolve it with the /// current encoding context and push the resulting `MacroExpansion` onto the stack. - fn push( + pub fn push( &mut self, context: EncodingContext<'top>, - invocation: Self::MacroInvocation, + invocation: impl Into>, ) -> IonResult<()> { - let expansion = self.resolve_invocation(context, invocation)?; - self.macro_stack_mut().push(expansion); + let macro_expr = invocation.into(); + println!("push {macro_expr:?}"); + let expansion = self.resolve_invocation(context, macro_expr)?; + self.macro_stack.push(expansion); Ok(()) } @@ -321,23 +435,19 @@ where /// /// The caller must verify that the stack's depth is greater than or equal to `depth_to_exhaust` /// before calling `next()`. - fn next( + pub fn next( &mut self, context: EncodingContext<'top>, depth_to_exhaust: usize, - ) -> IonResult>> - where - Self::MacroInvocation: 'top, - { + ) -> IonResult>> { debug_assert!( - self.stack_depth() >= depth_to_exhaust, + self.macro_stack_depth() >= depth_to_exhaust, "asked to exhaust a macro at an invalid depth" ); - // TODO: Could limit this to just the top template's args let environment = self.environment(); loop { // Get the expansion at the top of the stack. - let current_expansion = match self.macro_stack_mut().peek_mut() { + let current_expansion = match self.macro_stack.last_mut() { // NOTE: If the user specifies a `depth_to_exhaust` of 0, this is where the loop // will end. Behaviorally, this is identical to a `depth_to_exhaust` of 1, // which would return `Ok(None)` at the bottom of this method. It is always @@ -351,9 +461,12 @@ where use MacroExpansionStep::*; match current_expansion.next(context, environment)? { // If we get a value, return it to the caller. - ExpandedValue(value) => return Ok(Some(value)), + ExpandedValue(value) => { + println!("yield {value:?}"); + return Ok(Some(value)); + } // If we get another macro, push it onto the stack and continue evaluation. - AnotherMacroToEvaluate(invocation) => { + MacroToEvaluate(invocation) => { // If we encounter another macro invocation, put it on top of the stack. self.push(context, invocation)?; continue; @@ -361,9 +474,18 @@ where // If the current macro reports that its expansion is complete... Complete => { // ...pop it off the stack... - let _popped = self.macro_stack_mut().pop().unwrap(); + let completed = self.macro_stack.pop().unwrap(); + println!( + "pop {completed:?}, macro stack depth={}, top={:?}", + self.macro_stack_depth(), + self.macro_stack.last(), + ); + if matches!(completed.kind, MacroExpansionKind::Template(_)) { + let popped_env = self.env_stack.pop().unwrap(); + println!("pop {popped_env:?}"); + } // ...and see that was the macro the caller was interested in evaluating. - if self.macro_stack().len() < depth_to_exhaust { + if self.macro_stack.len() < depth_to_exhaust { // If so, there are no more values to yield, even though there may still // be macros on the stack. return Ok(None); @@ -378,40 +500,35 @@ where /// Attempts to resolve the provided `invocation` in the specified `context`. Upon success, /// returns an iterator that lazily computes the expansion of the macro invocation and yields /// its values. - fn evaluate<'iter>( + pub fn evaluate<'iter>( &'iter mut self, context: EncodingContext<'top>, - invocation: Self::MacroInvocation, - ) -> IonResult> + invocation: impl Into>, + ) -> IonResult> where 'data: 'top, Self: Sized, { - self.push(context, invocation)?; + self.push(context, invocation.into())?; Ok(EvaluatingIterator::new(self, context)) } } /// Yields the values produced by incrementally evaluating the macro that was at the top of the /// evaluator's stack when the iterator was created. -pub struct EvaluatingIterator< - 'iter, - 'top, - 'data: 'top, - D: LazyDecoder<'data>, - E: MacroEvaluator<'top, 'data, D>, -> { - evaluator: &'iter mut E, +pub struct EvaluatingIterator<'iter, 'top, 'data: 'top, D: LazyDecoder<'data>> { + evaluator: &'iter mut MacroEvaluator<'top, 'data, D>, context: EncodingContext<'top>, initial_stack_depth: usize, spooky: PhantomData<&'data D>, } -impl<'iter, 'top, 'data, D: LazyDecoder<'data>, E: MacroEvaluator<'top, 'data, D>> - EvaluatingIterator<'iter, 'top, 'data, D, E> -{ - pub fn new(evaluator: &'iter mut E, context: EncodingContext<'top>) -> Self { - let initial_stack_depth = evaluator.stack_depth(); +impl<'iter, 'top, 'data, D: LazyDecoder<'data>> EvaluatingIterator<'iter, 'top, 'data, D> { + pub fn new( + evaluator: &'iter mut MacroEvaluator<'top, 'data, D>, + context: EncodingContext<'top>, + ) -> Self { + let initial_stack_depth = evaluator.macro_stack_depth(); Self { evaluator, context, @@ -421,8 +538,8 @@ impl<'iter, 'top, 'data, D: LazyDecoder<'data>, E: MacroEvaluator<'top, 'data, D } } -impl<'iter, 'top, 'data: 'top, D: LazyDecoder<'data>, E: MacroEvaluator<'top, 'data, D> + 'top> - Iterator for EvaluatingIterator<'iter, 'top, 'data, D, E> +impl<'iter, 'top, 'data: 'top, D: LazyDecoder<'data>> Iterator + for EvaluatingIterator<'iter, 'top, 'data, D> { type Item = IonResult>; @@ -453,97 +570,97 @@ impl<'iter, 'top, 'data: 'top, D: LazyDecoder<'data>, E: MacroEvaluator<'top, 'd /// evaluator, making it (practically) impossible to modify the stack by pushing another /// MacroExpansion onto it. Instead, it creates an evaluator of its own using short-lived, /// bump-allocated storage and fully evaluates each argument. -pub struct EExpEvaluator<'top, 'data, D: LazyDecoder<'data>> { - macro_stack: BumpVec<'top, MacroExpansion<'data, D, D::MacroInvocation>>, - arg_stack: BumpVec<'top, RawArgumentExpr<'data, D>>, -} - -impl<'top, 'data, D: LazyDecoder<'data>> EExpEvaluator<'top, 'data, D> { - pub fn new(context: EncodingContext<'top>) -> Self { - Self { - macro_stack: BumpVec::new_in(context.allocator), - arg_stack: BumpVec::new_in(context.allocator), - } - } -} - -impl<'top, 'data: 'top, D: LazyDecoder<'data>> MacroEvaluator<'top, 'data, D> - for EExpEvaluator<'top, 'data, D> -{ - type MacroInvocation = D::MacroInvocation; - - fn macro_stack(&self) -> &MacroStack<'top, 'data, D, Self::MacroInvocation> { - &self.macro_stack - } - - fn macro_stack_mut(&mut self) -> &mut MacroStack<'top, 'data, D, Self::MacroInvocation> { - &mut self.macro_stack - } - - fn environment(&self) -> Environment<'top, 'data, D> { - todo!("env in TransientEExpEvaluator") - } - - fn environment_mut(&mut self) -> &mut Environment<'top, 'data, D> { - todo!("env_mut in TransientEExpEvaluator") - } -} - -#[derive(Clone, Copy, Debug)] -pub struct TemplateExpansion { - macro_address: MacroAddress, - num_args: usize, - // Which step (expression) of the macro definition we're in the process of expanding - step_index: TemplateBodyExprAddress, -} - -impl TemplateExpansion { - pub fn new(macro_address: MacroAddress, num_args: usize) -> Self { - Self { - macro_address, - num_args, - step_index: 0, - } - } -} - -/// A [`MacroEvaluator`] for expanding macro invocations found in a template, all in the context -/// of a data stream in the format `D`. -pub struct TemplateEvaluator<'top, 'data, D: LazyDecoder<'data>> { - macro_stack: BumpVec<'top, MacroExpansion<'data, D, TemplateMacroInvocationAddress>>, - environment: Environment<'top, 'data, D>, -} - -impl<'top, 'data, D: LazyDecoder<'data>> TemplateEvaluator<'top, 'data, D> { - pub fn new(context: EncodingContext<'top>, environment: Environment<'top, 'data, D>) -> Self { - Self { - environment, - macro_stack: BumpVec::new_in(context.allocator), - } - } -} - -impl<'top, 'data, D: LazyDecoder<'data>> MacroEvaluator<'top, 'data, D> - for TemplateEvaluator<'top, 'data, D> -{ - type MacroInvocation = TemplateMacroInvocationAddress; - - fn macro_stack(&self) -> &MacroStack<'top, 'data, D, Self::MacroInvocation> { - &self.macro_stack - } - - fn macro_stack_mut(&mut self) -> &mut MacroStack<'top, 'data, D, Self::MacroInvocation> { - &mut self.macro_stack - } - - fn environment(&self) -> Environment<'top, 'data, D> { - self.environment - } - - fn environment_mut(&mut self) -> &mut Environment<'top, 'data, D> { - &mut self.environment - } -} +// pub struct EExpEvaluator<'top, 'data, D: LazyDecoder<'data>> { +// macro_stack: BumpVec<'top, MacroExpansion<'top, 'data, D>>, +// arg_stack: BumpVec<'top, LazyRawValueExpr<'data, D>>, +// } +// +// impl<'top, 'data, D: LazyDecoder<'data>> EExpEvaluator<'top, 'data, D> { +// pub fn new(context: EncodingContext<'top>) -> Self { +// Self { +// macro_stack: BumpVec::new_in(context.allocator), +// arg_stack: BumpVec::new_in(context.allocator), +// } +// } +// } +// +// impl<'top, 'data: 'top, D: LazyDecoder<'data>> MacroEvaluator<'top, 'data, D> +// for EExpEvaluator<'top, 'data, D> +// { +// type MacroInvocation = D::MacroInvocation<'top>; +// +// fn macro_stack(&self) -> &MacroStack<'top, 'data, D, Self::MacroInvocation> { +// &self.macro_stack +// } +// +// fn macro_stack_mut(&mut self) -> &mut MacroStack<'top, 'data, D, Self::MacroInvocation> { +// &mut self.macro_stack +// } +// +// fn environment(&self) -> Environment<'top, 'data, D> { +// Environment::empty() +// } +// +// fn environment_mut(&mut self) -> &mut Environment<'top, 'data, D> { +// todo!("env_mut in TransientEExpEvaluator") +// } +// } +// +// #[derive(Clone, Copy, Debug)] +// pub struct TemplateExpansion { +// macro_address: MacroAddress, +// num_args: usize, +// // Which step (expression) of the macro definition we're in the process of expanding +// step_index: TemplateBodyExprAddress, +// } +// +// impl TemplateExpansion { +// pub fn new(macro_address: MacroAddress, num_args: usize) -> Self { +// Self { +// macro_address, +// num_args, +// step_index: 0, +// } +// } +// } +// +// /// A [`MacroEvaluator`] for expanding macro invocations found in a template, all in the context +// /// of a data stream in the format `D`. +// pub struct TemplateEvaluator<'top, 'data, D: LazyDecoder<'data>> { +// macro_stack: BumpVec<'top, MacroExpansion<'top, 'data, D, TemplateMacroInvocation<'top>>>, +// environment: Environment<'top, 'data, D>, +// } +// +// impl<'top, 'data, D: LazyDecoder<'data>> TemplateEvaluator<'top, 'data, D> { +// pub fn new(context: EncodingContext<'top>, environment: Environment<'top, 'data, D>) -> Self { +// Self { +// environment, +// macro_stack: BumpVec::new_in(context.allocator), +// } +// } +// } +// +// impl<'top, 'data, D: LazyDecoder<'data>> MacroEvaluator<'top, 'data, D> +// for TemplateEvaluator<'top, 'data, D> +// { +// type MacroInvocation = TemplateMacroInvocation<'top>; +// +// fn macro_stack(&self) -> &MacroStack<'top, 'data, D, Self::MacroInvocation> { +// &self.macro_stack +// } +// +// fn macro_stack_mut(&mut self) -> &mut MacroStack<'top, 'data, D, Self::MacroInvocation> { +// &mut self.macro_stack +// } +// +// fn environment(&self) -> Environment<'top, 'data, D> { +// self.environment +// } +// +// fn environment_mut(&mut self) -> &mut Environment<'top, 'data, D> { +// &mut self.environment +// } +// } // ===== Implementation of the `values` macro ===== @@ -557,16 +674,19 @@ impl<'top, 'data, D: LazyDecoder<'data>> MacroEvaluator<'top, 'data, D> /// (:values 1) => 1 /// (:values 1 2 3) => 1 2 3 /// (:values 1 2 (:values 3 4)) => 1 2 3 4 -pub struct ValuesExpansion<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { +pub struct ValuesExpansion<'top, 'data, D: LazyDecoder<'data>> { // Which argument the macro is in the process of expanding - arguments: M::ArgumentsIterator, + arguments: MacroExprArgsIterator<'top, 'data, D>, // The stack depth where this `values` call lives. When the stack shrinks below this depth, // evaluation is complete. initial_eval_stack_depth: usize, } -impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> ValuesExpansion<'data, D, M> { - pub fn new(arguments: M::ArgumentsIterator, initial_eval_stack_depth: usize) -> Self { +impl<'top, 'data, D: LazyDecoder<'data>> ValuesExpansion<'top, 'data, D> { + pub fn new( + arguments: MacroExprArgsIterator<'top, 'data, D>, + initial_eval_stack_depth: usize, + ) -> Self { Self { arguments, initial_eval_stack_depth, @@ -574,29 +694,25 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> ValuesExpansion } /// Yields the next [`MacroExpansionStep`] in this macro's evaluation. - pub fn next<'top>( + pub fn next( &mut self, - context: EncodingContext<'top>, - environment: Environment<'top, 'data, D>, - ) -> IonResult> - where - 'data: 'top, - M: 'top, - { + _context: EncodingContext<'top>, + _environment: Environment<'top, 'data, D>, + ) -> IonResult> { // We visit the argument expressions in the invocation in order from left to right. let arg_expr = match self.arguments.next() { Some(Err(e)) => return Err(e), - Some(Ok(arg)) => arg.to_arg_expr(context, environment), + Some(Ok(arg)) => arg, None => return Ok(MacroExpansionStep::Complete), }; match arg_expr { // If the argument is a value, return it. - ArgumentKind::ValueLiteral(value) => Ok(MacroExpansionStep::ExpandedValue(value)), - ArgumentKind::Variable(_variable) => todo!("variable expansion"), + ArgumentExpr::ValueLiteral(value) => Ok(MacroExpansionStep::ExpandedValue(value)), + ArgumentExpr::Variable(_variable) => todo!("variable expansion"), // If the argument is a macro invocation, yield it that so the evaluator can push it onto the stack. - ArgumentKind::MacroInvocation(invocation) => { - Ok(MacroExpansionStep::AnotherMacroToEvaluate(invocation)) + ArgumentExpr::MacroInvocation(invocation) => { + Ok(MacroExpansionStep::MacroToEvaluate(invocation)) } } } @@ -622,33 +738,25 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> ValuesExpansion /// (:make_string (:values "first" "_") $4) => "first_name" /// (:make_string) => "" /// (:make_string "foo" 7) => Error -pub struct MakeStringExpansion<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { - arguments: M::ArgumentsIterator, +pub struct MakeStringExpansion<'top, 'data, D: LazyDecoder<'data>> { + arguments: MacroExprArgsIterator<'top, 'data, D>, is_complete: bool, - spooky: PhantomData, } -impl<'top, 'data: 'top, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> - MakeStringExpansion<'data, D, M> -{ - pub fn new(arguments: M::ArgumentsIterator) -> Self { +impl<'top, 'data: 'top, D: LazyDecoder<'data>> MakeStringExpansion<'top, 'data, D> { + pub fn new(arguments: MacroExprArgsIterator<'top, 'data, D>) -> Self { Self { arguments, is_complete: false, - spooky: Default::default(), } } - /// Yields the next [`MacroExpansionStep`] in this macro's evaluation. + /// Yields the next [`MacroExpansionStep`] in this `make_string` macro's evaluation. pub fn next( &mut self, context: EncodingContext<'top>, environment: Environment<'top, 'data, D>, - ) -> IonResult> - where - 'data: 'top, - M: 'top, - { + ) -> IonResult> { // `make_string` always produces a single value. Once that value has been returned, it needs // to report `Complete` on the following call to `next()`. if self.is_complete { @@ -663,16 +771,16 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> // inside the evaluator, we cannot get a simultaneous mutable reference to the evaluator // itself. Instead, we use the bump allocator the make a transient macro evaluator // whose resources can be trivially reclaimed when the expansion is done. - let mut evaluator = M::make_transient_evaluator(context, environment); + let mut evaluator = MacroEvaluator::new(context, environment); - for arg in self.arguments.by_ref() { - let arg_expr: ArgumentKind = arg?.to_arg_expr(context, environment); + for arg_result in &mut self.arguments { + let arg_expr = arg_result?; match arg_expr { - ArgumentKind::ValueLiteral(value) => { + ArgumentExpr::ValueLiteral(value) => { Self::append_expanded_raw_text_value(context, &mut buffer, value.read()?)? } - ArgumentKind::Variable(_variable) => todo!("variable expansion"), - ArgumentKind::MacroInvocation(invocation) => { + ArgumentExpr::Variable(_variable) => todo!("variable expansion"), + ArgumentExpr::MacroInvocation(invocation) => { for value_result in evaluator.evaluate(context, invocation)? { let value = value_result?; let expanded = value.read()?; @@ -734,8 +842,9 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> #[cfg(test)] mod tests { + use crate::lazy::expanded::compiler::TemplateCompiler; use crate::lazy::reader::LazyTextReader_1_1; - use crate::{ElementReader, IonResult}; + use crate::{Element, ElementReader, IonResult}; /// Reads `input` and `expected` using an expanding reader and asserts that their output /// is the same. @@ -761,32 +870,31 @@ mod tests { /// /// This test exists to demonstrate that macro evaluation within the TDL context works the /// same as evaluation in the data stream. - fn eval_tdl_template_invocation(invocation: &str, expected: &str) -> IonResult<()> { - todo!() - // let macro_table = MacroTable::new(); - // let symbol_table = SymbolTable::new(); - // let allocator = BumpAllocator::new(); - // let context = EncodingContext::new(¯o_table, &symbol_table, &allocator); - // let invocation = Element::read_one(invocation)?; - // let mut evaluator = TemplateEvaluator::::new(context); - // let actuals = evaluator.evaluate(context, invocation.expect_sexp()?)?; - // let mut expected_reader = LazyTextReader_1_1::new(expected.as_bytes())?; - // for actual_result in actuals { - // // Read the next expected value as a raw value, then wrap it in an `ExpandedRawValueRef` - // // so it can be directly compared to the actual. - // let expected: Element = expected_reader.next()?.unwrap().read()?.try_into()?; - // let actual: Element = LazyValue::from(actual_result?).try_into()?; - // assert_eq!(actual, expected); - // } - // assert!(matches!(expected_reader.next(), Ok(None))); - // - // Ok(()) + fn eval_template_invocation(template: &str, invocation: &str, expected: &str) -> IonResult<()> { + let mut reader = LazyTextReader_1_1::new(invocation.as_bytes())?; + let template_macro = dbg!(TemplateCompiler::compile_from_text( + reader.context(), + template + )?); + let _macro_address = reader.register_template(template_macro)?; + let actuals = reader.read_all_elements()?; + let mut expected_reader = LazyTextReader_1_1::new(expected.as_bytes())?; + for actual in actuals { + // Read the next expected value as a raw value, then wrap it in an `ExpandedRawValueRef` + // so it can be directly compared to the actual. + let expected: Element = expected_reader.next()?.unwrap().read()?.try_into()?; + assert_eq!(actual, expected); + } + assert!(matches!(expected_reader.next(), Ok(None))); + + Ok(()) } #[test] fn values_tdl_macro_invocation() -> IonResult<()> { - eval_tdl_template_invocation( - r"(values 1 2 (values 3 4 (values 5 6) 7 8) 9 10)", + eval_template_invocation( + r"(macro foo () (values 1 2 (values 3 4 (values 5 6) 7 8) 9 10))", + "(:foo)", "1 2 3 4 5 6 7 8 9 10", ) } @@ -806,7 +914,11 @@ mod tests { #[test] fn void_tdl_macro_invocation() -> IonResult<()> { - eval_tdl_template_invocation(r"(values (void) (void) (void))", "/* nothing */") + eval_template_invocation( + r"(macro foo () (values (void) (void) (void)))", + "(:foo)", + "/* nothing */", + ) } #[test] @@ -827,14 +939,16 @@ mod tests { #[test] fn make_string_tdl_macro_invocation() -> IonResult<()> { let invocation = r#" - (values + (macro foo () + (values (make_string "foo" '''bar''' "\x62\u0061\U0000007A") (make_string '''Hello''' ''', ''' "world!")) + ) "#; - eval_tdl_template_invocation(invocation, r#" "foobarbaz" "Hello, world!" "#) + eval_template_invocation(invocation, "(:foo)", r#" "foobarbaz" "Hello, world!" "#) } #[test] @@ -848,8 +962,9 @@ mod tests { #[test] fn macros_inside_a_tdl_list() -> IonResult<()> { - eval_tdl_template_invocation( + eval_template_invocation( r#" + (macro foo () (values [ 1, 2, @@ -861,7 +976,9 @@ mod tests { (make_string "foo" "bar" "baz"), 7 ]) + ) "#, + "(:foo)", "[1, 2, 3, 4, 5, 6, \"foobarbaz\", 7]", )?; Ok(()) @@ -931,8 +1048,9 @@ mod tests { #[test] fn macros_inside_a_tdl_struct() -> IonResult<()> { - eval_tdl_template_invocation( + eval_template_invocation( r#" + (macro foo () (values { a: 1, @@ -959,7 +1077,9 @@ mod tests { g: 6 }) + ) "#, + "(:foo)", r#" { a: 1, @@ -975,6 +1095,7 @@ mod tests { }, g: 6, } + "#, )?; Ok(()) diff --git a/src/lazy/expanded/macro_table.rs b/src/lazy/expanded/macro_table.rs index 031c9727..52e2dfb1 100644 --- a/src/lazy/expanded/macro_table.rs +++ b/src/lazy/expanded/macro_table.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; -use crate::lazy::expanded::template::TemplateMacro; -use crate::lazy::text::raw::v1_1::reader::MacroAddress; +use crate::lazy::expanded::template::{TemplateMacro, TemplateMacroRef}; +use crate::lazy::text::raw::v1_1::reader::{MacroAddress, MacroIdRef}; use crate::result::IonFailure; use crate::IonResult; @@ -36,13 +36,13 @@ impl MacroKind { } } -#[derive(Debug, Clone)] -pub struct MacroKindRef<'top> { +#[derive(Debug, Copy, Clone)] +pub struct MacroRef<'top> { address: MacroAddress, kind: &'top MacroKind, } -impl<'top> MacroKindRef<'top> { +impl<'top> MacroRef<'top> { pub fn new(address: MacroAddress, kind: &'top MacroKind) -> Self { Self { address, kind } } @@ -52,6 +52,16 @@ impl<'top> MacroKindRef<'top> { pub fn kind(&self) -> &'top MacroKind { self.kind } + + pub fn expect_template(self) -> IonResult> { + if let MacroKind::Template(template) = &self.kind { + return Ok(TemplateMacroRef::new(self.address, template)); + } + IonResult::decoding_error(format!( + "expected a template macro but found {:?}", + self.kind + )) + } } /// Allows callers to resolve a macro ID (that is: name or address) to a [`MacroKind`], confirming @@ -82,29 +92,36 @@ impl MacroTable { } } - pub fn macro_at_address(&self, address: usize) -> Option> { + pub fn macro_with_id(&'_ self, id: MacroIdRef<'_>) -> Option> { + match id { + MacroIdRef::LocalName(name) => self.macro_with_name(name), + MacroIdRef::LocalAddress(address) => self.macro_at_address(address), + } + } + + pub fn macro_at_address(&self, address: usize) -> Option> { let kind = self.macros_by_address.get(address)?; - Some(MacroKindRef { address, kind }) + Some(MacroRef { address, kind }) } pub fn address_for_name(&self, name: &str) -> Option { self.macros_by_name.get(name).copied() } - pub fn macro_with_name(&self, name: &str) -> Option> { + pub fn macro_with_name(&self, name: &str) -> Option> { let address = *self.macros_by_name.get(name)?; let kind = self.macros_by_address.get(address)?; - Some(MacroKindRef { address, kind }) + Some(MacroRef { address, kind }) } pub fn add_macro(&mut self, template: TemplateMacro) -> IonResult { - if self.macros_by_name.contains_key(template.name()) { - return IonResult::decoding_error("macro named '{}' already exists"); + let name = template.name(); + if self.macros_by_name.contains_key(name) { + return IonResult::decoding_error(format!("macro named '{name}' already exists")); } let id = self.macros_by_address.len(); - self.macros_by_name.insert(template.name().to_owned(), id); + self.macros_by_name.insert(name.to_owned(), id); self.macros_by_address.push(MacroKind::Template(template)); - let id = 0; Ok(id) } } diff --git a/src/lazy/expanded/mod.rs b/src/lazy/expanded/mod.rs index d4e85b5f..68671be7 100644 --- a/src/lazy/expanded/mod.rs +++ b/src/lazy/expanded/mod.rs @@ -42,9 +42,9 @@ use sequence::{LazyExpandedList, LazyExpandedSExp}; use crate::element::iterators::SymbolsIterator; use crate::lazy::bytes_ref::BytesRef; -use crate::lazy::decoder::{LazyDecoder, LazyRawReader, LazyRawValue, RawArgumentExpr}; +use crate::lazy::decoder::{LazyDecoder, LazyRawReader, LazyRawValue}; use crate::lazy::encoding::RawValueLiteral; -use crate::lazy::expanded::macro_evaluator::{EExpEvaluator, MacroEvaluator}; +use crate::lazy::expanded::macro_evaluator::{MacroEvaluator, RawMacroInvocation}; use crate::lazy::expanded::macro_table::MacroTable; use crate::lazy::expanded::r#struct::LazyExpandedStruct; use crate::lazy::expanded::sequence::Environment; @@ -154,14 +154,14 @@ impl<'data, D: LazyDecoder<'data>> LazyExpandingReader<'data, D> { // Some(evaluator) // } - fn ptr_to_evaluator<'top>(evaluator_ptr: *mut ()) -> &'top mut EExpEvaluator<'top, 'data, D> { - let ptr: *mut EExpEvaluator<'top, 'data, D> = evaluator_ptr.cast(); - let evaluator: &'top mut EExpEvaluator<'top, 'data, D> = unsafe { &mut *ptr }; + fn ptr_to_evaluator<'top>(evaluator_ptr: *mut ()) -> &'top mut MacroEvaluator<'top, 'data, D> { + let ptr: *mut MacroEvaluator<'top, 'data, D> = evaluator_ptr.cast(); + let evaluator: &'top mut MacroEvaluator<'top, 'data, D> = unsafe { &mut *ptr }; evaluator } - fn evaluator_to_ptr(evaluator: &mut EExpEvaluator<'_, 'data, D>) -> *mut () { - let ptr: *mut EExpEvaluator<'_, 'data, D> = evaluator; + fn evaluator_to_ptr(evaluator: &mut MacroEvaluator<'_, 'data, D>) -> *mut () { + let ptr: *mut MacroEvaluator<'_, 'data, D> = evaluator; let untyped_ptr: *mut () = ptr.cast(); untyped_ptr } @@ -175,13 +175,14 @@ impl<'data, D: LazyDecoder<'data>> LazyExpandingReader<'data, D> { fn get_or_make_evaluator<'top>( maybe_evaluator_ptr: Option<*mut ()>, context: EncodingContext<'top>, - ) -> &'top mut EExpEvaluator<'top, 'data, D> { + ) -> &'top mut MacroEvaluator<'top, 'data, D> { if let Some(evaluator_ptr) = maybe_evaluator_ptr { return Self::ptr_to_evaluator::<'top>(evaluator_ptr); } + // If there isn't an evaluator, we create a new one with an empty environment. context .allocator - .alloc_with(move || EExpEvaluator::new(context)) + .alloc_with(move || MacroEvaluator::new(context, Environment::empty())) } /// Returns the next [`ExpandedStreamItem`] either by continuing to evaluate a macro invocation @@ -194,10 +195,10 @@ impl<'data, D: LazyDecoder<'data>> LazyExpandingReader<'data, D> { 'data: 'top, { loop { - let maybe_evaluator_ptr = (self.evaluator_ptr).clone(); + let maybe_evaluator_ptr = self.evaluator_ptr; if let Some(evaluator_ptr) = maybe_evaluator_ptr { let evaluator = Self::ptr_to_evaluator(evaluator_ptr); - if evaluator.stack_depth() > 0 { + if evaluator.macro_stack_depth() > 0 { // If the evaluator still has macro expansions in its stack, we need to give it the // opportunity to produce the next value. match evaluator.next(context, 0) { @@ -224,10 +225,10 @@ impl<'data, D: LazyDecoder<'data>> LazyExpandingReader<'data, D> { }), // It's another macro invocation, we'll start evaluating it. EExpression(e_exp) => { - let evaluator = - Self::get_or_make_evaluator((self.evaluator_ptr).clone(), context); + let resolved_e_exp = e_exp.resolve(context)?; + let evaluator = Self::get_or_make_evaluator(self.evaluator_ptr, context); // Push the invocation onto the evaluation stack. - evaluator.push(context, e_exp)?; + evaluator.push(context, resolved_e_exp)?; self.evaluator_ptr = Some(Self::evaluator_to_ptr(evaluator)); // Return to the top of the loop to pull the next value (if any) from the evaluator. continue; @@ -236,53 +237,17 @@ impl<'data, D: LazyDecoder<'data>> LazyExpandingReader<'data, D> { }; return Ok(expanded_item); } - - // loop { - // if self.evaluator.stack_depth() > 0 { - // // If the evaluator still has macro expansions in its stack, we need to give it the - // // opportunity to produce the next value. - // match self.evaluator.next(context, 0) { - // Ok(Some(value)) => return Ok(ExpandedStreamItem::Value(value)), - // Ok(None) => { - // // While the evaluator had macros in its stack, they did not produce any more - // // values. The stack is now empty. - // } - // Err(e) => return Err(e), - // }; - // } - // - // // If we reach this point, the evaluator's macro stack is empty. We'll pull another - // // expression from the input stream. - // use RawStreamItem::*; - // let expanded_item = match self.raw_reader.next()? { - // VersionMarker(major, minor) => ExpandedStreamItem::VersionMarker(major, minor), - // // We got our value; return it. - // Value(raw_value) => ExpandedStreamItem::Value(LazyExpandedValue { - // source: ExpandedValueSource::ValueLiteral(raw_value), - // context, - // }), - // // It's another macro invocation, we'll start evaluating it. - // EExpression(e_exp) => { - // // Push the invocation onto the evaluation stack. - // self.evaluator.push(context, e_exp)?; - // // Return to the top of the loop to pull the next value (if any) from the evaluator. - // continue; - // } - // EndOfStream => ExpandedStreamItem::EndOfStream, - // }; - // return Ok(expanded_item); - // } } } /// The source of data backing a [`LazyExpandedValue`]. -#[derive(Debug, Clone)] +#[derive(Clone)] pub enum ExpandedValueSource<'top, 'data, D: LazyDecoder<'data>> { /// This value was a literal in the input stream. ValueLiteral(D::Value), /// This value was part of a template definition. Template(Environment<'top, 'data, D>, TemplateElement<'top>), - /// This value was the computed result of a macro invocation like `(:make_string ...)`. + /// This value was the computed result of a macro invocation like `(:make_string `...)`. Constructed( // TODO: Make this an associated type on the LazyDecoder trait so 1.0 types can set // it to `Never` and the compiler can eliminate this code path where applicable. @@ -292,6 +257,18 @@ pub enum ExpandedValueSource<'top, 'data, D: LazyDecoder<'data>> { ), } +impl<'top, 'data, D: LazyDecoder<'data>> Debug for ExpandedValueSource<'top, 'data, D> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match &self { + ExpandedValueSource::ValueLiteral(v) => write!(f, "{v:?}"), + ExpandedValueSource::Template(_, template_element) => { + write!(f, "{:?}", template_element.value()) + } + ExpandedValueSource::Constructed(_, value) => write!(f, "{value:?}"), + } + } +} + // Converts the raw value literal types associated with each format decoder (e.g. LazyRawTextValue_1_1) // into an ExpandedValueSource. impl<'top, 'data, V: RawValueLiteral, D: LazyDecoder<'data, Value = V>> From @@ -302,14 +279,6 @@ impl<'top, 'data, V: RawValueLiteral, D: LazyDecoder<'data, Value = V>> From } } -// impl<'top, 'data, D: LazyDecoder<'data>> From> -// for ExpandedValueSource<'top, 'data, D> -// { -// fn from(template_element: TemplateElement<'top>) -> Self { -// ExpandedValueSource::Template(template_element) -// } -// } - /// A value produced by expanding the 'raw' view of the input data. #[derive(Clone)] pub struct LazyExpandedValue<'top, 'data, D: LazyDecoder<'data>> { @@ -381,15 +350,6 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedValue<'top, 'data, D> } } - fn environment(&self) -> &[RawArgumentExpr<'data, D>] { - todo!() - // match &self.source { - // ExpandedValueSource::ValueLiteral(_) => Environment::empty(), - // ExpandedValueSource::Template(environment, _) => environment.as_slice(), - // ExpandedValueSource::Constructed(_, _) => empty_environment(), - // } - } - pub fn read(&self) -> IonResult> { use ExpandedValueSource::*; match &self.source { diff --git a/src/lazy/expanded/sequence.rs b/src/lazy/expanded/sequence.rs index a2653db6..99b1938d 100644 --- a/src/lazy/expanded/sequence.rs +++ b/src/lazy/expanded/sequence.rs @@ -1,8 +1,6 @@ use crate::element::iterators::SymbolsIterator; -use crate::lazy::decoder::{ - LazyDecoder, LazyRawSequence, LazyRawValueExpr, RawArgumentExpr, RawValueExpr, -}; -use crate::lazy::expanded::macro_evaluator::{EExpEvaluator, MacroEvaluator, TemplateEvaluator}; +use crate::lazy::decoder::{LazyDecoder, LazyRawSequence, LazyRawValueExpr, RawValueExpr}; +use crate::lazy::expanded::macro_evaluator::{ArgumentExpr, MacroEvaluator, RawMacroInvocation}; use crate::lazy::expanded::template::{ AnnotationsRange, ExprRange, TemplateMacroRef, TemplateSequenceIterator, }; @@ -13,52 +11,29 @@ use crate::lazy::expanded::{ use crate::{IonResult, IonType}; use bumpalo::collections::Vec as BumpVec; - -// // TODO: Move somewhere more central, add doc comment, explain that we could optimize by only -// // copying the top stack frame's arguments -// pub type Environment<'top, 'data, D> = BumpVec<'top, RawArgumentExpr<'data, D>>; -// pub type EnvironmentRef<'top, 'data, D> = &'top Environment<'top, 'data, D>; - #[derive(Copy, Clone, Debug)] pub struct Environment<'top, 'data, D: LazyDecoder<'data>> { - args: &'top [RawArgumentExpr<'data, D>], - parent: Option<&'top Self>, + args: &'top [ArgumentExpr<'top, 'data, D>], } -pub const fn empty_args<'data, D: LazyDecoder<'data>>() -> &'data [RawArgumentExpr<'data, D>] { +pub const fn empty_args<'top, 'data, D: LazyDecoder<'data>>() -> &'top [ArgumentExpr<'top, 'data, D>] +{ &[] } impl<'top, 'data, D: LazyDecoder<'data>> Environment<'top, 'data, D> { - pub(crate) fn new(args: BumpVec<'top, RawArgumentExpr<'data, D>>) -> Self { + pub(crate) fn new(args: BumpVec<'top, ArgumentExpr<'top, 'data, D>>) -> Self { Environment { args: args.into_bump_slice(), - parent: None, } } - pub fn from_args( - context: EncodingContext<'top>, - args: &[RawArgumentExpr<'data, D>], - ) -> Environment<'top, 'data, D> { - let mut arg_stack = BumpVec::new_in(context.allocator); - arg_stack.extend_from_slice(args); - Environment::new(arg_stack) - } - - pub fn empty() -> Environment<'data, 'data, D> { - Environment { - args: empty_args(), - parent: None, - } + pub fn empty() -> Environment<'top, 'data, D> { + Environment { args: empty_args() } } - pub fn args(&self) -> &'top [RawArgumentExpr<'data, D>] { + pub fn args(&self) -> &'top [ArgumentExpr<'top, 'data, D>] { self.args } - - pub fn parent(&self) -> Option<&'top Self> { - self.parent - } } #[derive(Clone)] @@ -125,12 +100,12 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedList<'top, 'data, D> { pub fn iter(&self) -> ExpandedListIterator<'top, 'data, D> { let source = match &self.source { ExpandedListSource::ValueLiteral(list) => { - let evaluator = EExpEvaluator::new(self.context); + let evaluator = MacroEvaluator::new(self.context, Environment::empty()); ExpandedListIteratorSource::ValueLiteral(evaluator, list.iter()) } ExpandedListSource::Template(environment, template, _annotations, steps) => { let steps = template.body.expressions().get(steps.ops_range()).unwrap(); - let evaluator = TemplateEvaluator::new(self.context, *environment); + let evaluator = MacroEvaluator::new(self.context, *environment); ExpandedListIteratorSource::Template(TemplateSequenceIterator::new( self.context, evaluator, @@ -150,7 +125,7 @@ pub enum ExpandedListIteratorSource<'top, 'data, D: LazyDecoder<'data>> { ValueLiteral( // Giving the list iterator its own evaluator means that we can abandon the iterator // at any time without impacting the evaluation state of its parent container. - EExpEvaluator<'top, 'data, D>, + MacroEvaluator<'top, 'data, D>, >::Iterator, ), Template(TemplateSequenceIterator<'top, 'data, D>), @@ -218,12 +193,12 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedSExp<'top, 'data, D> { pub fn iter(&self) -> ExpandedSExpIterator<'top, 'data, D> { let source = match &self.source { ExpandedSExpSource::ValueLiteral(sexp) => { - let evaluator = EExpEvaluator::new(self.context); + let evaluator = MacroEvaluator::new(self.context, Environment::empty()); ExpandedSExpIteratorSource::ValueLiteral(evaluator, sexp.iter()) } ExpandedSExpSource::Template(environment, template, _annotations, steps) => { let steps = template.body.expressions().get(steps.ops_range()).unwrap(); - let evaluator = TemplateEvaluator::new(self.context, *environment); + let evaluator = MacroEvaluator::new(self.context, *environment); ExpandedSExpIteratorSource::Template(TemplateSequenceIterator::new( self.context, evaluator, @@ -262,7 +237,7 @@ pub enum ExpandedSExpIteratorSource<'top, 'data, D: LazyDecoder<'data>> { ValueLiteral( // Giving the sexp iterator its own evaluator means that we can abandon the iterator // at any time without impacting the evaluation state of its parent container. - EExpEvaluator<'top, 'data, D>, + MacroEvaluator<'top, 'data, D>, >::Iterator, ), Template(TemplateSequenceIterator<'top, 'data, D>), @@ -291,12 +266,12 @@ impl<'top, 'data, D: LazyDecoder<'data>> Iterator for ExpandedSExpIterator<'top, /// evaluation already in progress or reading the next item from the input stream. fn expand_next_sequence_value<'top, 'data, D: LazyDecoder<'data>>( context: EncodingContext<'top>, - evaluator: &mut EExpEvaluator<'top, 'data, D>, + evaluator: &mut MacroEvaluator<'top, 'data, D>, iter: &mut impl Iterator>>, ) -> Option>> { loop { // If the evaluator's stack is not empty, it's still expanding a macro. - if evaluator.stack_depth() > 0 { + if evaluator.macro_stack_depth() > 0 { let value = evaluator.next(context, 0).transpose(); if value.is_some() { // The `Some` may contain a value or an error; either way, that's the next return value. @@ -315,7 +290,11 @@ fn expand_next_sequence_value<'top, 'data, D: LazyDecoder<'data>>( })) } Some(Ok(RawValueExpr::MacroInvocation(invocation))) => { - let begin_expansion_result = evaluator.push(context, invocation); + let resolved_invocation = match invocation.resolve(context) { + Ok(resolved) => resolved, + Err(e) => return Some(Err(e)), + }; + let begin_expansion_result = evaluator.push(context, resolved_invocation); if let Err(e) = begin_expansion_result { return Some(Err(e)); } diff --git a/src/lazy/expanded/struct.rs b/src/lazy/expanded/struct.rs index 6944a870..185a0a2f 100644 --- a/src/lazy/expanded/struct.rs +++ b/src/lazy/expanded/struct.rs @@ -1,8 +1,10 @@ use std::ops::ControlFlow; use crate::element::iterators::SymbolsIterator; -use crate::lazy::decoder::{LazyDecoder, LazyRawStruct, RawFieldExpr, RawValueExpr}; -use crate::lazy::expanded::macro_evaluator::{EExpEvaluator, MacroEvaluator, TemplateEvaluator}; +use crate::lazy::decoder::{ + LazyDecoder, LazyRawFieldExpr, LazyRawStruct, RawFieldExpr, RawValueExpr, +}; +use crate::lazy::expanded::macro_evaluator::{MacroEvaluator, MacroExpr, RawMacroInvocation}; use crate::lazy::expanded::sequence::Environment; use crate::lazy::expanded::template::{ AnnotationsRange, ExprRange, TemplateMacroRef, TemplateStructRawFieldsIterator, @@ -96,12 +98,12 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedStruct<'top, 'data, D let source = match &self.source { ExpandedStructSource::ValueLiteral(raw_struct) => { ExpandedStructIteratorSource::ValueLiteral( - EExpEvaluator::new(self.context), + MacroEvaluator::new(self.context, Environment::empty()), raw_struct.iter(), ) } ExpandedStructSource::Template(environment, template, _annotations, expressions) => { - let evaluator = TemplateEvaluator::new(self.context, *environment); + let evaluator = MacroEvaluator::new(self.context, *environment); ExpandedStructIteratorSource::Template( evaluator, TemplateStructRawFieldsIterator::new( @@ -160,13 +162,13 @@ pub enum ExpandedStructIteratorSource<'top, 'data, D: LazyDecoder<'data>> { ValueLiteral( // Giving the struct iterator its own evaluator means that we can abandon the iterator // at any time without impacting the evaluation state of its parent container. - EExpEvaluator<'top, 'data, D>, + MacroEvaluator<'top, 'data, D>, >::Iterator, ), // The struct we're iterating over is a value in a TDL template. It may contain macro // invocations that need to be evaluated. Template( - TemplateEvaluator<'top, 'data, D>, + MacroEvaluator<'top, 'data, D>, TemplateStructRawFieldsIterator<'top, 'data, D>, ), // TODO: Constructed @@ -221,8 +223,26 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> Iterator for ExpandedStructIterat ExpandedStructIteratorSource::Template(tdl_macro_evaluator, template_iterator) => { Self::next_field_from(context, state, tdl_macro_evaluator, template_iterator) } - ExpandedStructIteratorSource::ValueLiteral(e_exp_evaluator, iter) => { - Self::next_field_from(context, state, e_exp_evaluator, iter) + ExpandedStructIteratorSource::ValueLiteral(e_exp_evaluator, raw_struct_iter) => { + let mut iter_adapter = raw_struct_iter.map( + |field: IonResult>| match field? { + RawFieldExpr::NameValuePair(name, RawValueExpr::MacroInvocation(m)) => { + let resolved_invocation = m.resolve(context)?; + Ok(RawFieldExpr::NameValuePair( + name, + RawValueExpr::MacroInvocation(resolved_invocation.into()), + )) + } + RawFieldExpr::NameValuePair(name, RawValueExpr::ValueLiteral(value)) => Ok( + RawFieldExpr::NameValuePair(name, RawValueExpr::ValueLiteral(value)), + ), + RawFieldExpr::MacroInvocation(invocation) => { + let resolved_invocation = invocation.resolve(context)?; + Ok(RawFieldExpr::MacroInvocation(resolved_invocation.into())) + } + }, + ); + Self::next_field_from(context, state, e_exp_evaluator, &mut iter_adapter) } } } @@ -245,18 +265,17 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> ExpandedStructIterator<'top, 'dat // The lifetime of the field name that we return; it needs to live at least as long as // `top -- the amount of time that the reader will be parked on this top level value. 'name: 'top, - E: MacroEvaluator<'top, 'data, D> + 'top, // We have an iterator (see `I` below) that gives us raw fields from an input struct. // This type, `V`, is the type of value in that raw field. For example: `LazyRawTextValue_1_1` // when reading text Ion 1.1, or `&'top Element` when evaluating a TDL macro. V: Into>, // An iterator over the struct we're expanding. It may be the fields iterator from a // LazyRawStruct, or it could be a `TemplateStructRawFieldsIterator`. - I: Iterator>>, + I: Iterator>>>, >( context: EncodingContext<'top>, state: &'a mut ExpandedStructIteratorState<'top, 'data, D>, - evaluator: &'a mut E, + evaluator: &'a mut MacroEvaluator<'top, 'data, D>, iter: &'a mut I, ) -> Option>> { // This method begins by pulling raw field expressions from the source iterator. @@ -324,13 +343,12 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> ExpandedStructIterator<'top, 'dat // These generics are all carried over from the function above. 'a, 'name: 'top, - E: MacroEvaluator<'top, 'data, D> + 'top, V: Into>, - I: Iterator>>, + I: Iterator>>>, >( context: EncodingContext<'top>, state: &mut ExpandedStructIteratorState<'top, 'data, D>, - evaluator: &mut E, + evaluator: &mut MacroEvaluator<'top, 'data, D>, iter: &mut I, ) -> ControlFlow>>> { // Because this helper function is always being invoked from within a loop, it uses @@ -387,15 +405,11 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> ExpandedStructIterator<'top, 'dat /// Pulls the next value from the evaluator, confirms that it's a struct, and then switches /// the iterator state to `InliningAStruct` so it can begin merging its fields. - fn begin_inlining_struct_from_macro< - 'a, - 'name: 'top, - E: MacroEvaluator<'top, 'data, D> + 'top, - >( + fn begin_inlining_struct_from_macro<'a, 'name: 'top>( context: EncodingContext<'top>, state: &mut ExpandedStructIteratorState<'top, 'data, D>, - evaluator: &mut E, - invocation: E::MacroInvocation, + evaluator: &mut MacroEvaluator<'top, 'data, D>, + invocation: MacroExpr<'top, 'data, D>, ) -> IonResult<()> { let mut evaluation = evaluator.evaluate(context, invocation)?; let expanded_value = match evaluation.next() { diff --git a/src/lazy/expanded/template.rs b/src/lazy/expanded/template.rs index 00a79a4d..ab35565b 100644 --- a/src/lazy/expanded/template.rs +++ b/src/lazy/expanded/template.rs @@ -1,11 +1,13 @@ +use std::fmt; +use std::fmt::{Debug, Formatter}; use std::marker::PhantomData; use std::ops::{Deref, Range}; -use crate::lazy::decoder::{LazyDecoder, RawArgumentExpr, RawFieldExpr, RawValueExpr}; +use crate::lazy::decoder::{LazyDecoder, RawFieldExpr, RawValueExpr}; use crate::lazy::expanded::macro_evaluator::{ - ArgumentKind, MacroEvaluator, MacroInvocation, TemplateEvaluator, ToArgumentKind, + ArgumentExpr, MacroEvaluator, MacroExpr, MacroInvocation, }; -use crate::lazy::expanded::macro_table::MacroKind; +use crate::lazy::expanded::macro_table::MacroRef; use crate::lazy::expanded::sequence::Environment; use crate::lazy::expanded::{ EncodingContext, ExpandedValueRef, ExpandedValueSource, LazyExpandedValue, @@ -59,35 +61,7 @@ impl MacroSignature { } } -/// A pairing of a template reference and the address in the macro table at which it was found. -/// This type implements Deref to allow easy access to the methods on the template. -#[derive(Debug, Clone, Copy)] -pub struct TemplateMacroRef<'top> { - template_address: MacroAddress, - template: &'top TemplateMacro, -} - -impl<'top> TemplateMacroRef<'top> { - pub fn new(template_address: MacroAddress, template: &'top TemplateMacro) -> Self { - Self { - template_address, - template, - } - } - pub fn address(&self) -> MacroAddress { - self.template_address - } -} - -impl<'top> Deref for TemplateMacroRef<'top> { - type Target = &'top TemplateMacro; - - fn deref(&self) -> &Self::Target { - &self.template - } -} - -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct TemplateMacro { // TODO: Make the name optional pub(crate) name: Symbol, @@ -95,6 +69,12 @@ pub struct TemplateMacro { pub(crate) body: TemplateBody, } +impl Debug for TemplateMacro { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.fmt_template(f) + } +} + impl TemplateMacro { pub fn name(&self) -> &str { self.name.text().expect("template names cannot be $0") @@ -105,12 +85,50 @@ impl TemplateMacro { pub fn body(&self) -> &TemplateBody { &self.body } + + pub(crate) fn fmt_template(&self, f: &mut Formatter<'_>) -> fmt::Result { + writeln!(f, "Template {}", self.name())?; + for param in self.signature().parameters() { + writeln!(f, " {param:?}")?; + } + writeln!(f, " body:")?; + let indentation = &mut String::from(" "); + let mut index = 0usize; + while let Some(expr) = self.body().expressions().get(index) { + index += TemplateBodyValueExpr::fmt_expr(f, indentation, expr, self)?; + } + + Ok(()) + } +} + +#[derive(Copy, Clone, Debug)] +pub struct TemplateMacroRef<'top> { + address: MacroAddress, + template: &'top TemplateMacro, +} + +impl<'top> TemplateMacroRef<'top> { + pub fn new(address: MacroAddress, template: &'top TemplateMacro) -> Self { + Self { address, template } + } + pub fn address(&self) -> MacroAddress { + self.address + } +} + +impl<'top> Deref for TemplateMacroRef<'top> { + type Target = &'top TemplateMacro; + + fn deref(&self) -> &Self::Target { + &self.template + } } pub struct TemplateSequenceIterator<'top, 'data, D: LazyDecoder<'data>> { context: EncodingContext<'top>, template: TemplateMacroRef<'top>, - evaluator: TemplateEvaluator<'top, 'data, D>, + evaluator: MacroEvaluator<'top, 'data, D>, value_expressions: &'top [TemplateBodyValueExpr], index: usize, } @@ -118,7 +136,7 @@ pub struct TemplateSequenceIterator<'top, 'data, D: LazyDecoder<'data>> { impl<'top, 'data, D: LazyDecoder<'data>> TemplateSequenceIterator<'top, 'data, D> { pub fn new( context: EncodingContext<'top>, - evaluator: TemplateEvaluator<'top, 'data, D>, + evaluator: MacroEvaluator<'top, 'data, D>, template: TemplateMacroRef<'top>, value_expressions: &'top [TemplateBodyValueExpr], ) -> Self { @@ -138,37 +156,50 @@ impl<'top, 'data, D: LazyDecoder<'data>> Iterator for TemplateSequenceIterator<' fn next(&mut self) -> Option { loop { // If the evaluator's stack is not empty, give it the opportunity to yield a value. - if self.evaluator.stack_depth() > 0 { + if self.evaluator.macro_stack_depth() > 0 { match self.evaluator.next(self.context, 0).transpose() { Some(value) => return Some(value), None => { // The stack did not produce values and is empty, pull - // the next expression from `self.sequence` + // the next expression from `self.value_expressions` } } } - let step_address = self.index; // We didn't get a value from the evaluator, so pull the next expansion step. - let step = self.value_expressions.get(step_address)?; + let step = self.value_expressions.get(self.index)?; self.index += 1; return match step { - TemplateBodyValueExpr::Element(element) => Some(Ok(LazyExpandedValue { - context: self.context, - source: ExpandedValueSource::Template( - self.evaluator.environment(), - TemplateElement::new(self.template, element), - ), - })), + TemplateBodyValueExpr::Element(element) => { + let value = LazyExpandedValue { + context: self.context, + source: ExpandedValueSource::Template( + self.evaluator.environment(), + TemplateElement::new(self.template, element), + ), + }; + println!("yield {value:?} from list"); + Some(Ok(value)) + } TemplateBodyValueExpr::MacroInvocation(body_invocation) => { // ...it's a TDL macro invocation. Push it onto the evaluator's stack and return // to the top of the loop. - let invocation_address = TemplateMacroInvocationAddress::new( - self.template.address(), - body_invocation.macro_address, - step_address, - body_invocation.arg_expr_range.len(), + let invoked_macro = self + .context + .macro_table + .macro_at_address(body_invocation.invoked_macro_address) + .unwrap(); + let invocation = TemplateMacroInvocation::new( + self.context, + self.template, + invoked_macro, + self.template + .body + .expressions() + .get(body_invocation.arg_expr_range().ops_range()) + .unwrap(), ); - match self.evaluator.push(self.context, invocation_address) { + self.index += invocation.arg_expressions.len(); + match self.evaluator.push(self.context, invocation) { Ok(_) => continue, Err(e) => return Some(Err(e)), }; @@ -215,7 +246,7 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> Iterator for TemplateStructRawFieldsIterator<'top, 'data, D> { type Item = IonResult< - RawFieldExpr<'top, ExpandedValueSource<'top, 'data, D>, TemplateMacroInvocationAddress>, + RawFieldExpr<'top, ExpandedValueSource<'top, 'data, D>, MacroExpr<'top, 'data, D>>, >; fn next(&mut self) -> Option { @@ -250,18 +281,35 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> Iterator )) } Some(TemplateBodyValueExpr::Element(element)) => { + match element.value() { + TemplateValue::List(range) + | TemplateValue::SExp(range) + | TemplateValue::Struct(range) => self.index += range.len(), + _ => {} + }; RawValueExpr::ValueLiteral(ExpandedValueSource::Template( self.environment, TemplateElement::new(self.template, element), )) } - Some(TemplateBodyValueExpr::MacroInvocation(invocation)) => { - RawValueExpr::MacroInvocation(TemplateMacroInvocationAddress::new( - self.template.address(), - invocation.macro_address(), - value_expr_address, - invocation.arg_expr_range().len(), - )) + Some(TemplateBodyValueExpr::MacroInvocation(body_invocation)) => { + let invoked_macro = self + .context + .macro_table + .macro_at_address(body_invocation.invoked_macro_address) + .unwrap(); + let invocation = TemplateMacroInvocation::new( + self.context, + self.template, + invoked_macro, + self.template + .body + .expressions() + .get(body_invocation.arg_expr_range().ops_range()) + .unwrap(), + ); + self.index += invocation.arg_expressions.len(); + RawValueExpr::MacroInvocation(MacroExpr::TemplateMacro(invocation)) } Some(TemplateBodyValueExpr::Variable(_)) => { todo!("variable expansion in template structs") @@ -300,10 +348,15 @@ impl TemplateBody { )) } - pub fn push_macro_invocation(&mut self, address: usize, expr_range: ExprRange) { + pub fn push_macro_invocation(&mut self, invoked_macro_address: usize, expr_range: ExprRange) { + let expression_address = self.expressions.len(); self.expressions .push(TemplateBodyValueExpr::MacroInvocation( - TemplateBodyMacroInvocation::new(address, expr_range), + TemplateBodyMacroInvocation::new( + expression_address, + invoked_macro_address, + expr_range, + ), )) } } @@ -340,6 +393,56 @@ pub enum TemplateBodyValueExpr { MacroInvocation(TemplateBodyMacroInvocation), } +impl TemplateBodyValueExpr { + pub(crate) fn fmt_expr( + f: &mut Formatter<'_>, + indentation: &mut String, + expr: &TemplateBodyValueExpr, + host_template: &TemplateMacro, + ) -> Result { + match &expr { + TemplateBodyValueExpr::Element(e) => { + // TODO: Annotations + writeln!(f, "{}{:?}", indentation, e.value)?; + Ok(1) + } + TemplateBodyValueExpr::Variable(v) => { + writeln!(f, "{}{:?}", indentation, v)?; + Ok(1) + } + TemplateBodyValueExpr::MacroInvocation(m) => { + Self::fmt_invocation(f, indentation, host_template, m) + } + } + } + + pub(crate) fn fmt_invocation( + f: &mut Formatter<'_>, + indentation: &mut String, + host_template: &TemplateMacro, + invocation: &TemplateBodyMacroInvocation, + ) -> Result { + writeln!( + f, + "{}invoke address {}", + indentation, invocation.invoked_macro_address + )?; + let args = host_template + .body + .expressions + .get(invocation.arg_expr_range.ops_range()) + .unwrap(); + + indentation.push_str(" "); + let mut expr_index: usize = 0; + while let Some(arg) = args.get(expr_index) { + expr_index += Self::fmt_expr(f, indentation, arg, host_template)?; + } + indentation.truncate(indentation.len() - 4); + Ok(1 + args.len()) + } +} + #[derive(Debug, Clone, Copy)] pub struct TemplateMacroArgExpr { // The address of the template macro in which this argument expression appears @@ -359,25 +462,64 @@ impl TemplateMacroArgExpr { } } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Copy, Clone)] +pub struct RawTemplateMacroInvocationArg { + host_template_address: MacroAddress, + template_body_expr_address: TemplateBodyExprAddress, +} + +#[derive(Debug, Copy, Clone, PartialEq)] pub struct TemplateBodyMacroInvocation { - macro_address: MacroAddress, + invocation_expr_address: TemplateBodyExprAddress, + invoked_macro_address: MacroAddress, arg_expr_range: ExprRange, } impl TemplateBodyMacroInvocation { - pub fn new(macro_address: MacroAddress, arg_expr_range: ExprRange) -> Self { + pub fn new( + invocation_expr_address: TemplateBodyExprAddress, + invoked_macro_address: MacroAddress, + arg_expr_range: ExprRange, + ) -> Self { Self { - macro_address, + invocation_expr_address, + invoked_macro_address, arg_expr_range, } } pub fn macro_address(&self) -> MacroAddress { - self.macro_address + self.invoked_macro_address } pub fn arg_expr_range(&self) -> ExprRange { self.arg_expr_range } + pub fn invocation_expr_address(&self) -> TemplateBodyExprAddress { + self.invocation_expr_address + } + + pub fn resolve<'top>( + self, + host_template: TemplateMacroRef<'top>, + context: EncodingContext<'top>, + ) -> IonResult> { + let invoked_macro = context + .macro_table + .macro_at_address(self.invoked_macro_address) + .unwrap(); + + let arg_expressions = host_template + .body + .expressions() + .get(self.arg_expr_range.ops_range()) + .unwrap(); + + Ok(TemplateMacroInvocation { + context, + host_template, + invoked_macro, + arg_expressions, + }) + } } impl TemplateBodyValueExpr { @@ -402,154 +544,50 @@ impl TemplateBodyValueExpr { } } -impl<'data, D: LazyDecoder<'data>> ToArgumentKind<'data, D, TemplateMacroInvocationAddress> - for TemplateMacroArgExpr -{ - fn to_arg_expr<'top>( - self, - context: EncodingContext<'top>, - environment: Environment<'top, 'data, D>, - ) -> ArgumentKind<'top, 'data, D, TemplateMacroInvocationAddress> - where - TemplateBodyValueExpr: 'top, - Self: 'top, - { - let Some(macro_ref) = &context - .macro_table - .macro_at_address(self.host_template_address) - else { - unreachable!( - "template address {} was not valid", - self.host_template_address - ); - }; - - let MacroKind::Template(template) = macro_ref.kind() else { - unreachable!( - "macro at address {} was not a template: {:?}", - self.host_template_address, - macro_ref.kind() - ); - }; - - let template_ref = TemplateMacroRef::new(macro_ref.address(), template); - - let invocation_expr = template - .body() - .expressions() - .get(self.value_expr_address) - .unwrap(); - match invocation_expr { - TemplateBodyValueExpr::Element(element) => { - let template_element = TemplateElement::new(template_ref, element); - ArgumentKind::ValueLiteral(LazyExpandedValue::from_template( - context, - environment, - template_element, - )) - } - TemplateBodyValueExpr::Variable(_variable) => { - todo!("variable expansion") - } - TemplateBodyValueExpr::MacroInvocation(invocation) => { - ArgumentKind::MacroInvocation(TemplateMacroInvocationAddress { - host_template_address: template_ref.address(), - invoked_macro_address: invocation.macro_address, - invocation_expression_address: self.value_expr_address, - num_arguments: invocation.arg_expr_range.len(), - }) - } - } - } -} - -#[derive(Copy, Clone, Debug)] -pub struct TemplateMacroInvocationAddress { - // The template where the invocation appeared - host_template_address: MacroAddress, - // The address of the macro being invoked - invoked_macro_address: MacroAddress, - // The address (Vec index) of the TemplateValueExpr::MacroInvocation, which enables an - // arguments iterator to visit each argument expression in the template. - invocation_expression_address: TemplateBodyExprAddress, - num_arguments: usize, -} - -impl TemplateMacroInvocationAddress { - pub fn new( - host_template_address: MacroAddress, - invoked_macro_address: MacroAddress, - invocation_expression_address: TemplateBodyExprAddress, - num_arguments: usize, - ) -> Self { - Self { - host_template_address, - invoked_macro_address, - invocation_expression_address, - num_arguments, - } - } - pub fn host_template_address(&self) -> MacroAddress { - self.host_template_address - } - pub fn invocation_expression_address(&self) -> usize { - self.invocation_expression_address - } - - pub fn arguments(&self) -> TemplateMacroInvocationArgsIterator { - TemplateMacroInvocationArgsIterator::new(*self) - } -} - /// A macro invocation in a template body. -#[derive(Debug, Copy, Clone)] +#[derive(Copy, Clone)] pub struct TemplateMacroInvocation<'top> { - // The address of the template in which this macro invocation appears - host_template_address: MacroAddress, + context: EncodingContext<'top>, // The definition of the template in which this macro invocation appears - host_template: &'top TemplateMacro, - // The address of the the macro invocation expression within the host template's body - invocation_expression_address: TemplateBodyExprAddress, + host_template: TemplateMacroRef<'top>, // The address of the macro being invoked - invoked_macro_address: MacroAddress, + invoked_macro: MacroRef<'top>, // The range of value expressions in the host template's body that are arguments to the // macro being invoked arg_expressions: &'top [TemplateBodyValueExpr], } +impl<'top> Debug for TemplateMacroInvocation<'top> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "TemplateMacroInvocation ", + self.invoked_macro.address() + ) + } +} + impl<'top> TemplateMacroInvocation<'top> { pub fn new( - host_template_address: MacroAddress, - host_template: &'top TemplateMacro, - invocation_expression_address: TemplateBodyExprAddress, - invoked_macro_address: MacroAddress, + context: EncodingContext<'top>, + host_template: TemplateMacroRef<'top>, + invoked_macro: MacroRef<'top>, arg_expressions: &'top [TemplateBodyValueExpr], ) -> Self { Self { - host_template_address, + context, host_template, - invocation_expression_address, - invoked_macro_address, + invoked_macro, arg_expressions, } } - pub fn invocation_address(&self) -> TemplateMacroInvocationAddress { - TemplateMacroInvocationAddress { - host_template_address: self.host_template_address, - invoked_macro_address: self.invoked_macro_address, - invocation_expression_address: 0, - num_arguments: self.arg_expressions.len(), - } - } - - pub fn macro_address(&self) -> MacroAddress { - self.invoked_macro_address - } - pub fn args(&self) -> TemplateMacroInvocationArgsIterator { - TemplateMacroInvocationArgsIterator::new(self.invocation_address()) + pub fn arguments<'data, D: LazyDecoder<'data>>( + &self, + environment: Environment<'top, 'data, D>, + ) -> TemplateMacroInvocationArgsIterator<'top, 'data, D> { + TemplateMacroInvocationArgsIterator::new(environment, *self) } - - pub fn template(&self) -> &'top TemplateMacro { + pub fn template(&self) -> TemplateMacroRef<'top> { self.host_template } pub fn arg_expressions(&self) -> &'top [TemplateBodyValueExpr] { @@ -557,79 +595,93 @@ impl<'top> TemplateMacroInvocation<'top> { } } -// TODO: note about why this can't hold a 'top lifetime -pub struct TemplateMacroInvocationArgsIterator { - invocation_address: TemplateMacroInvocationAddress, +impl<'top, 'data: 'top, D: LazyDecoder<'data>> From> + for MacroExpr<'top, 'data, D> +{ + fn from(value: TemplateMacroInvocation<'top>) -> Self { + MacroExpr::TemplateMacro(value) + } +} + +// TODO: This should hold an Environment<'top, 'data, D> instead of holding a PhantomData +pub struct TemplateMacroInvocationArgsIterator<'top, 'data, D: LazyDecoder<'data>> { + environment: Environment<'top, 'data, D>, + invocation: TemplateMacroInvocation<'top>, arg_index: usize, } -impl TemplateMacroInvocationArgsIterator { - pub fn new(invocation_address: TemplateMacroInvocationAddress) -> Self { +impl<'top, 'data, D: LazyDecoder<'data>> TemplateMacroInvocationArgsIterator<'top, 'data, D> { + pub fn new( + environment: Environment<'top, 'data, D>, + invocation: TemplateMacroInvocation<'top>, + ) -> Self { Self { - invocation_address, + environment, + invocation, arg_index: 0, } } } -impl Iterator for TemplateMacroInvocationArgsIterator { - type Item = IonResult; +impl<'top, 'data, D: LazyDecoder<'data>> Iterator + for TemplateMacroInvocationArgsIterator<'top, 'data, D> +{ + type Item = IonResult>; fn next(&mut self) -> Option { - if self.arg_index >= self.invocation_address.num_arguments { - return None; - } - let arg_expr = TemplateMacroArgExpr::new( - self.invocation_address.host_template_address(), - self.arg_index, - ); + let arg = self.invocation.arg_expressions().get(self.arg_index)?; self.arg_index += 1; + let arg_expr = match arg { + TemplateBodyValueExpr::Element(e) => { + // If it's a container, skip over its contents when this iterator resumes + match e.value() { + TemplateValue::List(range) + | TemplateValue::SExp(range) + | TemplateValue::Struct(range) => { + self.arg_index += range.len(); + } + _ => {} + }; + ArgumentExpr::ValueLiteral(LazyExpandedValue::from_template( + self.invocation.context, + self.environment, + TemplateElement::new(self.invocation.template(), e), + )) + } + TemplateBodyValueExpr::Variable(_v) => { + todo!("variable expansion") + } + TemplateBodyValueExpr::MacroInvocation(body_invocation) => { + let invocation = match body_invocation + .resolve(self.invocation.template(), self.invocation.context) + { + Ok(invocation) => { + // Skip over all of the expressions that belong to this invocation. + self.arg_index += invocation.arg_expressions.len(); + invocation + } + Err(e) => return Some(Err(e)), + }; + ArgumentExpr::MacroInvocation(invocation.into()) + } + }; + Some(Ok(arg_expr)) } } -// TODO: Explain why this is on the address and not the resolved invocation -impl<'data, D: LazyDecoder<'data>> MacroInvocation<'data, D> for TemplateMacroInvocationAddress { - type ArgumentExpr = TemplateMacroArgExpr; - type ArgumentsIterator = TemplateMacroInvocationArgsIterator; - type RawArgumentsIterator = TemplateMacroInvocationRawArgsIterator<'data, D>; - type TransientEvaluator<'context> = TemplateEvaluator<'context, 'data, D> where Self: 'context, 'data: 'context; - - fn id(&self) -> MacroIdRef { - MacroIdRef::LocalAddress(self.invoked_macro_address) - } - - fn arguments(&self) -> Self::ArgumentsIterator { - self.arguments() - } - - fn raw_arguments(&self) -> Self::RawArgumentsIterator { - todo!() - } +impl<'top, 'data: 'top, D: LazyDecoder<'data>> MacroInvocation<'top, 'data, D> + for TemplateMacroInvocation<'top> +{ + type ArgumentsIterator = TemplateMacroInvocationArgsIterator<'top, 'data, D>; - fn make_transient_evaluator<'context>( - context: EncodingContext<'context>, - environment: Environment<'context, 'data, D>, - ) -> Self::TransientEvaluator<'context> - where - 'data: 'context, - Self: 'context, - { - Self::TransientEvaluator::new(context, environment) + fn id(&self) -> MacroIdRef<'data> { + MacroIdRef::LocalAddress(self.invoked_macro.address()) } -} -pub struct TemplateMacroInvocationRawArgsIterator<'data, D: LazyDecoder<'data>> { - index: usize, - args: TemplateMacroInvocationArgsIterator, - spooky: PhantomData<&'data D>, -} - -impl<'data, D: LazyDecoder<'data>> Iterator for TemplateMacroInvocationRawArgsIterator<'data, D> { - type Item = RawArgumentExpr<'data, D>; - - fn next(&mut self) -> Option { - todo!() + fn arguments(&self, environment: Environment<'top, 'data, D>) -> Self::ArgumentsIterator { + // Delegate to the inherent impl on TemplateMacroInvocation + self.arguments(environment) } } diff --git a/src/lazy/never.rs b/src/lazy/never.rs index 912f6f22..bbe9a79d 100644 --- a/src/lazy/never.rs +++ b/src/lazy/never.rs @@ -1,11 +1,10 @@ use std::fmt::Debug; -use crate::lazy::decoder::{LazyDecoder, RawArgumentExpr}; +use crate::lazy::decoder::{LazyDecoder, LazyRawValueExpr}; use crate::lazy::expanded::macro_evaluator::{ - ArgumentKind, MacroEvaluator, MacroExpansion, MacroInvocation, MacroStack, ToArgumentKind, + ArgumentExpr, MacroExpr, MacroInvocation, RawMacroInvocation, }; use crate::lazy::expanded::sequence::Environment; -use crate::lazy::expanded::stack::{BumpVecStack, Stack}; use crate::lazy::expanded::EncodingContext; use crate::lazy::text::raw::v1_1::reader::MacroIdRef; use crate::IonResult; @@ -18,77 +17,44 @@ pub enum Never { // Ion 1.0 uses `Never` as a placeholder type for MacroInvocation. // The compiler should optimize these methods away. -impl<'data, D: LazyDecoder<'data>> MacroInvocation<'data, D> for Never { - type ArgumentExpr = Never; +impl<'data, D: LazyDecoder<'data>> RawMacroInvocation<'data, D> for Never { // These use Box to avoid defining yet another placeholder type. - type ArgumentsIterator = Box>>; - type RawArgumentsIterator = Box>>; + type RawArgumentsIterator<'a> = Box>>>; + type MacroInvocation<'top> = Never where 'data: 'top; - type TransientEvaluator<'context> = Never where Self: 'context, 'data: 'context; - - fn id(&self) -> MacroIdRef<'_> { + fn id(&self) -> MacroIdRef<'data> { unreachable!("macro in Ion 1.0 (method: id)") } - fn arguments(&self) -> Self::ArgumentsIterator { + fn raw_arguments(&self) -> Self::RawArgumentsIterator<'_> { unreachable!("macro in Ion 1.0 (method: arguments)") } - fn raw_arguments(&self) -> Self::RawArgumentsIterator { - unreachable!("macro in Ion 1.0 (method: raw_arguments)") - } - - fn make_transient_evaluator<'context>( - _context: EncodingContext<'context>, - _environment: Environment<'context, 'data, D>, - ) -> Self::TransientEvaluator<'context> - where - 'data: 'context, - Self: 'context, - { - unreachable!("macro in Ion 1.0 (method: make_transient_evaluator)") - } -} - -impl<'data, D: LazyDecoder<'data>> ToArgumentKind<'data, D, Self> for Never { - fn to_arg_expr<'top>( + fn resolve<'top>( self, _context: EncodingContext<'top>, - _environment: Environment<'top, 'data, D>, - ) -> ArgumentKind<'top, 'data, D, Never> + ) -> IonResult> where - Never: 'top, - Self: 'top, + 'data: 'top, { - unreachable!("macro in Ion 1.0 (method: to_arg_expr)") + unreachable!("macro in Ion 1.0 (method: resolve)") } } -impl<'top, 'data: 'top, D: LazyDecoder<'data>> MacroEvaluator<'top, 'data, D> for Never { - type MacroInvocation = Never; - fn macro_stack(&self) -> &MacroStack<'top, 'data, D, Never> { - unreachable!("macro in Ion 1.0 (method: macro_stack)") +impl<'top, 'data: 'top, D: LazyDecoder<'data>> From for MacroExpr<'top, 'data, D> { + fn from(_value: Never) -> Self { + unreachable!("macro in Ion 1.0 (method: into)") } +} - fn macro_stack_mut(&mut self) -> &mut MacroStack<'top, 'data, D, Never> { - unreachable!("macro in Ion 1.0 (method: macro_stack_mut)") - } +impl<'top, 'data: 'top, D: LazyDecoder<'data>> MacroInvocation<'top, 'data, D> for Never { + type ArgumentsIterator = Box>>>; - fn environment(&self) -> Environment<'top, 'data, D> { - unreachable!("macro in Ion 1.0 (method: environment)") + fn id(&self) -> MacroIdRef<'data> { + unreachable!("macro in Ion 1.0 (method: id)"); } - fn environment_mut(&mut self) -> &mut Environment<'top, 'data, D> { - unreachable!("macro in Ion 1.0 (method: environment_mut)") + fn arguments(&self, _environment: Environment<'top, 'data, D>) -> Self::ArgumentsIterator { + unreachable!("macro in Ion 1.0 (method: arguments)"); } - - // fn arg_stack(&self) -> &::Storage<'top, RawArgumentExpr<'data, D>> { - // unreachable!("macro in Ion 1.0 (method: arg_stack)") - // } - // - // fn arg_stack_mut( - // &mut self, - // ) -> &mut ::Storage<'top, RawArgumentExpr<'data, D>> { - // unreachable!("macro in Ion 1.0 (method: arg_stack_mut)") - // } } diff --git a/src/lazy/raw_stream_item.rs b/src/lazy/raw_stream_item.rs index 13f10138..ae26b5fb 100644 --- a/src/lazy/raw_stream_item.rs +++ b/src/lazy/raw_stream_item.rs @@ -14,7 +14,7 @@ pub enum RawStreamItem<'data, D: LazyDecoder<'data>> { /// for [`LazyRawBinaryValue`](crate::lazy::binary::raw::value::LazyRawBinaryValue). Value(D::Value), /// An Ion 1.1+ macro invocation. Ion 1.0 readers will never return a macro invocation. - EExpression(D::MacroInvocation), + EExpression(D::RawMacroInvocation), /// The end of the stream EndOfStream, } @@ -58,7 +58,7 @@ impl<'data, D: LazyDecoder<'data>> RawStreamItem<'data, D> { } impl<'data, D: LazyDecoder<'data> + EncodingWithMacroSupport> RawStreamItem<'data, D> { - pub fn as_macro_invocation(&self) -> Option<&D::MacroInvocation> { + pub fn as_macro_invocation(&self) -> Option<&D::RawMacroInvocation> { if let Self::EExpression(m) = self { Some(m) } else { @@ -66,7 +66,7 @@ impl<'data, D: LazyDecoder<'data> + EncodingWithMacroSupport> RawStreamItem<'dat } } - pub fn expect_macro_invocation(self) -> IonResult { + pub fn expect_macro_invocation(self) -> IonResult { if let Self::EExpression(m) = self { Ok(m) } else { diff --git a/src/lazy/reader.rs b/src/lazy/reader.rs index 74b9c7be..3b70ed06 100644 --- a/src/lazy/reader.rs +++ b/src/lazy/reader.rs @@ -6,9 +6,12 @@ use crate::element::Element; use crate::lazy::any_encoding::AnyEncoding; use crate::lazy::decoder::LazyDecoder; use crate::lazy::encoding::{BinaryEncoding_1_0, TextEncoding_1_0, TextEncoding_1_1}; +use crate::lazy::expanded::template::TemplateMacro; +use crate::lazy::expanded::EncodingContext; use crate::lazy::system_reader::{ LazySystemAnyReader, LazySystemBinaryReader, LazySystemReader, LazySystemTextReader_1_1, }; +use crate::lazy::text::raw::v1_1::reader::MacroAddress; use crate::lazy::value::LazyValue; use crate::result::IonFailure; use crate::{IonError, IonResult}; @@ -116,6 +119,14 @@ impl<'data> LazyTextReader_1_1<'data> { let system_reader = LazySystemTextReader_1_1::new(ion_data); Ok(LazyApplicationReader { system_reader }) } + + pub(crate) fn register_template(&mut self, template: TemplateMacro) -> IonResult { + self.system_reader.macro_table_mut().add_macro(template) + } + + pub(crate) fn context(&self) -> EncodingContext<'_> { + self.system_reader.context() + } } pub struct LazyElementIterator<'iter, 'data, D: LazyDecoder<'data>> { diff --git a/src/lazy/system_reader.rs b/src/lazy/system_reader.rs index 0b118a66..f46b9838 100644 --- a/src/lazy/system_reader.rs +++ b/src/lazy/system_reader.rs @@ -1,7 +1,5 @@ #![allow(non_camel_case_types)] -use std::cell::RefCell; - use bumpalo::Bump as BumpAllocator; use crate::lazy::any_encoding::{AnyEncoding, LazyRawAnyReader}; @@ -88,6 +86,16 @@ pub struct LazySystemReader<'data, D: LazyDecoder<'data>> { allocator: BumpAllocator, } +impl<'data, D: LazyDecoder<'data>> LazySystemReader<'data, D> { + pub(crate) fn macro_table_mut(&mut self) -> &mut MacroTable { + &mut self.macro_table + } + + pub(crate) fn context(&self) -> EncodingContext<'_> { + EncodingContext::new(&self.macro_table, &self.symbol_table, &self.allocator) + } +} + pub type LazySystemBinaryReader<'data> = LazySystemReader<'data, BinaryEncoding_1_0>; pub type LazySystemTextReader_1_0<'data> = LazySystemReader<'data, TextEncoding_1_0>; pub type LazySystemTextReader_1_1<'data> = LazySystemReader<'data, TextEncoding_1_1>; @@ -106,7 +114,7 @@ impl<'data> LazySystemAnyReader<'data> { let raw_reader = LazyRawAnyReader::new(ion_data); let expanding_reader = LazyExpandingReader::new(raw_reader); LazySystemReader { - expanding_reader: expanding_reader, + expanding_reader, pending_lst: PendingLst { is_lst_append: false, symbols: Vec::new(), @@ -123,7 +131,7 @@ impl<'data> LazySystemBinaryReader<'data> { let raw_reader = LazyRawBinaryReader::new(ion_data); let expanding_reader = LazyExpandingReader::new(raw_reader); LazySystemReader { - expanding_reader: expanding_reader, + expanding_reader, pending_lst: PendingLst { is_lst_append: false, symbols: Vec::new(), @@ -140,7 +148,7 @@ impl<'data> LazySystemTextReader_1_1<'data> { let raw_reader = LazyRawTextReader_1_1::new(ion_data); let expanding_reader = LazyExpandingReader::new(raw_reader); LazySystemReader { - expanding_reader: expanding_reader, + expanding_reader, pending_lst: PendingLst { is_lst_append: false, symbols: Vec::new(), diff --git a/src/lazy/text/buffer.rs b/src/lazy/text/buffer.rs index 4db929a3..51f2abd0 100644 --- a/src/lazy/text/buffer.rs +++ b/src/lazy/text/buffer.rs @@ -35,7 +35,7 @@ use crate::lazy::text::parse_result::{IonMatchResult, IonParseResult}; use crate::lazy::text::raw::r#struct::LazyRawTextField_1_0; use crate::lazy::text::raw::sequence::{RawTextListIterator_1_0, RawTextSExpIterator_1_0}; use crate::lazy::text::raw::v1_1::reader::{ - EncodedTextMacroInvocation, MacroIdRef, RawTextListIterator_1_1, RawTextMacroInvocation, + EncodedTextMacroInvocation, MacroIdRef, RawTextEExpression_1_1, RawTextListIterator_1_1, RawTextSExpIterator_1_1, RawTextStructIterator_1_1, }; use crate::lazy::text::value::{LazyRawTextValue_1_0, LazyRawTextValue_1_1, MatchedRawTextValue}; @@ -479,7 +479,7 @@ impl<'data> TextBufferView<'data> { 'data, ( (MatchedFieldName, Range), - RawTextMacroInvocation<'data>, + RawTextEExpression_1_1<'data>, ), > { terminated( @@ -1024,7 +1024,7 @@ impl<'data> TextBufferView<'data> { /// Matches an e-expression invoking a macro. /// /// If the input does not contain the entire e-expression, returns `IonError::Incomplete(_)`. - pub fn match_e_expression(self) -> IonParseResult<'data, RawTextMacroInvocation<'data>> { + pub fn match_e_expression(self) -> IonParseResult<'data, RawTextEExpression_1_1<'data>> { let (exp_body, _) = tag("(:")(self)?; // TODO: Support macro ID kinds besides unqualified names let (exp_body_after_id, (macro_id_bytes, _matched_symbol)) = @@ -1061,7 +1061,7 @@ impl<'data> TextBufferView<'data> { // For the matched span, we use `self` again to include the opening `(:` let matched = self.slice(0, span.len()); let remaining = self.slice_to_end(span.len()); - let macro_invocation = RawTextMacroInvocation::new( + let macro_invocation = RawTextEExpression_1_1::new( macro_id, EncodedTextMacroInvocation::new(macro_id_bytes.len() as u16), matched, diff --git a/src/lazy/text/raw/v1_1/reader.rs b/src/lazy/text/raw/v1_1/reader.rs index fbc098a4..ccbc43f3 100644 --- a/src/lazy/text/raw/v1_1/reader.rs +++ b/src/lazy/text/raw/v1_1/reader.rs @@ -1,5 +1,6 @@ #![allow(non_camel_case_types)] +use std::fmt; use std::fmt::{Debug, Formatter}; use std::ops::Range; @@ -39,20 +40,20 @@ pub enum MacroIdRef<'data> { } #[derive(Copy, Clone)] -pub struct RawTextMacroInvocation<'data> { +pub struct RawTextEExpression_1_1<'data> { pub(crate) encoded_expr: EncodedTextMacroInvocation, pub(crate) input: TextBufferView<'data>, pub(crate) id: MacroIdRef<'data>, } -impl<'data> Debug for RawTextMacroInvocation<'data> { +impl<'data> Debug for RawTextEExpression_1_1<'data> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { // This is a text macro and the parser accepted it, so it's valid UTF-8. We can `unwrap()`. write!(f, "", self.input.as_text().unwrap()) } } -impl<'data> RawTextMacroInvocation<'data> { +impl<'data> RawTextEExpression_1_1<'data> { pub(crate) fn new( id: MacroIdRef<'data>, encoded_expr: EncodedTextMacroInvocation, @@ -127,11 +128,23 @@ impl<'data> LazyRawReader<'data, TextEncoding_1_1> for LazyRawTextReader_1_1<'da } } -#[derive(Debug, Copy, Clone)] +#[derive(Copy, Clone)] pub struct LazyRawTextList_1_1<'data> { pub(crate) value: LazyRawTextValue_1_1<'data>, } +impl<'a> Debug for LazyRawTextList_1_1<'a> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "[")?; + for value in self.iter() { + write!(f, "{:?}, ", value?.expect_value()?.read()?)?; + } + write!(f, "]").unwrap(); + + Ok(()) + } +} + #[derive(Debug, Copy, Clone)] pub struct RawTextListIterator_1_1<'data> { input: TextBufferView<'data>, @@ -190,11 +203,23 @@ impl<'data> RawTextListIterator_1_1<'data> { } } -#[derive(Debug, Copy, Clone)] +#[derive(Copy, Clone)] pub struct LazyRawTextSExp_1_1<'data> { pub(crate) value: LazyRawTextValue_1_1<'data>, } +impl<'a> Debug for LazyRawTextSExp_1_1<'a> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "(")?; + for value in self.iter() { + write!(f, "{:?} ", value?.expect_value()?.read()?)?; + } + write!(f, ")").unwrap(); + + Ok(()) + } +} + #[derive(Debug, Copy, Clone)] pub struct RawTextSExpIterator_1_1<'data> { input: TextBufferView<'data>, @@ -211,11 +236,35 @@ impl<'data> RawTextSExpIterator_1_1<'data> { } } -#[derive(Debug, Copy, Clone)] +#[derive(Copy, Clone)] pub struct LazyRawTextStruct_1_1<'data> { pub(crate) value: LazyRawTextValue_1_1<'data>, } +impl<'a> Debug for LazyRawTextStruct_1_1<'a> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{{")?; + for field in self.iter() { + match field? { + LazyRawFieldExpr::::NameValuePair( + name, + RawValueExpr::ValueLiteral(value), + ) => write!(f, "{name:?}: {value:?}, "), + LazyRawFieldExpr::::NameValuePair( + name, + RawValueExpr::MacroInvocation(invocation), + ) => write!(f, "{name:?}: {invocation:?}, "), + LazyRawFieldExpr::::MacroInvocation(invocation) => { + write!(f, "{invocation:?}, ") + } + }?; + } + write!(f, "}}").unwrap(); + + Ok(()) + } +} + #[derive(Debug, Copy, Clone)] pub struct RawTextStructIterator_1_1<'data> { input: TextBufferView<'data>, diff --git a/src/types/struct.rs b/src/types/struct.rs index bd59a389..1d1bb607 100644 --- a/src/types/struct.rs +++ b/src/types/struct.rs @@ -267,7 +267,7 @@ where by_name .entry(field_name.clone()) - .or_insert_with(IndexVec::new) + .or_default() .push(by_index.len()); by_index.push((field_name, field_value)); }