Merge branch 'main' into feat-decon-owned-elt

src/lazy/any_encoding.rs

@@ -65,6 +65,7 @@ use crate::lazy::text::value::{
     LazyRawTextVersionMarker_1_1, RawTextAnnotationsIterator,
 };
 use crate::symbol_table::{SystemSymbolTable, SYSTEM_SYMBOLS_1_0, SYSTEM_SYMBOLS_1_1};
+use crate::LazyRawValueKind::{Binary_1_0, Binary_1_1, Text_1_0, Text_1_1};
 use crate::{try_next, Encoding, IonResult, IonType, RawStreamItem, RawSymbolRef};
 
 /// An implementation of the `LazyDecoder` trait that can read any encoding of Ion.
@@ -1046,6 +1047,15 @@ impl<'top> LazyRawValue<'top, AnyEncoding> for LazyRawAnyValue<'top> {
         }
     }
 
+    fn is_delimited(&self) -> bool {
+        match &self.encoding {
+            Text_1_0(v) => v.is_delimited(),
+            Binary_1_0(v) => v.is_delimited(),
+            Text_1_1(v) => v.is_delimited(),
+            Binary_1_1(v) => v.is_delimited(),
+        }
+    }
+
     fn has_annotations(&self) -> bool {
         use LazyRawValueKind::*;
         match &self.encoding {

src/lazy/binary/raw/v1_1/value.rs

@@ -144,6 +144,10 @@ impl<'top> LazyRawValue<'top, BinaryEncoding_1_1> for &'top LazyRawBinaryValue_1
         self.encoded_value.header().is_null()
     }
 
+    fn is_delimited(&self) -> bool {
+        !self.delimited_contents.is_none()
+    }
+
     fn has_annotations(&self) -> bool {
         self.encoded_value.has_annotations()
     }

src/lazy/binary/raw/value.rs

@@ -115,6 +115,10 @@ impl<'top> LazyRawValue<'top, BinaryEncoding_1_0> for LazyRawBinaryValue_1_0<'to
         self.is_null()
     }
 
+    fn is_delimited(&self) -> bool {
+        false
+    }
+
     fn has_annotations(&self) -> bool {
         self.has_annotations()
     }
@@ -221,6 +225,19 @@ pub trait EncodedBinaryValue<'top, D: Decoder>: LazyRawValue<'top, D> {
         let body_bytes = &value_span.bytes()[value_span.len() - body_length..];
         Span::with_offset(value_span.range().end - body_length, body_bytes)
     }
+
+    fn delimited_end_span(&self) -> Span<'top> {
+        let bytes = self.span().bytes();
+        let end = bytes.len();
+        let range = if !self.is_delimited() {
+            end..end
+        } else {
+            debug_assert!(bytes[end - 1] == 0xF0);
+            end - 1..end
+        };
+        let end_bytes = bytes.get(range).unwrap();
+        Span::with_offset(self.range().end - end_bytes.len(), end_bytes)
+    }
 }
 
 impl<'top> EncodedBinaryValue<'top, BinaryEncoding_1_0> for LazyRawBinaryValue_1_0<'top> {

src/lazy/decoder.rs

@@ -18,8 +18,9 @@ use crate::lazy::streaming_raw_reader::RawReaderState;
 use crate::read_config::ReadConfig;
 use crate::result::IonFailure;
 use crate::{
-    v1_0, v1_1, Catalog, Encoding, IonResult, IonType, LazyExpandedFieldName, LazyExpandedValue,
-    LazyRawWriter, MacroExpr, RawSymbolRef, ValueExpr, ValueRef,
+    v1_0, v1_1, Catalog, Encoding, FieldExpr, IonResult, IonType, LazyExpandedFieldName,
+    LazyExpandedValue, LazyRawAnyFieldName, LazyRawWriter, MacroExpr, RawSymbolRef, ValueExpr,
+    ValueRef,
 };
 
 pub trait HasSpan<'top>: HasRange {
@@ -257,6 +258,21 @@ pub enum LazyRawFieldExpr<'top, D: Decoder> {
 }
 
 impl<'top, D: Decoder> LazyRawFieldExpr<'top, D> {
+    pub fn resolve(self, context: EncodingContextRef<'top>) -> IonResult<FieldExpr<'top, D>> {
+        use LazyRawFieldExpr::*;
+        let field = match self {
+            NameValue(name, value) => FieldExpr::NameValue(
+                name.resolve(context),
+                LazyExpandedValue::from_literal(context, value),
+            ),
+            NameEExp(name, eexp) => {
+                FieldExpr::NameMacro(name.resolve(context), eexp.resolve(context)?.into())
+            }
+            EExp(eexp) => FieldExpr::EExp(eexp.resolve(context)?),
+        };
+        Ok(field)
+    }
+
     pub fn expect_name_value(self) -> IonResult<(D::FieldName<'top>, D::Value<'top>)> {
         let LazyRawFieldExpr::NameValue(name, value) = self else {
             return IonResult::decoding_error(format!(
@@ -381,8 +397,8 @@ impl<D: Decoder> HasRange for LazyRawFieldExpr<'_, D> {
 // internal code that is defined in terms of `LazyRawField` to call the private `into_value()`
 // function while also preventing users from seeing or depending on it.
 pub(crate) mod private {
-    use crate::lazy::expanded::macro_evaluator::{MacroExpr, RawEExpression};
-    use crate::lazy::expanded::r#struct::UnexpandedField;
+    use crate::lazy::expanded::macro_evaluator::RawEExpression;
+    use crate::lazy::expanded::r#struct::FieldExpr;
     use crate::lazy::expanded::EncodingContextRef;
     use crate::{try_next, try_or_some_err, IonResult, LazyExpandedValue, LazyRawFieldName};
 
@@ -395,47 +411,44 @@ pub(crate) mod private {
     }
 
     pub trait LazyRawStructPrivate<'top, D: Decoder> {
-        /// Creates an iterator that converts each raw struct field into an `UnexpandedField`, a
+        /// Creates an iterator that converts each raw struct field into an `FieldExpr`, a
         /// common representation for both raw fields and template fields that is used in the
         /// expansion process.
-        fn unexpanded_fields(
+        fn field_exprs(
             &self,
             context: EncodingContextRef<'top>,
-        ) -> RawStructUnexpandedFieldsIterator<'top, D>;
+        ) -> RawStructFieldExprIterator<'top, D>;
     }
 
-    pub struct RawStructUnexpandedFieldsIterator<'top, D: Decoder> {
+    pub struct RawStructFieldExprIterator<'top, D: Decoder> {
         context: EncodingContextRef<'top>,
         raw_fields: <D::Struct<'top> as LazyRawStruct<'top, D>>::Iterator,
     }
 
-    impl<'top, D: Decoder> RawStructUnexpandedFieldsIterator<'top, D> {
+    impl<'top, D: Decoder> RawStructFieldExprIterator<'top, D> {
         pub fn context(&self) -> EncodingContextRef<'top> {
             self.context
         }
     }
 
-    impl<'top, D: Decoder> Iterator for RawStructUnexpandedFieldsIterator<'top, D> {
-        type Item = IonResult<UnexpandedField<'top, D>>;
+    impl<'top, D: Decoder> Iterator for RawStructFieldExprIterator<'top, D> {
+        type Item = IonResult<FieldExpr<'top, D>>;
 
         fn next(&mut self) -> Option<Self::Item> {
             let field: LazyRawFieldExpr<'top, D> = try_next!(self.raw_fields.next());
             use LazyRawFieldExpr::*;
             let unexpanded_field = match field {
-                NameValue(name, value) => UnexpandedField::NameValue(
+                NameValue(name, value) => FieldExpr::NameValue(
                     name.resolve(self.context),
                     LazyExpandedValue::from_literal(self.context, value),
                 ),
                 NameEExp(name, raw_eexp) => {
                     let eexp = try_or_some_err!(raw_eexp.resolve(self.context));
-                    UnexpandedField::NameMacro(
-                        name.resolve(self.context),
-                        MacroExpr::from_eexp(eexp),
-                    )
+                    FieldExpr::NameMacro(name.resolve(self.context), eexp.into())
                 }
                 EExp(raw_eexp) => {
                     let eexp = try_or_some_err!(raw_eexp.resolve(self.context));
-                    UnexpandedField::Macro(MacroExpr::from_eexp(eexp))
+                    FieldExpr::EExp(eexp)
                 }
             };
             Some(Ok(unexpanded_field))
@@ -446,12 +459,12 @@ pub(crate) mod private {
     where
         S: LazyRawStruct<'top, D>,
     {
-        fn unexpanded_fields(
+        fn field_exprs(
             &self,
             context: EncodingContextRef<'top>,
-        ) -> RawStructUnexpandedFieldsIterator<'top, D> {
+        ) -> RawStructFieldExprIterator<'top, D> {
             let raw_fields = <Self as LazyRawStruct<'top, D>>::iter(self);
-            RawStructUnexpandedFieldsIterator {
+            RawStructFieldExprIterator {
                 context,
                 raw_fields,
             }
@@ -600,6 +613,8 @@ pub trait LazyRawValue<'top, D: Decoder>:
 {
     fn ion_type(&self) -> IonType;
     fn is_null(&self) -> bool;
+
+    fn is_delimited(&self) -> bool;
     fn has_annotations(&self) -> bool;
     fn annotations(&self) -> D::AnnotationsIterator<'top>;
     fn read(&self) -> IonResult<RawValueRef<'top, D>>;
@@ -675,7 +690,7 @@ pub trait LazyRawStruct<'top, D: Decoder>:
 }
 
 pub trait LazyRawFieldName<'top, D: Decoder<FieldName<'top> = Self>>:
-    HasSpan<'top> + Copy + Debug + Clone
+    Into<LazyRawAnyFieldName<'top>> + HasSpan<'top> + Copy + Debug + Clone
 {
     fn read(&self) -> IonResult<RawSymbolRef<'top>>;
 

src/lazy/encoder/binary/v1_1/flex_sym.rs

@@ -95,7 +95,7 @@ impl<'top> FlexSym<'top> {
             Ordering::Equal => {
                 // Get the first byte following the leading FlexInt
                 let flex_int_len = value.size_in_bytes();
-                if input.len() < flex_int_len {
+                if input.len() <= flex_int_len {
                     return IonResult::incomplete("reading a FlexSym", offset);
                 }
                 let byte = input[flex_int_len];

src/lazy/expanded/macro_evaluator.rs

@@ -114,7 +114,7 @@ where
 #[derive(Copy, Clone)]
 pub struct MacroExpr<'top, D: Decoder> {
     kind: MacroExprKind<'top, D>,
-    variable: Option<TemplateVariableReference<'top>>,
+    pub(crate) variable: Option<TemplateVariableReference<'top>>,
 }
 
 impl<D: Decoder> Debug for MacroExpr<'_, D> {
@@ -137,8 +137,8 @@ impl<'top, D: Decoder> MacroExpr<'top, D> {
         }
     }
 
-    pub fn via_variable(mut self, variable_ref: TemplateVariableReference<'top>) -> Self {
-        self.variable = Some(variable_ref);
+    pub fn via_variable(mut self, variable_ref: Option<TemplateVariableReference<'top>>) -> Self {
+        self.variable = variable_ref;
         self
     }
 
@@ -149,6 +149,30 @@ impl<'top, D: Decoder> MacroExpr<'top, D> {
             MacroExprKind::EExp(e) => e.expand(),
             MacroExprKind::EExpArgGroup(g) => g.expand(),
         }
+        .map(|expansion| expansion.via_variable(self.variable))
+    }
+
+    pub fn range(&self) -> Option<Range<usize>> {
+        self.span().as_ref().map(Span::range)
+    }
+
+    /// If this `ValueExpr` represents an entity encoded in the data stream, returns `Some(range)`.
+    /// If it represents an ephemeral value produced by a macro evaluation, returns `None`.
+    pub fn span(&self) -> Option<Span<'top>> {
+        use MacroExprKind::*;
+        match self.kind {
+            TemplateMacro(_) | TemplateArgGroup(_) => None,
+            EExp(eexp) => Some(eexp.span()),
+            EExpArgGroup(group) => Some(group.span()),
+        }
+    }
+
+    pub fn is_eexp(&self) -> bool {
+        matches!(self.kind, MacroExprKind::EExp(_))
+    }
+
+    pub fn is_tdl_macro(&self) -> bool {
+        matches!(self.kind, MacroExprKind::TemplateMacro(_))
     }
 }
 
@@ -389,6 +413,19 @@ impl<D: Decoder> Debug for ValueExpr<'_, D> {
 }
 
 impl<'top, D: Decoder> ValueExpr<'top, D> {
+    pub fn via_variable(
+        self,
+        template_variable_ref: Option<TemplateVariableReference<'top>>,
+    ) -> Self {
+        use ValueExpr::*;
+        match self {
+            ValueLiteral(value) => ValueLiteral(value.via_variable(template_variable_ref)),
+            MacroInvocation(invocation) => {
+                MacroInvocation(invocation.via_variable(template_variable_ref))
+            }
+        }
+    }
+
     /// Like [`evaluate_singleton_in`](Self::evaluate_singleton_in), but uses an empty environment.
     pub fn evaluate_singleton(&self) -> IonResult<LazyExpandedValue<'top, D>> {
         self.evaluate_singleton_in(Environment::empty())
@@ -475,7 +512,7 @@ impl<'top, D: Decoder> ValueExpr<'top, D> {
             ValueExpr::ValueLiteral(value) => {
                 use ExpandedValueSource::*;
                 match value.source {
-                    SingletonEExp(_) => todo!(),
+                    SingletonEExp(eexp) => Some(eexp.span()),
                     ValueLiteral(literal) => Some(literal.span()),
                     Template(_, _) | Constructed(_, _) => None,
                 }
@@ -522,9 +559,15 @@ pub struct MacroExpansion<'top, D: Decoder> {
     kind: MacroExpansionKind<'top, D>,
     environment: Environment<'top, D>,
     is_complete: bool,
+    variable_ref: Option<TemplateVariableReference<'top>>,
 }
 
 impl<'top, D: Decoder> MacroExpansion<'top, D> {
+    pub fn via_variable(mut self, variable_ref: Option<TemplateVariableReference<'top>>) -> Self {
+        self.variable_ref = variable_ref;
+        self
+    }
+
     pub fn context(&self) -> EncodingContextRef<'top> {
         self.context
     }
@@ -565,6 +608,7 @@ impl<'top, D: Decoder> MacroExpansion<'top, D> {
             kind,
             context,
             is_complete: false,
+            variable_ref: None,
         }
     }
 
@@ -591,6 +635,7 @@ impl<'top, D: Decoder> MacroExpansion<'top, D> {
             // `none` is trivial and requires no delegation
             None => Ok(MacroExpansionStep::FinalStep(Option::None)),
         }
+        .map(|expansion| expansion.via_variable(self.variable_ref))
     }
 }
 
@@ -635,6 +680,15 @@ impl<'top, D: Decoder> MacroExpansionStep<'top, D> {
     pub fn is_final(&self) -> bool {
         matches!(self, MacroExpansionStep::FinalStep(_))
     }
+
+    pub fn via_variable(mut self, variable_ref: Option<TemplateVariableReference<'top>>) -> Self {
+        use MacroExpansionStep::*;
+        match &mut self {
+            Step(expr) => Step(expr.via_variable(variable_ref)),
+            FinalStep(Some(expr)) => FinalStep(Some(expr.via_variable(variable_ref))),
+            FinalStep(None) => FinalStep(None),
+        }
+    }
 }
 
 /// The internal bookkeeping representation used by a [`MacroEvaluator`].
@@ -768,7 +822,7 @@ impl<'top, D: Decoder> MacroEvaluator<'top, D> {
     }
 
     pub fn for_macro_expr(macro_expr: MacroExpr<'top, D>) -> IonResult<Self> {
-        let expansion = MacroExpansion::initialize(macro_expr)?;
+        let expansion = macro_expr.expand()?;
         Ok(Self::for_expansion(expansion))
     }
 
@@ -889,7 +943,7 @@ impl<'top, D: Decoder> StackedMacroEvaluator<'top, D> {
     /// current encoding context and push the resulting `MacroExpansion` onto the stack.
     pub fn push(&mut self, invocation: impl Into<MacroExpr<'top, D>>) -> IonResult<()> {
         let macro_expr = invocation.into();
-        let expansion = match MacroExpansion::initialize(macro_expr) {
+        let expansion = match macro_expr.expand() {
             Ok(expansion) => expansion,
             Err(e) => return Err(e),
         };

src/lazy/expanded/mod.rs

@@ -46,9 +46,7 @@ use crate::lazy::decoder::{Decoder, LazyRawValue};
 use crate::lazy::encoding::RawValueLiteral;
 use crate::lazy::expanded::compiler::TemplateCompiler;
 use crate::lazy::expanded::e_expression::EExpression;
-use crate::lazy::expanded::macro_evaluator::{
-    MacroEvaluator, MacroExpansion, MacroExpr, RawEExpression,
-};
+use crate::lazy::expanded::macro_evaluator::{MacroEvaluator, MacroExpr, RawEExpression};
 use crate::lazy::expanded::macro_table::{Macro, MacroTable, ION_1_1_SYSTEM_MACROS};
 use crate::lazy::expanded::r#struct::LazyExpandedStruct;
 use crate::lazy::expanded::sequence::Environment;
@@ -887,8 +885,11 @@ impl<'top, Encoding: Decoder> LazyExpandedValue<'top, Encoding> {
         }
     }
 
-    pub(crate) fn via_variable(mut self, variable_ref: TemplateVariableReference<'top>) -> Self {
-        self.variable = Some(variable_ref);
+    pub(crate) fn via_variable(
+        mut self,
+        variable_ref: Option<TemplateVariableReference<'top>>,
+    ) -> Self {
+        self.variable = variable_ref;
         self
     }
 
@@ -978,7 +979,7 @@ impl<'top, Encoding: Decoder> LazyExpandedValue<'top, Encoding> {
         &self,
         eexp: &EExpression<'top, Encoding>,
     ) -> IonResult<ValueRef<'top, Encoding>> {
-        let new_expansion = MacroExpansion::initialize(MacroExpr::from_eexp(*eexp))?;
+        let new_expansion = MacroExpr::from_eexp(*eexp).expand()?;
         new_expansion.expand_singleton()?.read_resolved()
     }