Merge branch 'main' into winnow-experiment

src/element/mod.rs

@@ -423,6 +423,21 @@ impl Element {
         &self.value
     }
 
+    /// Consumes self and returns this [Element]'s [Value].
+    pub fn into_value(self) -> Value {
+        self.value
+    }
+
+    /// Consumes self and returns this [Element]'s [Annotations].
+    pub fn into_annotations(self) -> Annotations {
+        self.annotations
+    }
+
+    /// Consumes self and returns this [Element]'s [Annotations] and [Value].
+    pub fn into_parts(self) -> (Annotations, Value) {
+        (self.annotations, self.value)
+    }
+
     pub fn null(null_type: IonType) -> Element {
         null_type.into()
     }
@@ -974,7 +989,7 @@ mod tests {
     use ElemOp::*;
 
     use crate::element::annotations::IntoAnnotations;
-    use crate::{ion_list, ion_sexp, ion_struct, Decimal, Int, IonType, Symbol, Timestamp};
+    use crate::{ion_list, ion_sexp, ion_struct, Decimal, Int, IonType, Symbol, Timestamp, Value};
     use crate::{Annotations, Element, IntoAnnotatedElement, Struct};
 
     /// Makes a timestamp from an RFC-3339 string and panics if it can't
@@ -1412,6 +1427,14 @@ mod tests {
             owned_asserts: vec![
                 assert_pass_try_into!(try_into_lob),
                 assert_pass_try_into!(try_into_blob),
+                Box::new(|e: Element| {
+                    if let Value::Blob(bytes) = e.into_value() {
+                        let data: Vec<_> = bytes.into();
+                        assert_eq!(b"hello", data.as_slice());
+                    } else {
+                        panic!("expected blob");
+                    }
+                }),
             ],
         }
     }
@@ -1429,6 +1452,14 @@ mod tests {
             owned_asserts: vec![
                 assert_pass_try_into!(try_into_lob),
                 assert_pass_try_into!(try_into_clob),
+                Box::new(|e: Element| {
+                    if let Value::Clob(bytes) = e.into_value() {
+                        let data: Vec<_> = bytes.into();
+                        assert_eq!(b"goodbye", data.as_slice());
+                    } else {
+                        panic!("expected clob");
+                    }
+                }),
             ],
         }
     }
@@ -1583,6 +1614,18 @@ mod tests {
         // construct an element to test
         assert_eq!(input_case.ion_type, input_case.elem.ion_type());
 
+        // assert value & annotation accessors
+        let val_ref = input_case.elem.value();
+        let ann_ref = input_case.elem.annotations();
+        let val_owned = input_case.elem.clone().into_value();
+        let ann_owned = input_case.elem.clone().into_annotations();
+        let (ann_owned2, val_owned2) = input_case.elem.clone().into_parts();
+        assert_eq!(val_ref, &val_owned);
+        assert_eq!(val_owned, val_owned2);
+        assert_eq!(ann_ref, &ann_owned);
+        assert_eq!(ann_owned, ann_owned2);
+
+        // assert element operations
         for assert in op_assertions {
             assert(&input_case.elem);
         }

src/lazy/any_encoding.rs

@@ -62,6 +62,7 @@ use crate::lazy::text::value::{
     LazyRawTextVersionMarker_1_1, RawTextAnnotationsIterator,
 };
 use crate::symbol_table::{SystemSymbolTable, SYSTEM_SYMBOLS_1_0, SYSTEM_SYMBOLS_1_1};
+use crate::LazyRawValueKind::{Binary_1_0, Binary_1_1, Text_1_0, Text_1_1};
 use crate::{try_next, Encoding, IonResult, IonType, RawStreamItem, RawSymbolRef};
 use std::fmt::Debug;
 use std::ops::Range;
@@ -1041,6 +1042,15 @@ impl<'top> LazyRawValue<'top, AnyEncoding> for LazyRawAnyValue<'top> {
         }
     }
 
+    fn is_delimited(&self) -> bool {
+        match &self.encoding {
+            Text_1_0(v) => v.is_delimited(),
+            Binary_1_0(v) => v.is_delimited(),
+            Text_1_1(v) => v.is_delimited(),
+            Binary_1_1(v) => v.is_delimited(),
+        }
+    }
+
     fn has_annotations(&self) -> bool {
         use LazyRawValueKind::*;
         match &self.encoding {

src/lazy/binary/raw/v1_1/value.rs

@@ -144,6 +144,10 @@ impl<'top> LazyRawValue<'top, BinaryEncoding_1_1> for &'top LazyRawBinaryValue_1
         self.encoded_value.header().is_null()
     }
 
+    fn is_delimited(&self) -> bool {
+        !self.delimited_contents.is_none()
+    }
+
     fn has_annotations(&self) -> bool {
         self.encoded_value.has_annotations()
     }

src/lazy/binary/raw/value.rs

@@ -115,6 +115,10 @@ impl<'top> LazyRawValue<'top, BinaryEncoding_1_0> for LazyRawBinaryValue_1_0<'to
         self.is_null()
     }
 
+    fn is_delimited(&self) -> bool {
+        false
+    }
+
     fn has_annotations(&self) -> bool {
         self.has_annotations()
     }
@@ -221,6 +225,19 @@ pub trait EncodedBinaryValue<'top, D: Decoder>: LazyRawValue<'top, D> {
         let body_bytes = &value_span.bytes()[value_span.len() - body_length..];
         Span::with_offset(value_span.range().end - body_length, body_bytes)
     }
+
+    fn delimited_end_span(&self) -> Span<'top> {
+        let bytes = self.span().bytes();
+        let end = bytes.len();
+        let range = if !self.is_delimited() {
+            end..end
+        } else {
+            debug_assert!(bytes[end - 1] == 0xF0);
+            end - 1..end
+        };
+        let end_bytes = bytes.get(range).unwrap();
+        Span::with_offset(self.range().end - end_bytes.len(), end_bytes)
+    }
 }
 
 impl<'top> EncodedBinaryValue<'top, BinaryEncoding_1_0> for LazyRawBinaryValue_1_0<'top> {

src/lazy/decoder.rs

@@ -18,8 +18,9 @@ use crate::lazy::streaming_raw_reader::RawReaderState;
 use crate::read_config::ReadConfig;
 use crate::result::IonFailure;
 use crate::{
-    v1_0, v1_1, Catalog, Encoding, IonResult, IonType, LazyExpandedFieldName, LazyExpandedValue,
-    LazyRawWriter, MacroExpr, RawSymbolRef, ValueExpr, ValueRef,
+    v1_0, v1_1, Catalog, Encoding, FieldExpr, IonResult, IonType, LazyExpandedFieldName,
+    LazyExpandedValue, LazyRawAnyFieldName, LazyRawWriter, MacroExpr, RawSymbolRef, ValueExpr,
+    ValueRef,
 };
 
 pub trait HasSpan<'top>: HasRange {
@@ -257,6 +258,21 @@ pub enum LazyRawFieldExpr<'top, D: Decoder> {
 }
 
 impl<'top, D: Decoder> LazyRawFieldExpr<'top, D> {
+    pub fn resolve(self, context: EncodingContextRef<'top>) -> IonResult<FieldExpr<'top, D>> {
+        use LazyRawFieldExpr::*;
+        let field = match self {
+            NameValue(name, value) => FieldExpr::NameValue(
+                name.resolve(context),
+                LazyExpandedValue::from_literal(context, value),
+            ),
+            NameEExp(name, eexp) => {
+                FieldExpr::NameMacro(name.resolve(context), eexp.resolve(context)?.into())
+            }
+            EExp(eexp) => FieldExpr::EExp(eexp.resolve(context)?),
+        };
+        Ok(field)
+    }
+
     pub fn expect_name_value(self) -> IonResult<(D::FieldName<'top>, D::Value<'top>)> {
         let LazyRawFieldExpr::NameValue(name, value) = self else {
             return IonResult::decoding_error(format!(
@@ -381,8 +397,8 @@ impl<D: Decoder> HasRange for LazyRawFieldExpr<'_, D> {
 // internal code that is defined in terms of `LazyRawField` to call the private `into_value()`
 // function while also preventing users from seeing or depending on it.
 pub(crate) mod private {
-    use crate::lazy::expanded::macro_evaluator::{MacroExpr, RawEExpression};
-    use crate::lazy::expanded::r#struct::UnexpandedField;
+    use crate::lazy::expanded::macro_evaluator::RawEExpression;
+    use crate::lazy::expanded::r#struct::FieldExpr;
     use crate::lazy::expanded::EncodingContextRef;
     use crate::{try_next, try_or_some_err, IonResult, LazyExpandedValue, LazyRawFieldName};
 
@@ -395,47 +411,44 @@ pub(crate) mod private {
     }
 
     pub trait LazyRawStructPrivate<'top, D: Decoder> {
-        /// Creates an iterator that converts each raw struct field into an `UnexpandedField`, a
+        /// Creates an iterator that converts each raw struct field into an `FieldExpr`, a
         /// common representation for both raw fields and template fields that is used in the
         /// expansion process.
-        fn unexpanded_fields(
+        fn field_exprs(
             &self,
             context: EncodingContextRef<'top>,
-        ) -> RawStructUnexpandedFieldsIterator<'top, D>;
+        ) -> RawStructFieldExprIterator<'top, D>;
     }
 
-    pub struct RawStructUnexpandedFieldsIterator<'top, D: Decoder> {
+    pub struct RawStructFieldExprIterator<'top, D: Decoder> {
         context: EncodingContextRef<'top>,
         raw_fields: <D::Struct<'top> as LazyRawStruct<'top, D>>::Iterator,
     }
 
-    impl<'top, D: Decoder> RawStructUnexpandedFieldsIterator<'top, D> {
+    impl<'top, D: Decoder> RawStructFieldExprIterator<'top, D> {
         pub fn context(&self) -> EncodingContextRef<'top> {
             self.context
         }
     }
 
-    impl<'top, D: Decoder> Iterator for RawStructUnexpandedFieldsIterator<'top, D> {
-        type Item = IonResult<UnexpandedField<'top, D>>;
+    impl<'top, D: Decoder> Iterator for RawStructFieldExprIterator<'top, D> {
+        type Item = IonResult<FieldExpr<'top, D>>;
 
         fn next(&mut self) -> Option<Self::Item> {
             let field: LazyRawFieldExpr<'top, D> = try_next!(self.raw_fields.next());
             use LazyRawFieldExpr::*;
             let unexpanded_field = match field {
-                NameValue(name, value) => UnexpandedField::NameValue(
+                NameValue(name, value) => FieldExpr::NameValue(
                     name.resolve(self.context),
                     LazyExpandedValue::from_literal(self.context, value),
                 ),
                 NameEExp(name, raw_eexp) => {
                     let eexp = try_or_some_err!(raw_eexp.resolve(self.context));
-                    UnexpandedField::NameMacro(
-                        name.resolve(self.context),
-                        MacroExpr::from_eexp(eexp),
-                    )
+                    FieldExpr::NameMacro(name.resolve(self.context), eexp.into())
                 }
                 EExp(raw_eexp) => {
                     let eexp = try_or_some_err!(raw_eexp.resolve(self.context));
-                    UnexpandedField::Macro(MacroExpr::from_eexp(eexp))
+                    FieldExpr::EExp(eexp)
                 }
             };
             Some(Ok(unexpanded_field))
@@ -446,12 +459,12 @@ pub(crate) mod private {
     where
         S: LazyRawStruct<'top, D>,
     {
-        fn unexpanded_fields(
+        fn field_exprs(
             &self,
             context: EncodingContextRef<'top>,
-        ) -> RawStructUnexpandedFieldsIterator<'top, D> {
+        ) -> RawStructFieldExprIterator<'top, D> {
             let raw_fields = <Self as LazyRawStruct<'top, D>>::iter(self);
-            RawStructUnexpandedFieldsIterator {
+            RawStructFieldExprIterator {
                 context,
                 raw_fields,
             }
@@ -591,6 +604,8 @@ pub trait LazyRawValue<'top, D: Decoder>:
 {
     fn ion_type(&self) -> IonType;
     fn is_null(&self) -> bool;
+
+    fn is_delimited(&self) -> bool;
     fn has_annotations(&self) -> bool;
     fn annotations(&self) -> D::AnnotationsIterator<'top>;
     fn read(&self) -> IonResult<RawValueRef<'top, D>>;
@@ -666,7 +681,7 @@ pub trait LazyRawStruct<'top, D: Decoder>:
 }
 
 pub trait LazyRawFieldName<'top, D: Decoder<FieldName<'top> = Self>>:
-    HasSpan<'top> + Copy + Debug + Clone
+    Into<LazyRawAnyFieldName<'top>> + HasSpan<'top> + Copy + Debug + Clone
 {
     fn read(&self) -> IonResult<RawSymbolRef<'top>>;
 

src/lazy/encoder/binary/v1_1/flex_sym.rs

@@ -95,7 +95,7 @@ impl<'top> FlexSym<'top> {
             Ordering::Equal => {
                 // Get the first byte following the leading FlexInt
                 let flex_int_len = value.size_in_bytes();
-                if input.len() < flex_int_len {
+                if input.len() <= flex_int_len {
                     return IonResult::incomplete("reading a FlexSym", offset);
                 }
                 let byte = input[flex_int_len];

src/lazy/encoder/value_writer.rs

@@ -505,7 +505,6 @@ mod tests {
             // ...when writing the symbol once more.
             .write_symbol(name_symbol)?;
         let bytes = writer.close()?;
-        println!("bytes: {bytes:#02x?}");
         let actual = Element::read_all(&bytes)?;
         let expected = ion_seq!(
             "name".as_symbol_ref()