Adds incompleteness detection tests (#819)

src/lazy/encoder/text/v1_0/value_writer.rs

@@ -166,19 +166,6 @@ pub(crate) struct TextContainerWriter_1_0<'a, W: Write> {
     trailing_delimiter: &'static str,
 }
 
-impl<'a, W: Write> Drop for TextContainerWriter_1_0<'a, W> {
-    fn drop(&mut self) {
-        // If the user didn't call `end`, the closing delimiter was not written to output.
-        // It's too late to call it here because we can't return a `Result`.
-        if !self.has_been_closed {
-            panic!(
-                "Container writer ({:?}) was dropped without calling `end()`.",
-                self.container_type
-            );
-        }
-    }
-}
-
 impl<'a, W: Write> TextContainerWriter_1_0<'a, W> {
     pub fn new(
         writer: &'a mut LazyRawTextWriter_1_0<W>,

src/lazy/expanded/macro_evaluator.rs

@@ -1781,7 +1781,7 @@ mod tests {
           (values
             (make_string "foo" '''bar''' "\x62\u0061\U0000007A")
             (make_string 
-                '''Hello'''  
+                '''Hello'''
                 ''', '''
                 "world!"))
         )

src/lazy/expanded/struct.rs

@@ -534,7 +534,10 @@ impl<'top, D: Decoder> ExpandedStructIterator<'top, D> {
         evaluator.push(expansion);
         let expanded_value = match evaluator.next()? {
             Some(item) => item,
-            None => return IonResult::decoding_error(format!("macros in field name position must produce a single struct; '{:?}' produced nothing", invocation)),
+            None => {
+                // The macro produced an empty stream; return to reading from input.
+                return Ok(());
+            }
         };
         let struct_ = match expanded_value.read()? {
             ExpandedValueRef::Struct(s) => s,

src/lazy/text/matched.rs

@@ -309,8 +309,10 @@ impl MatchedDecimal {
 
         let digits_text = sanitized.as_utf8(digits.offset())?;
         let magnitude: Int = i128::from_str(digits_text)
-            .map_err(|_| {
-                IonError::decoding_error("decimal magnitude was larger than supported size")
+            .map_err(|e| {
+                IonError::decoding_error(format!(
+                    "decimal magnitude '{digits_text}' was larger than supported size ({e:?}"
+                ))
             })?
             .into();
 

src/lazy/text/parse_result.rs

@@ -141,13 +141,23 @@ impl<'data> From<InvalidInputError<'data>> for IonError {
         let (buffer_head, buffer_tail) = match input.as_text() {
             // The buffer contains UTF-8 bytes, so we'll display it as text
             Ok(text) => {
-                let head = text.chars().take(NUM_CHARS_TO_SHOW).collect::<String>();
-                let tail_backwards = text
-                    .chars()
-                    .rev()
+                let mut head_chars = text.chars();
+                let mut head = (&mut head_chars)
+                    .take(NUM_CHARS_TO_SHOW)
+                    .collect::<String>();
+                if head_chars.next().is_some() {
+                    head.push_str("...");
+                }
+                let mut tail_chars = text.chars().rev();
+                let tail_backwards = (&mut tail_chars)
                     .take(NUM_CHARS_TO_SHOW)
                     .collect::<Vec<char>>();
-                let tail = tail_backwards.iter().rev().collect::<String>();
+                let mut tail = String::new();
+                if tail_chars.next().is_some() {
+                    tail.push_str("...");
+                }
+                tail.push_str(tail_backwards.iter().rev().collect::<String>().as_str());
+
                 (head, tail)
             }
             // The buffer contains non-text bytes, so we'll show its contents as formatted hex
@@ -170,8 +180,8 @@ impl<'data> From<InvalidInputError<'data>> for IonError {
             message,
             r#"
         offset={}
-        buffer head=<{}...>
-        buffer tail=<...{}>
+        buffer head=<{}>
+        buffer tail=<{}>
         buffer len={}
         "#,
             invalid_input_error.input.offset(),

src/lazy/text/raw/v1_1/reader.rs

@@ -228,7 +228,7 @@ impl<'a> Debug for LazyRawTextList_1_1<'a> {
 #[derive(Debug, Copy, Clone)]
 pub struct RawTextListIterator_1_1<'top> {
     input: TextBufferView<'top>,
-    // If this iterator has returned an error, it should return `None` forever afterwards
+    // If this iterator has returned an error, it should return `None` forever afterward
     has_returned_error: bool,
 }
 

tests/ion_tests/detect_incomplete_text.rs

@@ -0,0 +1,48 @@
+#![cfg(feature = "experimental-reader-writer")]
+
+use crate::ion_tests::{DataStraw, ELEMENT_GLOBAL_SKIP_LIST};
+use ion_rs::{AnyEncoding, ElementReader, IonError, IonStream, Reader};
+use std::fs;
+use std::io::BufReader;
+use test_generator::test_resources;
+
+#[test_resources("ion-tests/iontestdata_1_1/good/**/*.ion")]
+fn detect_incomplete_input(file_name: &str) {
+    // Canonicalize the file name so it can be compared to skip list file names without worrying
+    // about path separators.
+    let file_name: String = fs::canonicalize(file_name)
+        .unwrap()
+        .to_string_lossy()
+        .into();
+    // Map each 1.0 skip list file name to the 1.1 equivalent
+    let skip_list_1_1: Vec<String> = ELEMENT_GLOBAL_SKIP_LIST
+        .iter()
+        .map(|file_1_0| file_1_0.replace("_1_0", "_1_1"))
+        .filter_map(|filename| {
+            // Canonicalize the skip list file names so they're in the host OS' preferred format.
+            // This involves looking up the actual file; if canonicalization fails, the file could
+            // not be found/read which could mean the skip list is outdated.
+            fs::canonicalize(filename).ok()
+        })
+        .map(|filename| filename.to_string_lossy().into())
+        .collect();
+    if skip_list_1_1.contains(&file_name.to_owned()) {
+        return;
+    }
+    println!("testing {file_name}");
+    let file = fs::File::open(file_name).unwrap();
+    let buf_reader = BufReader::new(file);
+    let input = DataStraw::new(buf_reader);
+    let ion_stream = IonStream::new(input);
+    let mut reader = Reader::new(AnyEncoding, ion_stream).unwrap();
+    // Manually unwrap to allow for pretty-printing of errors
+    match reader.read_all_elements() {
+        Ok(_) => {}
+        Err(IonError::Decoding(e)) => {
+            panic!("{:?}: {}", e.position(), e);
+        }
+        Err(other) => {
+            panic!("{other:#?}");
+        }
+    }
+}

tests/ion_tests/mod.rs

@@ -3,6 +3,8 @@
 #![allow(dead_code)]
 
 use std::fs::read;
+use std::io;
+use std::io::Read;
 use std::path::MAIN_SEPARATOR_STR as PATH_SEPARATOR;
 
 use ion_rs::v1_0;
@@ -12,6 +14,7 @@ use ion_rs::{
     Symbol, Value,
 };
 
+mod detect_incomplete_text;
 pub mod lazy_element_ion_tests;
 
 /// Concatenates two slices of string slices together.
@@ -435,3 +438,29 @@ pub const ELEMENT_EQUIVS_SKIP_LIST: SkipList = &[
     "ion-tests/iontestdata_1_0/good/equivs/localSymbolTableNullSlots.ion",
     "ion-tests/iontestdata_1_0/good/equivs/nonIVMNoOps.ion",
 ];
+
+/// An implementation of `io::Read` that only yields a single byte on each
+/// call to `read()`. This is used in tests to confirm that the reader's
+/// data-incompleteness and retry logic will properly handle all corner
+/// cases.
+pub(crate) struct DataStraw<R> {
+    input: R,
+}
+
+impl<R> DataStraw<R> {
+    pub fn new(input: R) -> Self {
+        Self { input }
+    }
+}
+
+impl<R: Read> Read for DataStraw<R> {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        let single_byte_buffer = &mut [0u8; 1];
+        let bytes_read = self.input.read(single_byte_buffer)?;
+        if bytes_read == 0 {
+            return Ok(0);
+        }
+        buf[0] = single_byte_buffer[0];
+        Ok(1)
+    }
+}