ds: add valid check

src/idx/dataset/any.rs

@@ -116,6 +116,8 @@ pub trait DatasetExt {
 
  fn chunk_shape(&self) -> &[u64];
 
+ fn valid(&self) -> anyhow::Result<bool>;
+
  fn as_par_reader(&self, p: &dyn AsRef<Path>) -> anyhow::Result<Box<dyn DatasetExtReader + '_>>;
 }
 
@@ -140,6 +142,10 @@ impl<'a> DatasetExt for DatasetD<'a> {
  self.inner().chunk_shape()
  }
 
+ fn valid(&self) -> anyhow::Result<bool> {
+ self.inner().valid()
+ }
+
  fn as_par_reader(&self, p: &dyn AsRef<Path>) -> anyhow::Result<Box<dyn DatasetExtReader + '_>> {
  self.inner().as_par_reader(p)
  }

src/idx/dataset/dataset.rs

@@ -300,6 +300,26 @@ impl<const D: usize> Dataset<'_, D> {
  self.shape.is_empty()
  }
 
+ /// Test whether dataset and chunk layout is valid.
+ pub fn valid(&self) -> anyhow::Result<bool> {
+ for chunk in self.chunks.iter() {
+ let offset = chunk.offset.iter().map(|u| u.get()).collect::<Vec<_>>();
+ ensure!(chunk.contains(&offset, &self.chunk_shape) == std::cmp::Ordering::Equal, "chunk does not contain its offset");
+ }
+
+ let end: u64 = self.shape.iter().product();
+ let chunk_sh: u64 = self.chunk_shape.iter().product();
+
+ ensure!(end % chunk_sh == 0, "chunks not modulo of dataset shape: {0:?} vs {1:?}", self.shape, self.chunk_shape);
+ let chunks = end / chunk_sh;
+ ensure!(chunks == self.chunks.len() as u64, "number of chunks does not match dataset shape: {chunks} != {}", self.chunks.len());
+
+
+
+
+ Ok(true)
+ }
+
  /// Returns an iterator over chunk, offset and size which if joined will make up the specified slice through the
  /// variable.
  pub fn chunk_slices(
@@ -410,6 +430,10 @@ impl<const D: usize> DatasetExt for Dataset<'_, D> {
  self.chunk_shape.as_slice()
  }
 
+ fn valid(&self) -> anyhow::Result<bool> {
+ self.valid()
+ }
+
  fn as_par_reader(&self, p: &dyn AsRef<Path>) -> anyhow::Result<Box<dyn DatasetExtReader + '_>> {
  use crate::reader::direct::Direct;
 

src/idx/dataset/slicer.rs

@@ -157,40 +157,49 @@ impl<'a, const D: usize> Iterator for ChunkSlice<'a, D> {
  //
  // chunk dimension will always be less or equal to the dataset
  // dimension, so we do not need to check it.
+
+ // When a chunk_dimension is size 1 it does make the chunk greater. So
+ // we ignore it. Is the case when all dimensions are 1 special?
  //
  // When all the higher chunk dimensions are size one we
  // will reach the next chunk and we can stop. If we advance to the end of the chunk. We must however advance at least one.
- if self.dataset.chunk_shape[di] == 1 {
- // if advance == 0 {
- // advance = 1;
- // }
- continue;
- }
+ // if self.dataset.chunk_shape[di] == 1 {
+ // if advance == 0 {
+ // advance = 1;
+ // }
+ //  continue;
+ // }
 
  // Assert that we have not advanced to the next chunk.
- assert_eq!(chunk, self.dataset.chunk_at_coord(&I));
+ assert_eq!(chunk, self.dataset.chunk_at_coord(&I), "advanced into next chunk");
 
+ // Assert that the coordinates are in this chunk.
  debug_assert!(
- chunk.contains(&I, &self.dataset.chunk_shape) == std::cmp::Ordering::Equal
+ chunk.contains(&I, &self.dataset.chunk_shape) == std::cmp::Ordering::Equal,
+ "coordinates are not in this chunk."
  );
 
  // End of chunk dimension.
  let chunk_d = chunk.offset[di].get() + self.dataset.chunk_shape[di];
+ debug_assert!(chunk_d <= self.dataset.shape[di]);
 
  // End of count dimension.
  let count_d = self.slice_start[di] + self.slice_counts[di];
+ debug_assert!(count_d <= self.dataset.shape[di]);
 
  let Id = I[di]; // Coordinate in current dimension of entire
  // dataset.
  let nId = min(chunk_d, count_d); // New coordinate in current
  // dimension of entire
  // dataset.
- debug_assert!(nId < self.dataset.shape[di]);
+ debug_assert!(nId <= self.dataset.shape[di], "coordinate above shape of dataset dimension.");
 
  dbg!(chunk_d);
  dbg!(count_d);
 
- assert!(nId > Id);
+ assert!(nId > Id); // XXX: This one should probably go. There might be
+ // cases when we don't advance in this dim, but in the
+ // next.
 
  let dim_sz = self.dataset.dim_sz[di];
 
@@ -223,6 +232,7 @@ impl<'a, const D: usize> Iterator for ChunkSlice<'a, D> {
  let chunk_end = chunk_start + advance;
 
  self.slice_offset += advance;
+ assert!(self.slice_offset <= self.slice_end, "advanced further than slice end.");
 
  assert!(advance > 0, "Iterator not advancing");
 
@@ -382,6 +392,8 @@ mod tests {
  )
  .unwrap();
 
+ ds.valid().unwrap();
+
  ChunkSlice::new(&ds, [0, 0, 0], [2, 32, 580]).for_each(drop);
 
  // Should be all chunks.

tests/read_norkyst.rs

@@ -57,6 +57,8 @@ fn wind() {
  let hUw = hi.reader("Uwind").unwrap().values::<i32>(None, None).unwrap();
  let hVw = hi.reader("Vwind").unwrap().values::<i32>(None, None).unwrap();
 
+ hi.dataset("Uwind").unwrap().valid().unwrap();
+
  assert_eq!(Uw, hUw);
  assert_eq!(Vw, hVw);
 }
@@ -75,6 +77,8 @@ fn current() {
 
  assert_eq!(u.len(), hi.dataset("u_eastward").unwrap().size());
 
+ // hi.dataset("u_eastward").unwrap().valid().unwrap();
+
  let hu = hi.reader("u_eastward").unwrap().values::<f32>(None, None).unwrap();
  let hv = hi.reader("v_northward").unwrap().values::<f32>(None, None).unwrap();