Lazy sst (#135)

Make shared string loading lazy and only happen when the string is accessed.

docs/ReleaseNotes.md

@@ -3,6 +3,8 @@
 _0.4.17_
 - Exclude phonetic component when reading string values.
 - Allow invalid ref values, which Excel appears to treat as missing. 
+- SharedString tables are read lazily instead of eagerly for .xlsx and .xlsb files, allowing
+ faster access to early records in some cases.
 
 _0.4.16_
 - Adds ExcelFileType class that exposes constants about supported Excel formats: extensions and content types.

source/Sylvan.Data.Excel.Tests/CustomTests.cs

@@ -143,14 +143,25 @@ public void EmptyInlineStr()
  [Fact]
  public void EmptyTrailingRow()
  {
- // If the final (or trailing) row contains a shared string referencing
- // an empty string, treat it as a null/empty value.
+ // In the case that there is a trailing row that references empty strings
+ // we will read it as a valid row. This should be uncommon enough that it won't affect anyone
+ // in practice.
 
  var reader = XlsxBuilder.Create(TestData.EmptySSTrailingRow, TestData.SharedStringEmpty);
  Assert.True(reader.Read());
  Assert.Equal(3, reader.RowFieldCount);
  Assert.Equal("a", reader.GetString(0));
  Assert.Equal("a", reader.GetString(1));
+
+ Assert.True(reader.Read());
+
+ for (int i = 0; i < reader.RowFieldCount; i++)
+ {
+ Assert.True(reader.IsDBNull(i));
+ Assert.Equal("", reader.GetString(i));
+
+ }
+
  Assert.False(reader.Read());
  }
 

source/Sylvan.Data.Excel.Tests/ExcelDataWriterTests.cs

@@ -129,7 +129,7 @@ public void Violence()
  {
  w.Write(reader);
  }
- Open(f);
+ //Open(f);
  Validate(f);
  }
 

source/Sylvan.Data.Excel.Tests/ExternalDataTests.cs

@@ -41,6 +41,47 @@ public static IEnumerable<object[]> GetInputs()
  }
  }
 
+
+ [Fact]
+ public void AnalyzeFiles()
+ {
+ var root = Environment.GetEnvironmentVariable("SylvanExcelTestData");
+ if (string.IsNullOrEmpty(root))
+ return;
+ var files = Directory.EnumerateFiles(root, "*.xlsx");
+ foreach (var file in files)
+ {
+ AnalyzeFile(file);
+ }
+ }
+
+ void AnalyzeFile(string file)
+ {
+ try
+ {
+ //using var s = File.OpenRead(file);
+ //using var za = new ZipArchive(s, ZipArchiveMode.Read);
+ var edr = ExcelDataReader.Create(file);
+ while (edr.Read())
+ {
+ for (int i = 0; i < edr.RowFieldCount; i++)
+ {
+ if (edr.GetExcelDataType(i) == ExcelDataType.String)
+ {
+ if (edr.GetString(i) == "")
+ {
+ o.WriteLine($"{Path.GetFileName(file)} {edr.RowNumber} {i}");
+ }
+ }
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ o.WriteLine($"{Path.GetFileName(file)} ERROR {e.Message}");
+ }
+ }
+
  [Fact]
  public void XmlCharRegex()
  {

source/Sylvan.Data.Excel/ExcelDataReader+FieldInfo.cs

@@ -9,7 +9,9 @@ private protected struct FieldInfo
  public static readonly FieldInfo Null = default;
 
  public ExcelDataType type;
+ public bool isSS;
  public string? strValue;
+ public int ssIdx;
  public double numValue;
  public DateTime dtValue;
  public int xfIdx;

source/Sylvan.Data.Excel/ExcelDataReader.cs

@@ -3,7 +3,6 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
-using System.ComponentModel.Design;
 using System.Data;
 using System.Data.Common;
 using System.Globalization;
@@ -857,9 +856,16 @@ public sealed override string GetString(int ordinal)
  case ExcelDataType.Numeric:
  return FormatVal(fi.xfIdx, fi.numValue);
  }
- return fi.strValue ?? string.Empty;
+ return ProcString(in fi);
  }
 
+ string ProcString(in FieldInfo fi)
+ {
+ return (fi.isSS ? GetSharedString(fi.ssIdx) : fi.strValue) ?? string.Empty;
+ }
+
+ private protected abstract string GetSharedString(int idx);
+
  string FormatVal(int xfIdx, double val)
  {
  var fmtIdx = xfIdx >= this.xfMap.Length ? -1 : this.xfMap[xfIdx];
@@ -891,7 +897,7 @@ public sealed override double GetDouble(int ordinal)
  switch (cell.type)
  {
  case ExcelDataType.String:
- return double.Parse(cell.strValue!, culture);
+ return double.Parse(ProcString(in cell), culture);
  case ExcelDataType.Numeric:
  return cell.numValue;
  case ExcelDataType.Error:
@@ -924,7 +930,7 @@ public sealed override bool GetBoolean(int ordinal)
  var trueString = col?.TrueString ?? this.trueString;
  var falseString = col?.FalseString ?? this.falseString;
 
- var strVal = fi.strValue;
+ var strVal = ProcString(in fi);
  var c = StringComparer.OrdinalIgnoreCase;
 
  if (trueString != null && c.Equals(strVal, trueString))

source/Sylvan.Data.Excel/Sylvan.Data.Excel.csproj

@@ -4,7 +4,6 @@
  <TargetFrameworks>net6.0;netstandard2.1;netstandard2.0</TargetFrameworks>
  <LangVersion>latest</LangVersion>
  <VersionPrefix>0.4.17</VersionPrefix>
- <VersionSuffix>b0003</VersionSuffix>
  <Description>A cross-platform .NET library for reading Excel data files.</Description>
  <PackageTags>excel;xls;xlsx;xlsb;datareader</PackageTags>
  <Nullable>enable</Nullable>

source/Sylvan.Data.Excel/Xls/XlsWorkbookReader.cs

@@ -94,6 +94,12 @@ public override bool Read()
  return NextRow();
  }
 
+ private protected override string GetSharedString(int idx)
+ {
+ // .xls eagerly loads the shared strings.
+ return sst[idx];
+ }
+
  public override int MaxFieldCount => 256;
 
  BOFType ReadBOF()

source/Sylvan.Data.Excel/Xlsb/XlsbWorkbookReader.cs

@@ -22,11 +22,17 @@ sealed class XlsbWorkbookReader : ExcelDataReader
  int parsedRowIndex = -1;
  int curFieldCount = -1;
 
+ readonly ZipArchiveEntry? sstPart;
+ Stream? sstStream;
+ RecordReader? sstReader;
+ int sstIdx = -1;
+
  public override ExcelWorkbookType WorkbookType => ExcelWorkbookType.ExcelXml;
 
  public override void Close()
  {
  this.sheetStream?.Close();
+ this.sstStream?.Close();
  base.Close();
  }
 
@@ -53,7 +59,7 @@ public XlsbWorkbookReader(Stream stream, ExcelDataReaderOptions opts) : base(str
 
  var stylePart = package.GetEntry(stylesPartName);
 
- sst = ReadSharedStrings(sharedStringsPartName);
+ this.sstPart = package.GetEntry(sharedStringsPartName);
 
  var sheetNameList = new List<SheetInfo>();
  using (Stream sheetsStream = workbookPart.Open())
@@ -213,41 +219,57 @@ bool InitializeSheet()
  return true;
  }
 
- string[] ReadSharedStrings(string sharedStringsPartName)
+ bool LoadSst(int idx)
  {
- var ssPart = package.GetEntry(sharedStringsPartName);
- if (ssPart == null)
+ var reader = this.sstReader;
+ if (sstPart == null)
  {
- return Array.Empty<string>();
+ return false;
  }
- using (var stream = ssPart.Open())
+ if (reader == null)
  {
- var reader = new RecordReader(stream);
-
+ this.sstStream = sstPart.Open();
+ reader = this.sstReader = new RecordReader(this.sstStream);
  reader.NextRecord();
  if (reader.RecordType != RecordType.SSTBegin)
  throw new InvalidDataException();
 
  int totalCount = reader.GetInt32(0);
  int count = reader.GetInt32(4);
 
- var ss = new string[count];
+ if (count > 128)
+ count = 128;
+ this.sst = new string[count];
+ }
+ while (idx > this.sstIdx)
+ {
+ if (!reader.NextRecord() || reader.RecordType != RecordType.SSTItem)
+ {
+ throw new InvalidDataException();
+ }
 
- for (int i = 0; i < count; i++)
+ var flags = reader.GetByte(0);
+ var str = reader.GetString(1);
+ this.sstIdx++;
+ if (sstIdx >= this.sst.Length)
  {
- reader.NextRecord();
-  if (reader.RecordType != RecordType.SSTItem)
-  {
-  reader.DebugInfo("fail");
-  throw new InvalidDataException();
-  }
+ Array.Resize(ref sst, sst.Length * 2);
+ }
+ sst[sstIdx] = str;
+ }
+ return true;
+ }
 
- var flags = reader.GetByte(0);
- var str = reader.GetString(1);
- ss[i] = str;
+ private protected override string GetSharedString(int idx)
+ {
+ if (this.sstIdx < idx)
+ {
+ if (!LoadSst(idx))
+ {
+ throw new InvalidDataException();
  }
- return ss;
  }
+ return sst[idx];
  }
 
  public override bool Read()
@@ -427,7 +449,10 @@ static void EnsureCols(ref FieldInfo[] values, int c)
  case RecordType.CellIsst:
  type = ExcelDataType.String;
  var sstIdx = reader.GetInt32(8);
- fi.strValue = sst[sstIdx];
+
+ fi.isSS = true;
+ fi.ssIdx = sstIdx;
+ //fi.strValue = sst[sstIdx];
  notNull++;
  break;
  case RecordType.CellSt:
@@ -443,7 +468,6 @@ static void EnsureCols(ref FieldInfo[] values, int c)
  break;
  }
 
-
  fi.type = type;
  fi.xfIdx = sf;
  count = col + 1;