[e57] support for ScaledInteger with 33 to 63 bits (1 to 32, and 64 w…

…as already supported)

src/Aardvark.Algodat.Tests/Program.cs

@@ -1788,13 +1788,13 @@ internal static void TestFilterSerialization()
 
         internal static void Test_Import_Regression()
         {
-            var filenames = new[]
-            {
-                //@"W:\Datasets\Vgm\Data\E57\JBs_Haus.e57",
-                @"W:\Datasets\pointclouds\tests\JB_Haus_2022_KG.e57"
-            };
+            //var filenames = new[]
+            //{
+            //    //@"W:\Datasets\Vgm\Data\E57\JBs_Haus.e57",
+            //    @"W:\Datasets\pointclouds\tests\JB_Haus_2022_KG.e57"
+            //};
 
-            //var filenames = Directory.GetFiles(@"W:\Datasets\pointclouds\tests");
+            var filenames = Directory.GetFiles(@"W:\Datasets\pointclouds\tests");
 
             //filenames = Directory
             //    .EnumerateFiles(@"W:\Datasets\plytest", "*.ply", SearchOption.AllDirectories)
@@ -2761,6 +2761,23 @@ V3d[] randomPoints(int n, Box3d bb)
 
         public static async Task Main(string[] _)
         {
+            //{
+            //    var chunks = E57.Chunks(@"W:\Datasets\Vgm\Data\2024-04-30_bugreport\F_240205.e57", ParseConfig.Default);
+            //    var i = 0;
+            //    foreach (var chunk in chunks)
+            //    {
+            //        var bb = new Box3d(chunk.Positions);
+            //        WriteLine($"[{i++}] {bb}");
+            //    }
+            //    return;
+            //}
+
+            //await CreateStore(
+            //    @"C:\Data\F_240205.e57",
+            //    @"t:\tmp\20240501_aardvark",
+            //    minDist: 0.005
+            //    );
+
             //await Task.Delay(0); // avoid warnings if main contains no await
 
             //await CreateStore(
@@ -2769,17 +2786,17 @@ public static async Task Main(string[] _)
             //    minDist: 0.001
             //    );
 
-            await CreateStore(
-                @"W:\Datasets\Vgm\Data\E57\erdgeschoss.e57",
-                @"W:\Datasets\Vgm\Stores\erdgeschoss.e57_0.0025",
-                minDist: 0.0025
-                );
+            //await CreateStore(
+            //    @"W:\Datasets\Vgm\Data\E57\erdgeschoss.e57",
+            //    @"W:\Datasets\Vgm\Stores\erdgeschoss.e57_0.0025",
+            //    minDist: 0.0025
+            //    );
 
             //await Parts_Test_20231006_Merge();
 
             //await Parts_Test_20231006();
 
-            //Test_Import_Regression();
+            Test_Import_Regression();
 
             //await Ranges_Test_20230802();
 

src/Aardvark.Data.E57/ASTM_E57.cs

@@ -255,6 +255,9 @@ public class E57ScaledInteger : IE57Element, IBitPack
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
             public double Compute(uint rawValue) => (Minimum.HasValue ? (Minimum.Value + rawValue) : rawValue) * Scale + Offset;
 
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            public double ComputeL(ulong rawValue) => (Minimum.HasValue ? (Minimum.Value + (long)rawValue) : rawValue) * Scale + Offset;
+
             #endregion
 
             internal static E57ScaledInteger Parse(XElement root)
@@ -805,9 +808,18 @@ static double[] UnpackScaledInteger(byte[] buffer, BitPacker packer, E57ScaledIn
                 {
                     checked
                     {
-                        var xs = packer.UnpackUInts(buffer);
-                        var rs = xs.Map(proto.Compute);
-                        return rs;
+                        if (packer.BitsPerValue <= 32)
+                        {
+                            var xs = packer.UnpackUInts(buffer);
+                            var rs = xs.Map(proto.Compute);
+                            return rs;
+                        }
+                        else
+                        {
+                            var xs = packer.UnpackULongs(buffer);
+                            var rs = xs.Map(proto.ComputeL);
+                            return rs;
+                        }
                     }
                 }
                 static Array UnpackIntegers(byte[] buffer, BitPacker packer, E57Integer proto)

src/Aardvark.Data.E57/BitPack.cs

@@ -20,22 +20,27 @@ namespace Aardvark.Base
     public class BitPacker
     {
         private uint m_rest = 0;
+        private ulong m_restL = 0ul;
         private int m_restBitCount = 0;
         /// <summary></summary>
         public int BitsPerValue { get; }
 
         /// <summary></summary>
         public BitPacker(int bitsPerValue)
         {
-            if (bitsPerValue < 0 || (bitsPerValue > 32 && bitsPerValue != 64)) throw new ArgumentOutOfRangeException(
-                nameof(bitsPerValue), $"BitsPerValue must be [1,32] but is {bitsPerValue}");
+            if (bitsPerValue < 0 || bitsPerValue > 64) throw new ArgumentOutOfRangeException(
+                nameof(bitsPerValue), $"BitsPerValue must be in range [1,64] but is {bitsPerValue}. Invariant 7944df56-ad1d-4140-bdd1-2d52d11540b1.");
 
             BitsPerValue = bitsPerValue;
         }
 
         /// <summary></summary>
         public uint[] UnpackUInts(byte[] buffer)
         {
+            if (BitsPerValue > 32) throw new Exception(
+                $"Calling BitPack.UnpackUInts(...) for BitsPerValue={BitsPerValue} is not possible. Call BitPack.UnpackULongs(...) instead. Error 63098b8c-15a0-4ad2-8325-a927ea74e075."
+                );
+
             var count = (m_restBitCount + buffer.Length * 8) / BitsPerValue;
             var result = new uint[count];
             var bufferByteIndex = 0;
@@ -87,6 +92,65 @@ public uint[] UnpackUInts(byte[] buffer)
 
             return result;
         }
+
+        /// <summary></summary>
+        public ulong[] UnpackULongs(byte[] buffer)
+        {
+            if (BitsPerValue <= 32) throw new Exception(
+                $"Calling BitPack.UnpackULongs(...) for BitsPerValue={BitsPerValue} is a bad idea. Call BitPack.UnpackUInts(...) instead. Error 18582197-9403-4c0c-9db3-5b99f6ae2091."
+                );
+
+            var count = (m_restBitCount + buffer.Length * 8) / BitsPerValue;
+            var result = new ulong[count];
+            var bufferByteIndex = 0;
+            for (var i = 0; i < count; i++)
+            {
+                if (m_restBitCount >= BitsPerValue)
+                {
+                    result[i] = m_restL & ((1ul << BitsPerValue) - 1);
+                    m_restL >>= BitsPerValue;
+                    m_restBitCount -= BitsPerValue;
+                    continue;
+                }
+
+                var value = 0ul;
+                var valueBitIndex = 0;
+
+                // init value with m_restL
+                if (m_restBitCount > 0) { value = m_restL; valueBitIndex = m_restBitCount; m_restL = 0; m_restBitCount = 0; }
+
+                // add full byte(s) to value
+                while (valueBitIndex + 8 <= BitsPerValue)
+                {
+                    value |= (ulong)buffer[bufferByteIndex++] << valueBitIndex;
+                    valueBitIndex += 8;
+                }
+                var numberOfBitsUntilCompletion = BitsPerValue - valueBitIndex;
+#if DEBUG
+                if (numberOfBitsUntilCompletion < 0 || numberOfBitsUntilCompletion > 7) throw new InvalidOperationException();
+#endif
+
+                // if value has been filled up with latest full byte, then we are done
+                if (numberOfBitsUntilCompletion == 0) { result[i] = value; continue; }
+
+                // ... else we use a part of the next byte to fill remaining bits
+                var b = (ulong)buffer[bufferByteIndex++];
+                value |= (b & ((1ul << numberOfBitsUntilCompletion) - 1)) << valueBitIndex;
+                result[i] = value;
+                // ... and save remaining bits of current byte to m_rest
+                m_restBitCount = 8 - numberOfBitsUntilCompletion;
+                m_restL = b >> numberOfBitsUntilCompletion;
+            }
+
+            // save trailing bytes to m_rest ...
+            while (bufferByteIndex < buffer.Length)
+            {
+                m_restL |= (ulong)buffer[bufferByteIndex++] << m_restBitCount;
+                m_restBitCount += 8;
+            }
+
+            return result;
+        }
     }
 
     /// <summary></summary>

src/Aardvark.Data.E57/ImportE57.cs

@@ -195,6 +195,12 @@ public static IEnumerable<Chunk> Chunks(this Stream stream, long streamLengthInB
                 var yieldedRecordCount = 0L;
 
                 var partIndex = config.PartIndexOffset;
+
+                foreach (var data3d in header.E57Root.Data3D)
+                {
+                    Console.WriteLine($"[Data3D] {data3d.Name} {data3d.Points.ByteStreamsCount}");
+                }
+
                 foreach (var data3d in header.E57Root.Data3D)
                 {
                     foreach (var (Positions, Properties) in data3d.StreamPointsFull(config.MaxChunkPointCount, config.Verbose, exclude))

src/Apps/Viewer/Program.fs

@@ -100,7 +100,7 @@ let main args =
     //view @"C:\Users\sm\Downloads\C_30DN2.LAZ.store" [File.readAllText @"C:\Users\sm\Downloads\C_30DN2.LAZ.key"] (Args.parse [||])
     //view @"C:\Users\sm\Downloads\test.store" ["128330b1-8761-4a07-b160-76bcd7e2f70a"; "ab2f6f76-7eae-47c9-82d1-ad28b816abb9"] (Args.parse [||])
 
-    let store = @"E:\e57tests\stores\inference_full.binary.ply\data.uds"
+    let store = @"W:\Datasets\Vgm\Stores\2024-04-30_bugreport\F_240205.e57\data.uds"
     let key = Path.combine [System.IO.Path.GetDirectoryName store;"key.txt"] |> File.readAllText
     view store [key] (Args.parse [||])