From 05b1f1f6edc0b1ed50d824959d068f4bc224a31e Mon Sep 17 00:00:00 2001
From: Stefan Maierhofer <sm@stefanmaierhofer.com>
Date: Sun, 8 Oct 2023 15:53:39 +0200
Subject: [PATCH] add convenience function `bool
 IPointCloudNode.TryGetPartIndices(out int[]? result)` + tests

---
 src/Aardvark.Algodat.Tests/Program.cs         | 11 ++-
 .../ViewsTests/ViewsFilterTests.cs            | 38 ++++++++
 src/Aardvark.Data.E57/ASTM_E57.cs             |  6 +-
 .../Octrees/IPointCloudNode.cs                | 10 ++-
 src/Aardvark.Geometry.PointSet/Octrees/Lod.cs |  1 +
 .../Octrees/PointSetNode.cs                   | 86 ++++++++++++-------
 .../Views/FilteredNode.cs                     | 44 +++++++++-
 7 files changed, 155 insertions(+), 41 deletions(-)

diff --git a/src/Aardvark.Algodat.Tests/Program.cs b/src/Aardvark.Algodat.Tests/Program.cs
index 319bd8f0..2a131b9e 100644
--- a/src/Aardvark.Algodat.Tests/Program.cs
+++ b/src/Aardvark.Algodat.Tests/Program.cs
@@ -1751,7 +1751,7 @@ internal static void Test_Import_Regression()
             //var filenames = new[]
             //{
             //    @"W:\Datasets\Vgm\Data\E57\JBs_Haus.e57",
-            //    @"W:\Datasets\pointclouds\tests\JB_Haus_2022_KG.e57"
+            //    //@"W:\Datasets\pointclouds\tests\JB_Haus_2022_KG.e57"
             //};
 
             var filenames = Directory.GetFiles(@"W:\Datasets\pointclouds\tests");
@@ -1775,7 +1775,7 @@ internal static void Test_Import_Regression()
 
                 var key = Path.GetFileName(filename);
 
-                var storePath = $@"W:\aardvark\stores\{key}";
+                var storePath = $@"W:\aardvark\stores_noparts\{key}";
                 Directory.CreateDirectory(storePath);
                 using var storeRaw = new SimpleDiskStore(Path.Combine(storePath, "data.uds"));
                 var store = storeRaw.ToPointCloudStore();
@@ -1796,7 +1796,7 @@ internal static void Test_Import_Regression()
                     .WithMinDist(0)
                     .WithNormalizePointDensityGlobal(false)
                     //.WithProgressCallback(p => { Report.Line($"{p:0.00}"); })
-                    .WithEnabledPartIndices(true)
+                    .WithEnabledPartIndices(false)
                     ;
 
                 var pcl = PointCloud
@@ -2685,12 +2685,17 @@ static Task Parts_Test_20231006()
             using var store = new SimpleDiskStore(Path.Combine(STOREPATH, "data.uds")).ToPointCloudStore();
             var key = File.ReadAllText(Path.Combine(STOREPATH, "key.txt"));
             var ps = store.GetPointSet(key);
+            var root = ps.Root.Value;
 
             return Task.CompletedTask;
         }
 
         public static async Task Main(string[] _)
         {
+            await Task.Delay(0); // avoid warnings if main contains no await
+
+
+
             //await Parts_Test_20231006();
 
             Test_Import_Regression();
diff --git a/src/Aardvark.Algodat.Tests/ViewsTests/ViewsFilterTests.cs b/src/Aardvark.Algodat.Tests/ViewsTests/ViewsFilterTests.cs
index 67c65137..67cd9229 100644
--- a/src/Aardvark.Algodat.Tests/ViewsTests/ViewsFilterTests.cs
+++ b/src/Aardvark.Algodat.Tests/ViewsTests/ViewsFilterTests.cs
@@ -55,6 +55,7 @@ private static IPointCloudNode CreateNode(Storage storage, V3f[] psGlobal, int[]
                 .Add(Durable.Octree.PointCountTreeLeafs, psLocal.LongLength)
                 .Add(Durable.Octree.PositionsLocal3fReference, psLocalId)
                 .Add(Durable.Octree.PointRkdTreeFDataReference, kdLocalId)
+                .Add(Durable.Octree.PerCellPartIndex1ui, 42u)
                 ;
 
             if (intensities != null)
@@ -68,6 +69,19 @@ private static IPointCloudNode CreateNode(Storage storage, V3f[] psGlobal, int[]
             return result;
         }
 
+        private static void CheckPartIndices(IPointCloudNode n)
+        {
+            if (n.TryGetPartIndices(out var qs))
+            {
+                Assert.True(qs.Length == n.PointCountCell);
+                Assert.True(qs.All(q => q == 42));
+            }
+            else
+            {
+                Assert.Fail("Node has no part indices.");
+            }
+        }
+
         #region FilterInsideBox3d
 
         [Test]
@@ -80,6 +94,8 @@ public void FilterInsideBox3d_AllInside()
             Assert.IsTrue(f.HasPositions);
             var ps = f.PositionsAbsolute;
             Assert.IsTrue(ps.Length == 100);
+
+            CheckPartIndices(f);
         }
 
         [Test]
@@ -90,6 +106,8 @@ public void FilterInsideBox3d_AllOutside()
 
             var f = FilteredNode.Create(a, new FilterInsideBox3d(a.BoundingBoxExactGlobal + V3d.IOO));
             Assert.IsTrue(f.PointCountCell == 0);
+
+            CheckPartIndices(f);
         }
 
         [Test]
@@ -128,6 +146,8 @@ public void FilterInsideBox3d_Partial()
             var ps = f.PositionsAbsolute;
             var count = ps.Count(p => p.Z <= 0.5);
             Assert.IsTrue(ps.Length == count);
+
+            CheckPartIndices(f);
         }
 
         #endregion
@@ -144,6 +164,8 @@ public void FilterOutsideBox3d_AllInside()
             Assert.IsTrue(f.HasPositions);
             var ps = f.PositionsAbsolute;
             Assert.IsTrue(ps.Length == 100);
+
+            CheckPartIndices(f);
         }
 
         [Test]
@@ -154,6 +176,8 @@ public void FilterOutsideBox3d_AllOutside()
 
             var f = FilteredNode.Create(a, new FilterOutsideBox3d(a.BoundingBoxExactGlobal));
             Assert.IsTrue(f.PointCountCell == 0);
+
+            CheckPartIndices(f);
         }
 
         [Test]
@@ -167,6 +191,8 @@ public void FilterOutsideBox3d_Partial()
             var ps = f.PositionsAbsolute;
             var count = ps.Count(p => p.Z <= 0.5);
             Assert.IsTrue(ps.Length == count);
+
+            CheckPartIndices(f);
         }
 
         #endregion
@@ -184,6 +210,8 @@ public void FilterIntensity_AllInside()
             Assert.IsTrue(f.HasIntensities);
             var js = f.Intensities.Value;
             Assert.IsTrue(js.Length == 100);
+
+            CheckPartIndices(f);
         }
         
         [Test]
@@ -194,6 +222,8 @@ public void FilterIntensity_AllOutside()
 
             var f = FilteredNode.Create(a, new FilterIntensity(new Range1i(-30000, -10000)));
             Assert.IsTrue(f.PointCountCell == 0);
+
+            CheckPartIndices(f);
         }
 
         [Test]
@@ -211,6 +241,8 @@ public void FilterIntensity_Partial()
             Assert.IsTrue(js.Length == 2);
             Assert.IsTrue(js[0] == 10000);
             Assert.IsTrue(js[1] == 20000);
+
+            CheckPartIndices(f);
         }
 
         #endregion
@@ -328,10 +360,14 @@ public void EncodeDecodeRoundtrip()
             var buffer = ((IPointCloudNode)f).Encode();
             Assert.IsTrue(buffer != null);
 
+            CheckPartIndices(f);
+
             var g = FilteredNode.Decode(storage, buffer);
             Assert.IsTrue(f.Id == g.Id);
             Assert.IsTrue(f.Node.Id == g.Node.Id);
 
+            CheckPartIndices(g);
+
             var fFilterJson = f.Filter.Serialize().ToString();
             var gFilterJson = g.Filter.Serialize().ToString();
             Assert.IsTrue(fFilterJson == gFilterJson);
@@ -366,6 +402,8 @@ public void CanDeletePoints()
             Assert.IsTrue(!b.QueryAllPoints().SelectMany(chunk => chunk.Positions).Any(p => q1.Contains(p)));
 
             Assert.IsTrue(b.HasCentroidLocal);
+
+            CheckPartIndices(f);
         }
 
         #endregion
diff --git a/src/Aardvark.Data.E57/ASTM_E57.cs b/src/Aardvark.Data.E57/ASTM_E57.cs
index 4ef1d20b..3166bf3c 100644
--- a/src/Aardvark.Data.E57/ASTM_E57.cs
+++ b/src/Aardvark.Data.E57/ASTM_E57.cs
@@ -44,7 +44,7 @@ public static class ASTM_E57
         /// <summary>
         /// Physical E57 file offset.
         /// </summary>
-        public struct E57PhysicalOffset
+        public readonly struct E57PhysicalOffset
         {
             public readonly long Value;
             public E57PhysicalOffset(long value) => Value
@@ -59,7 +59,7 @@ public E57PhysicalOffset(long value) => Value
         /// <summary>
         /// Logical E57 file offset.
         /// </summary>
-        public struct E57LogicalOffset
+        public readonly struct E57LogicalOffset
         {
             public readonly long Value;
             public E57LogicalOffset(long value) { Value = value; }
@@ -2263,7 +2263,7 @@ public struct E57DataPacketHeader
             /// </summary>
             internal ushort ByteStreamCount;
             
-            internal bool CompressorRestart => (PacketFlags & 0b00000001) != 0;
+            internal readonly bool CompressorRestart => (PacketFlags & 0b00000001) != 0;
             
             internal static E57DataPacketHeader Parse(byte[] buffer) => new()
             {
diff --git a/src/Aardvark.Geometry.PointSet/Octrees/IPointCloudNode.cs b/src/Aardvark.Geometry.PointSet/Octrees/IPointCloudNode.cs
index 30e254ef..40338c41 100644
--- a/src/Aardvark.Geometry.PointSet/Octrees/IPointCloudNode.cs
+++ b/src/Aardvark.Geometry.PointSet/Octrees/IPointCloudNode.cs
@@ -244,7 +244,9 @@ public interface IPointCloudNode
 
         #region PartIndices
 
-        /// <summary></summary>
+        /// <summary>
+        /// True if this node has part indices.
+        /// </summary>
         bool HasPartIndices { get; }
 
         /// <summary>
@@ -252,6 +254,12 @@ public interface IPointCloudNode
         /// </summary>
         object? PartIndices { get; }
 
+        /// <summary>
+        /// Get per-point part indices as an int array (regardless of internal representation).
+        /// Returns false if node has no part indices.
+        /// </summary>
+        bool TryGetPartIndices([NotNullWhen(true)] out int[]? result);
+
         #endregion
 
         #region Velocities
diff --git a/src/Aardvark.Geometry.PointSet/Octrees/Lod.cs b/src/Aardvark.Geometry.PointSet/Octrees/Lod.cs
index a318326a..78ca5dee 100644
--- a/src/Aardvark.Geometry.PointSet/Octrees/Lod.cs
+++ b/src/Aardvark.Geometry.PointSet/Octrees/Lod.cs
@@ -195,6 +195,7 @@ internal static Array AggregateSubArrays(int[] counts, int splitLimit, object[]
             if (xss.All(xs => xs == null || xs is uint))
             {
                 var perCellIndices = xss.Where(xs => xs != null).Select(xs => (uint)xs!).ToArray();
+                if (perCellIndices.Length == 0) return null;
                 var allIdentical = true;
                 for (var i = 1; i < perCellIndices.Length; i++) if (perCellIndices[i] != perCellIndices[0]) { allIdentical = false; break; }
                 if (allIdentical) return perCellIndices[0];
diff --git a/src/Aardvark.Geometry.PointSet/Octrees/PointSetNode.cs b/src/Aardvark.Geometry.PointSet/Octrees/PointSetNode.cs
index b45cabd8..f1a4e0f6 100644
--- a/src/Aardvark.Geometry.PointSet/Octrees/PointSetNode.cs
+++ b/src/Aardvark.Geometry.PointSet/Octrees/PointSetNode.cs
@@ -506,16 +506,16 @@ public PointSetNode WriteToStore()
 
 #endregion
 
-#region Properties (state to serialize)
+        #region Properties (state to serialize)
 
         /// <summary>
         /// Durable properties.
         /// </summary>
         private ImmutableDictionary<Durable.Def, object> Data { get; } = ImmutableDictionary<Durable.Def, object>.Empty;
 
-#endregion
+        #endregion
 
-#region Properties (derived/runtime, non-serialized)
+        #region Properties (derived/runtime, non-serialized)
 
         /// <summary>
         /// Runtime.
@@ -529,7 +529,7 @@ public PointSetNode WriteToStore()
         [JsonIgnore]
         private Dictionary<Durable.Def, object> PersistentRefs { get; } = new Dictionary<Durable.Def, object>();
 
-#region Cell attributes
+        #region Cell attributes
 
         /// <summary>
         /// This node's unique id (16 bytes).
@@ -582,9 +582,9 @@ public Guid?[]? SubnodeIds
             }
         }
 
-#endregion
+        #endregion
 
-#region Positions
+        #region Positions
 
         /// <summary></summary>
         [JsonIgnore]
@@ -681,9 +681,9 @@ public PersistentRef<V3f[]> Positions
         [JsonIgnore]
         public V3d[] PositionsAbsolute => Positions?.Value.Map(p => new V3d(Center.X + p.X, Center.Y + p.Y, Center.Z + p.Z))!;
 
-#endregion
+        #endregion
 
-#region BoundingBoxExactLocal
+        #region BoundingBoxExactLocal
 
         /// <summary></summary>
         [JsonIgnore]
@@ -693,9 +693,9 @@ public PersistentRef<V3f[]> Positions
         [JsonIgnore]
         public Box3f BoundingBoxExactLocal => Data.TryGetValue(Durable.Octree.BoundingBoxExactLocal, out var o) ? (Box3f)o : default;
 
-#endregion
+        #endregion
 
-#region BoundingBoxExactGlobal
+        #region BoundingBoxExactGlobal
 
         /// <summary></summary>
         [JsonIgnore]
@@ -705,9 +705,9 @@ public PersistentRef<V3f[]> Positions
         [JsonIgnore]
         public Box3d BoundingBoxExactGlobal => Data.TryGetValue(Durable.Octree.BoundingBoxExactGlobal, out var o) ? (Box3d)o : default;
 
-#endregion
+        #endregion
 
-#region Colors
+        #region Colors
 
         /// <summary></summary>
         [JsonIgnore]
@@ -778,9 +778,9 @@ public PersistentRef<C4b[]>? Colors
             }
         }
 
-#endregion
+        #endregion
 
-#region Normals
+        #region Normals
 
         /// <summary></summary>
         [JsonIgnore]
@@ -817,9 +817,9 @@ public PersistentRef<V3f[]>? Normals
             }
         }
 
-#endregion
+        #endregion
 
-#region Intensities
+        #region Intensities
 
         /// <summary></summary>
         [JsonIgnore]
@@ -856,9 +856,9 @@ public PersistentRef<int[]>? Intensities
             }
         }
 
-#endregion
+        #endregion
 
-#region Classifications
+        #region Classifications
 
         /// <summary></summary>
         [JsonIgnore]
@@ -899,9 +899,11 @@ public PersistentRef<byte[]>? Classifications
 
         #region PartIndices
 
-        /// <summary></summary>
-        [JsonIgnore]
 
+        /// <summary>
+        /// True if this node has part indices.
+        /// </summary>
+        [JsonIgnore]
         [MemberNotNullWhen(true, nameof(PartIndices))]
         public bool HasPartIndices =>
             Data.ContainsKey(Durable.Octree.PerCellPartIndex1ui) ||
@@ -926,6 +928,26 @@ public object? PartIndices
             }
         }
 
+        /// <summary>
+        /// Get per-point part indices as an int array (regardless of internal representation).
+        /// Returns false if node has no part indices.
+        /// </summary>
+        public bool TryGetPartIndices([NotNullWhen(true)] out int[]? result)
+        {
+            switch (PartIndices)
+            {
+                case null: result = null; return false;
+                case uint x: checked { result = new int[PointCountCell].Set((int)x); return true; }
+                case byte[] xs: result = xs.Map(x => (int)x); return true;
+                case short[] xs: result = xs.Map(x => (int)x); return true;
+                case int[] xs: result = xs; return true;
+                default: throw new Exception(
+                    $"Unexpected type {PartIndices.GetType().FullName}. " +
+                    $"Error 278c88f6-d504-4a17-9752-8cca614505f1."
+                    );
+            }
+        }
+
         #endregion
 
         #region Velocities
@@ -948,7 +970,7 @@ public object? PartIndices
 
 #endregion
 
-#region KdTree
+        #region KdTree
 
         /// <summary></summary>
         [JsonIgnore]
@@ -975,9 +997,9 @@ public object? PartIndices
             : null
             ;
 
-#endregion
+        #endregion
 
-#region CentroidLocal
+        #region CentroidLocal
 
         /// <summary></summary>
         public bool HasCentroidLocal => PointCountCell > 0 || Data.ContainsKey(Durable.Octree.PositionsLocal3fCentroid);
@@ -1048,9 +1070,9 @@ public float CentroidLocalStdDev
             }
         }
 
-#endregion
+        #endregion
 
-#region TreeDepth
+        #region TreeDepth
 
             /// <summary></summary>
         public bool HasMinTreeDepth => Data.ContainsKey(Durable.Octree.MinTreeDepth);
@@ -1064,9 +1086,9 @@ public float CentroidLocalStdDev
         /// <summary></summary>
         public int MaxTreeDepth => (int)Data.Get(Durable.Octree.MaxTreeDepth);
 
-#endregion
+        #endregion
 
-#region PointDistance
+        #region PointDistance
 
         /// <summary></summary>
         public bool HasPointDistanceAverage => Data.ContainsKey(Durable.Octree.AveragePointDistance);
@@ -1080,7 +1102,7 @@ public float CentroidLocalStdDev
         /// <summary></summary>
         public float PointDistanceStandardDeviation => (Data.TryGetValue(Durable.Octree.AveragePointDistanceStdDev, out var value) && value is float x) ? x : -1.0f;
 
-#endregion
+        #endregion
 
         /// <summary>
         /// Subnodes (8), or null if leaf.
@@ -1230,7 +1252,7 @@ private static Guid ComputeAndStoreKdTree(Storage storage, V3f[] ps)
 
 #endregion
 
-#region Durable codec
+        #region Durable codec
 
         /// <summary>
         /// </summary>
@@ -1246,9 +1268,9 @@ public static PointSetNode Decode(Storage storage, byte[] buffer)
             return new PointSetNode(data, storage, false);
         }
 
-#endregion
+        #endregion
 
-#region IPointCloudNode
+        #region IPointCloudNode
 
         Guid IPointCloudNode.Id => Id;
 
@@ -1305,6 +1327,6 @@ public Box3d BoundingBoxApproximate
 
         IPointCloudNode IPointCloudNode.With(IReadOnlyDictionary<Durable.Def, object> replacements) => With(replacements);
 
-#endregion
+        #endregion
     }
 }
diff --git a/src/Aardvark.Geometry.PointSet/Views/FilteredNode.cs b/src/Aardvark.Geometry.PointSet/Views/FilteredNode.cs
index e3fc6cf5..12cc22a2 100644
--- a/src/Aardvark.Geometry.PointSet/Views/FilteredNode.cs
+++ b/src/Aardvark.Geometry.PointSet/Views/FilteredNode.cs
@@ -438,14 +438,38 @@ public PersistentRef<PointRkdTreeF<V3f[], V3f>> KdTree
 
         #region PartIndices
 
-        /// <summary></summary>
+        /// <summary>
+        /// True if this node has part indices.
+        /// </summary>
         [MemberNotNullWhen(true, nameof(PartIndices))]
         public bool HasPartIndices => Node.HasPartIndices;
 
         /// <summary>
         /// Octree. Per-point or per-cell part indices.
         /// </summary>
-        public object? PartIndices => Node.PartIndices;
+        public object? PartIndices => PartIndexUtils.Subset(Node.PartIndices, null!);
+
+        /// <summary>
+        /// Get per-point part indices as an int array (regardless of internal representation).
+        /// Returns false if node has no part indices.
+        /// </summary>
+        public bool TryGetPartIndices([NotNullWhen(true)] out int[]? result)
+        {
+            var qs = SubsetIndexArray != null ? PartIndexUtils.Subset(PartIndices, SubsetIndexArray) : PartIndices;
+            switch (qs)
+            {
+                case null: result = null; return false;
+                case uint x: checked { result = new int[PointCountCell].Set((int)x); return true; }
+                case byte[] xs: result = xs.Map(x => (int)x); return true;
+                case short[] xs: result = xs.Map(x => (int)x); return true;
+                case int[] xs: result = xs; return true;
+                default:
+                    throw new Exception(
+                    $"Unexpected type {qs.GetType().FullName}. " +
+                    $"Error ccc0b898-fe4f-4373-ac15-42da763fe5ab."
+                    );
+            }
+        }
 
         #endregion
 
@@ -521,6 +545,7 @@ public PersistentRef<PointRkdTreeF<V3f[], V3f>> KdTree
         #endregion
 
         private readonly Dictionary<Guid, object> m_cache = new();
+
         private PersistentRef<T[]>? GetSubArray<T>(Def def, PersistentRef<T[]>? originalValue)
         {
             if (m_cache.TryGetValue(def.Id, out var o) && o is PersistentRef<T[]> x) return x;
@@ -536,6 +561,21 @@ public PersistentRef<PointRkdTreeF<V3f[], V3f>> KdTree
             return result;
         }
 
+        private int[]? _subsetIndexArray = null;
+        private int[]? SubsetIndexArray
+        {
+            get
+            {
+                if (_subsetIndexArray != null) return _subsetIndexArray;
+                if (m_activePoints == null) return null;
+
+                var imax = PointCountCell;
+                var xs = new List<int>();
+                for (var i = 0; i < imax; i++) if (m_activePoints.Contains(i)) xs.Add(i);
+                return _subsetIndexArray = xs.ToArray();
+            }
+        }
+
         #endregion
 
         #region Not supported ...